drivers/infiniband/hw/bnxt_re/ib_verbs.c

   1 /*
   2  * Broadcom NetXtreme-E RoCE driver.
   3  *
   4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * BSD license below:
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in
  21  *    the documentation and/or other materials provided with the
  22  *    distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Description: IB Verbs interpreter
  37  */
  38
  39 #include <linux/interrupt.h>
  40 #include <linux/types.h>
  41 #include <linux/pci.h>
  42 #include <linux/netdevice.h>
  43 #include <linux/if_ether.h>
  44
  45 #include <rdma/ib_verbs.h>
  46 #include <rdma/ib_user_verbs.h>
  47 #include <rdma/ib_umem.h>
  48 #include <rdma/ib_addr.h>
  49 #include <rdma/ib_mad.h>
  50 #include <rdma/ib_cache.h>
  51 #include <rdma/uverbs_ioctl.h>
  52
  53 #include "bnxt_ulp.h"
  54
  55 #include "roce_hsi.h"
  56 #include "qplib_res.h"
  57 #include "qplib_sp.h"
  58 #include "qplib_fp.h"
  59 #include "qplib_rcfw.h"
  60
  61 #include "bnxt_re.h"
  62 #include "ib_verbs.h"
  63 #include <rdma/bnxt_re-abi.h>
  64
  65 static int __from_ib_access_flags(int iflags)
  66 {
  67         int qflags = 0;
  68
  69         if (iflags & IB_ACCESS_LOCAL_WRITE)
  70                 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
  71         if (iflags & IB_ACCESS_REMOTE_READ)
  72                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
  73         if (iflags & IB_ACCESS_REMOTE_WRITE)
  74                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
  75         if (iflags & IB_ACCESS_REMOTE_ATOMIC)
  76                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
  77         if (iflags & IB_ACCESS_MW_BIND)
  78                 qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
  79         if (iflags & IB_ZERO_BASED)
  80                 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
  81         if (iflags & IB_ACCESS_ON_DEMAND)
  82                 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
  83         return qflags;
  84 };
  85
  86 static enum ib_access_flags __to_ib_access_flags(int qflags)
  87 {
  88         enum ib_access_flags iflags = 0;
  89
  90         if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
  91                 iflags |= IB_ACCESS_LOCAL_WRITE;
  92         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
  93                 iflags |= IB_ACCESS_REMOTE_WRITE;
  94         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
  95                 iflags |= IB_ACCESS_REMOTE_READ;
  96         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
  97                 iflags |= IB_ACCESS_REMOTE_ATOMIC;
  98         if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
  99                 iflags |= IB_ACCESS_MW_BIND;
 100         if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
 101                 iflags |= IB_ZERO_BASED;
 102         if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
 103                 iflags |= IB_ACCESS_ON_DEMAND;
 104         return iflags;
 105 };
 106
 107 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
 108                              struct bnxt_qplib_sge *sg_list, int num)
 109 {
 110         int i, total = 0;
 111
 112         for (i = 0; i < num; i++) {
 113                 sg_list[i].addr = ib_sg_list[i].addr;
 114                 sg_list[i].lkey = ib_sg_list[i].lkey;
 115                 sg_list[i].size = ib_sg_list[i].length;
 116                 total += sg_list[i].size;
 117         }
 118         return total;
 119 }
 120
 121 /* Device */
 122 int bnxt_re_query_device(struct ib_device *ibdev,
 123                          struct ib_device_attr *ib_attr,
 124                          struct ib_udata *udata)
 125 {
 126         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 127         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
 128
 129         memset(ib_attr, 0, sizeof(*ib_attr));
 130         memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
 131                min(sizeof(dev_attr->fw_ver),
 132                    sizeof(ib_attr->fw_ver)));
 133         bnxt_qplib_get_guid(rdev->netdev->dev_addr,
 134                             (u8 *)&ib_attr->sys_image_guid);
 135         ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
 136         ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M;
 137
 138         ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
 139         ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
 140         ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
 141         ib_attr->max_qp = dev_attr->max_qp;
 142         ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
 143         ib_attr->device_cap_flags =
 144                                     IB_DEVICE_CURR_QP_STATE_MOD
 145                                     | IB_DEVICE_RC_RNR_NAK_GEN
 146                                     | IB_DEVICE_SHUTDOWN_PORT
 147                                     | IB_DEVICE_SYS_IMAGE_GUID
 148                                     | IB_DEVICE_LOCAL_DMA_LKEY
 149                                     | IB_DEVICE_RESIZE_MAX_WR
 150                                     | IB_DEVICE_PORT_ACTIVE_EVENT
 151                                     | IB_DEVICE_N_NOTIFY_CQ
 152                                     | IB_DEVICE_MEM_WINDOW
 153                                     | IB_DEVICE_MEM_WINDOW_TYPE_2B
 154                                     | IB_DEVICE_MEM_MGT_EXTENSIONS;
 155         ib_attr->max_send_sge = dev_attr->max_qp_sges;
 156         ib_attr->max_recv_sge = dev_attr->max_qp_sges;
 157         ib_attr->max_sge_rd = dev_attr->max_qp_sges;
 158         ib_attr->max_cq = dev_attr->max_cq;
 159         ib_attr->max_cqe = dev_attr->max_cq_wqes;
 160         ib_attr->max_mr = dev_attr->max_mr;
 161         ib_attr->max_pd = dev_attr->max_pd;
 162         ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
 163         ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
 164         ib_attr->atomic_cap = IB_ATOMIC_NONE;
 165         ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
 166         if (dev_attr->is_atomic) {
 167                 ib_attr->atomic_cap = IB_ATOMIC_GLOB;
 168                 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
 169         }
 170
 171         ib_attr->max_ee_rd_atom = 0;
 172         ib_attr->max_res_rd_atom = 0;
 173         ib_attr->max_ee_init_rd_atom = 0;
 174         ib_attr->max_ee = 0;
 175         ib_attr->max_rdd = 0;
 176         ib_attr->max_mw = dev_attr->max_mw;
 177         ib_attr->max_raw_ipv6_qp = 0;
 178         ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
 179         ib_attr->max_mcast_grp = 0;
 180         ib_attr->max_mcast_qp_attach = 0;
 181         ib_attr->max_total_mcast_qp_attach = 0;
 182         ib_attr->max_ah = dev_attr->max_ah;
 183
 184         ib_attr->max_srq = dev_attr->max_srq;
 185         ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
 186         ib_attr->max_srq_sge = dev_attr->max_srq_sges;
 187
 188         ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
 189
 190         ib_attr->max_pkeys = 1;
 191         ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
 192         return 0;
 193 }
 194
 195 /* Port */
 196 int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,
 197                        struct ib_port_attr *port_attr)
 198 {
 199         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 200         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
 201
 202         memset(port_attr, 0, sizeof(*port_attr));
 203
 204         if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
 205                 port_attr->state = IB_PORT_ACTIVE;
 206                 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 207         } else {
 208                 port_attr->state = IB_PORT_DOWN;
 209                 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 210         }
 211         port_attr->max_mtu = IB_MTU_4096;
 212         port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
 213         port_attr->gid_tbl_len = dev_attr->max_sgid;
 214         port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 215                                     IB_PORT_DEVICE_MGMT_SUP |
 216                                     IB_PORT_VENDOR_CLASS_SUP;
 217         port_attr->ip_gids = true;
 218
 219         port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
 220         port_attr->bad_pkey_cntr = 0;
 221         port_attr->qkey_viol_cntr = 0;
 222         port_attr->pkey_tbl_len = dev_attr->max_pkey;
 223         port_attr->lid = 0;
 224         port_attr->sm_lid = 0;
 225         port_attr->lmc = 0;
 226         port_attr->max_vl_num = 4;
 227         port_attr->sm_sl = 0;
 228         port_attr->subnet_timeout = 0;
 229         port_attr->init_type_reply = 0;
 230         port_attr->active_speed = rdev->active_speed;
 231         port_attr->active_width = rdev->active_width;
 232
 233         return 0;
 234 }
 235
 236 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
 237                                struct ib_port_immutable *immutable)
 238 {
 239         struct ib_port_attr port_attr;
 240
 241         if (bnxt_re_query_port(ibdev, port_num, &port_attr))
 242                 return -EINVAL;
 243
 244         immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
 245         immutable->gid_tbl_len = port_attr.gid_tbl_len;
 246         immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
 247         immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
 248         immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 249         return 0;
 250 }
 251
 252 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)
 253 {
 254         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 255
 256         snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d",
 257                  rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1],
 258                  rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]);
 259 }
 260
 261 int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,
 262                        u16 index, u16 *pkey)
 263 {
 264         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 265
 266         /* Ignore port_num */
 267
 268         memset(pkey, 0, sizeof(*pkey));
 269         return bnxt_qplib_get_pkey(&rdev->qplib_res,
 270                                    &rdev->qplib_res.pkey_tbl, index, pkey);
 271 }
 272
 273 int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,
 274                       int index, union ib_gid *gid)
 275 {
 276         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 277         int rc = 0;
 278
 279         /* Ignore port_num */
 280         memset(gid, 0, sizeof(*gid));
 281         rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
 282                                  &rdev->qplib_res.sgid_tbl, index,
 283                                  (struct bnxt_qplib_gid *)gid);
 284         return rc;
 285 }
 286
 287 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)
 288 {
 289         int rc = 0;
 290         struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
 291         struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 292         struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 293         struct bnxt_qplib_gid *gid_to_del;
 294         u16 vlan_id = 0xFFFF;
 295
 296         /* Delete the entry from the hardware */
 297         ctx = *context;
 298         if (!ctx)
 299                 return -EINVAL;
 300
 301         if (sgid_tbl && sgid_tbl->active) {
 302                 if (ctx->idx >= sgid_tbl->max)
 303                         return -EINVAL;
 304                 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
 305                 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
 306                 /* DEL_GID is called in WQ context(netdevice_event_work_handler)
 307                  * or via the ib_unregister_device path. In the former case QP1
 308                  * may not be destroyed yet, in which case just return as FW
 309                  * needs that entry to be present and will fail it's deletion.
 310                  * We could get invoked again after QP1 is destroyed OR get an
 311                  * ADD_GID call with a different GID value for the same index
 312                  * where we issue MODIFY_GID cmd to update the GID entry -- TBD
 313                  */
 314                 if (ctx->idx == 0 &&
 315                     rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
 316                     ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) {
 317                         ibdev_dbg(&rdev->ibdev,
 318                                   "Trying to delete GID0 while QP1 is alive\n");
 319                         return -EFAULT;
 320                 }
 321                 ctx->refcnt--;
 322                 if (!ctx->refcnt) {
 323                         rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
 324                                                  vlan_id,  true);
 325                         if (rc) {
 326                                 ibdev_err(&rdev->ibdev,
 327                                           "Failed to remove GID: %#x", rc);
 328                         } else {
 329                                 ctx_tbl = sgid_tbl->ctx;
 330                                 ctx_tbl[ctx->idx] = NULL;
 331                                 kfree(ctx);
 332                         }
 333                 }
 334         } else {
 335                 return -EINVAL;
 336         }
 337         return rc;
 338 }
 339
 340 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)
 341 {
 342         int rc;
 343         u32 tbl_idx = 0;
 344         u16 vlan_id = 0xFFFF;
 345         struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
 346         struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 347         struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 348
 349         rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL);
 350         if (rc)
 351                 return rc;
 352
 353         rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid,
 354                                  rdev->qplib_res.netdev->dev_addr,
 355                                  vlan_id, true, &tbl_idx);
 356         if (rc == -EALREADY) {
 357                 ctx_tbl = sgid_tbl->ctx;
 358                 ctx_tbl[tbl_idx]->refcnt++;
 359                 *context = ctx_tbl[tbl_idx];
 360                 return 0;
 361         }
 362
 363         if (rc < 0) {
 364                 ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc);
 365                 return rc;
 366         }
 367
 368         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 369         if (!ctx)
 370                 return -ENOMEM;
 371         ctx_tbl = sgid_tbl->ctx;
 372         ctx->idx = tbl_idx;
 373         ctx->refcnt = 1;
 374         ctx_tbl[tbl_idx] = ctx;
 375         *context = ctx;
 376
 377         return rc;
 378 }
 379
 380 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
 381                                             u32 port_num)
 382 {
 383         return IB_LINK_LAYER_ETHERNET;
 384 }
 385
 386 #define BNXT_RE_FENCE_PBL_SIZE  DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE)
 387
 388 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)
 389 {
 390         struct bnxt_re_fence_data *fence = &pd->fence;
 391         struct ib_mr *ib_mr = &fence->mr->ib_mr;
 392         struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
 393
 394         memset(wqe, 0, sizeof(*wqe));
 395         wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
 396         wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
 397         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
 398         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
 399         wqe->bind.zero_based = false;
 400         wqe->bind.parent_l_key = ib_mr->lkey;
 401         wqe->bind.va = (u64)(unsigned long)fence->va;
 402         wqe->bind.length = fence->size;
 403         wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
 404         wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
 405
 406         /* Save the initial rkey in fence structure for now;
 407          * wqe->bind.r_key will be set at (re)bind time.
 408          */
 409         fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
 410 }
 411
 412 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
 413 {
 414         struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
 415                                              qplib_qp);
 416         struct ib_pd *ib_pd = qp->ib_qp.pd;
 417         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 418         struct bnxt_re_fence_data *fence = &pd->fence;
 419         struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
 420         struct bnxt_qplib_swqe wqe;
 421         int rc;
 422
 423         memcpy(&wqe, fence_wqe, sizeof(wqe));
 424         wqe.bind.r_key = fence->bind_rkey;
 425         fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
 426
 427         ibdev_dbg(&qp->rdev->ibdev,
 428                   "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
 429                 wqe.bind.r_key, qp->qplib_qp.id, pd);
 430         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
 431         if (rc) {
 432                 ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n");
 433                 return rc;
 434         }
 435         bnxt_qplib_post_send_db(&qp->qplib_qp);
 436
 437         return rc;
 438 }
 439
 440 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)
 441 {
 442         struct bnxt_re_fence_data *fence = &pd->fence;
 443         struct bnxt_re_dev *rdev = pd->rdev;
 444         struct device *dev = &rdev->en_dev->pdev->dev;
 445         struct bnxt_re_mr *mr = fence->mr;
 446
 447         if (fence->mw) {
 448                 bnxt_re_dealloc_mw(fence->mw);
 449                 fence->mw = NULL;
 450         }
 451         if (mr) {
 452                 if (mr->ib_mr.rkey)
 453                         bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
 454                                              true);
 455                 if (mr->ib_mr.lkey)
 456                         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
 457                 kfree(mr);
 458                 fence->mr = NULL;
 459         }
 460         if (fence->dma_addr) {
 461                 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES,
 462                                  DMA_BIDIRECTIONAL);
 463                 fence->dma_addr = 0;
 464         }
 465 }
 466
 467 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
 468 {
 469         int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
 470         struct bnxt_re_fence_data *fence = &pd->fence;
 471         struct bnxt_re_dev *rdev = pd->rdev;
 472         struct device *dev = &rdev->en_dev->pdev->dev;
 473         struct bnxt_re_mr *mr = NULL;
 474         dma_addr_t dma_addr = 0;
 475         struct ib_mw *mw;
 476         int rc;
 477
 478         dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
 479                                   DMA_BIDIRECTIONAL);
 480         rc = dma_mapping_error(dev, dma_addr);
 481         if (rc) {
 482                 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n");
 483                 rc = -EIO;
 484                 fence->dma_addr = 0;
 485                 goto fail;
 486         }
 487         fence->dma_addr = dma_addr;
 488
 489         /* Allocate a MR */
 490         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 491         if (!mr) {
 492                 rc = -ENOMEM;
 493                 goto fail;
 494         }
 495         fence->mr = mr;
 496         mr->rdev = rdev;
 497         mr->qplib_mr.pd = &pd->qplib_pd;
 498         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
 499         mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
 500         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
 501         if (rc) {
 502                 ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n");
 503                 goto fail;
 504         }
 505
 506         /* Register MR */
 507         mr->ib_mr.lkey = mr->qplib_mr.lkey;
 508         mr->qplib_mr.va = (u64)(unsigned long)fence->va;
 509         mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
 510         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
 511                                BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE);
 512         if (rc) {
 513                 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
 514                 goto fail;
 515         }
 516         mr->ib_mr.rkey = mr->qplib_mr.rkey;
 517
 518         /* Create a fence MW only for kernel consumers */
 519         mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
 520         if (IS_ERR(mw)) {
 521                 ibdev_err(&rdev->ibdev,
 522                           "Failed to create fence-MW for PD: %p\n", pd);
 523                 rc = PTR_ERR(mw);
 524                 goto fail;
 525         }
 526         fence->mw = mw;
 527
 528         bnxt_re_create_fence_wqe(pd);
 529         return 0;
 530
 531 fail:
 532         bnxt_re_destroy_fence_mr(pd);
 533         return rc;
 534 }
 535
 536 /* Protection Domains */
 537 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
 538 {
 539         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 540         struct bnxt_re_dev *rdev = pd->rdev;
 541
 542         bnxt_re_destroy_fence_mr(pd);
 543
 544         if (pd->qplib_pd.id)
 545                 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
 546                                       &pd->qplib_pd);
 547         return 0;
 548 }
 549
 550 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 551 {
 552         struct ib_device *ibdev = ibpd->device;
 553         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 554         struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(
 555                 udata, struct bnxt_re_ucontext, ib_uctx);
 556         struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);
 557         int rc;
 558
 559         pd->rdev = rdev;
 560         if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) {
 561                 ibdev_err(&rdev->ibdev, "Failed to allocate HW PD");
 562                 rc = -ENOMEM;
 563                 goto fail;
 564         }
 565
 566         if (udata) {
 567                 struct bnxt_re_pd_resp resp;
 568
 569                 if (!ucntx->dpi.dbr) {
 570                         /* Allocate DPI in alloc_pd to avoid failing of
 571                          * ibv_devinfo and family of application when DPIs
 572                          * are depleted.
 573                          */
 574                         if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl,
 575                                                  &ucntx->dpi, ucntx)) {
 576                                 rc = -ENOMEM;
 577                                 goto dbfail;
 578                         }
 579                 }
 580
 581                 resp.pdid = pd->qplib_pd.id;
 582                 /* Still allow mapping this DBR to the new user PD. */
 583                 resp.dpi = ucntx->dpi.dpi;
 584                 resp.dbr = (u64)ucntx->dpi.umdbr;
 585
 586                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
 587                 if (rc) {
 588                         ibdev_err(&rdev->ibdev,
 589                                   "Failed to copy user response\n");
 590                         goto dbfail;
 591                 }
 592         }
 593
 594         if (!udata)
 595                 if (bnxt_re_create_fence_mr(pd))
 596                         ibdev_warn(&rdev->ibdev,
 597                                    "Failed to create Fence-MR\n");
 598         return 0;
 599 dbfail:
 600         bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
 601                               &pd->qplib_pd);
 602 fail:
 603         return rc;
 604 }
 605
 606 /* Address Handles */
 607 int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
 608 {
 609         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 610         struct bnxt_re_dev *rdev = ah->rdev;
 611
 612         bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah,
 613                               !(flags & RDMA_DESTROY_AH_SLEEPABLE));
 614         return 0;
 615 }
 616
 617 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
 618 {
 619         u8 nw_type;
 620
 621         switch (ntype) {
 622         case RDMA_NETWORK_IPV4:
 623                 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
 624                 break;
 625         case RDMA_NETWORK_IPV6:
 626                 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
 627                 break;
 628         default:
 629                 nw_type = CMDQ_CREATE_AH_TYPE_V1;
 630                 break;
 631         }
 632         return nw_type;
 633 }
 634
 635 int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
 636                       struct ib_udata *udata)
 637 {
 638         struct ib_pd *ib_pd = ib_ah->pd;
 639         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 640         struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 641         const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
 642         struct bnxt_re_dev *rdev = pd->rdev;
 643         const struct ib_gid_attr *sgid_attr;
 644         struct bnxt_re_gid_ctx *ctx;
 645         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 646         u8 nw_type;
 647         int rc;
 648
 649         if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
 650                 ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set");
 651                 return -EINVAL;
 652         }
 653
 654         ah->rdev = rdev;
 655         ah->qplib_ah.pd = &pd->qplib_pd;
 656
 657         /* Supply the configuration for the HW */
 658         memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
 659                sizeof(union ib_gid));
 660         sgid_attr = grh->sgid_attr;
 661         /* Get the HW context of the GID. The reference
 662          * of GID table entry is already taken by the caller.
 663          */
 664         ctx = rdma_read_gid_hw_context(sgid_attr);
 665         ah->qplib_ah.sgid_index = ctx->idx;
 666         ah->qplib_ah.host_sgid_index = grh->sgid_index;
 667         ah->qplib_ah.traffic_class = grh->traffic_class;
 668         ah->qplib_ah.flow_label = grh->flow_label;
 669         ah->qplib_ah.hop_limit = grh->hop_limit;
 670         ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
 671
 672         /* Get network header type for this GID */
 673         nw_type = rdma_gid_attr_network_type(sgid_attr);
 674         ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type);
 675
 676         memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
 677         rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah,
 678                                   !(init_attr->flags &
 679                                     RDMA_CREATE_AH_SLEEPABLE));
 680         if (rc) {
 681                 ibdev_err(&rdev->ibdev, "Failed to allocate HW AH");
 682                 return rc;
 683         }
 684
 685         /* Write AVID to shared page. */
 686         if (udata) {
 687                 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
 688                         udata, struct bnxt_re_ucontext, ib_uctx);
 689                 unsigned long flag;
 690                 u32 *wrptr;
 691
 692                 spin_lock_irqsave(&uctx->sh_lock, flag);
 693                 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
 694                 *wrptr = ah->qplib_ah.id;
 695                 wmb(); /* make sure cache is updated. */
 696                 spin_unlock_irqrestore(&uctx->sh_lock, flag);
 697         }
 698
 699         return 0;
 700 }
 701
 702 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 703 {
 704         return 0;
 705 }
 706
 707 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 708 {
 709         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 710
 711         ah_attr->type = ib_ah->type;
 712         rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
 713         memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
 714         rdma_ah_set_grh(ah_attr, NULL, 0,
 715                         ah->qplib_ah.host_sgid_index,
 716                         0, ah->qplib_ah.traffic_class);
 717         rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data);
 718         rdma_ah_set_port_num(ah_attr, 1);
 719         rdma_ah_set_static_rate(ah_attr, 0);
 720         return 0;
 721 }
 722
 723 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
 724         __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
 725 {
 726         unsigned long flags;
 727
 728         spin_lock_irqsave(&qp->scq->cq_lock, flags);
 729         if (qp->rcq != qp->scq)
 730                 spin_lock(&qp->rcq->cq_lock);
 731         else
 732                 __acquire(&qp->rcq->cq_lock);
 733
 734         return flags;
 735 }
 736
 737 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
 738                         unsigned long flags)
 739         __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
 740 {
 741         if (qp->rcq != qp->scq)
 742                 spin_unlock(&qp->rcq->cq_lock);
 743         else
 744                 __release(&qp->rcq->cq_lock);
 745         spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
 746 }
 747
 748 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
 749 {
 750         struct bnxt_re_qp *gsi_sqp;
 751         struct bnxt_re_ah *gsi_sah;
 752         struct bnxt_re_dev *rdev;
 753         int rc = 0;
 754
 755         rdev = qp->rdev;
 756         gsi_sqp = rdev->gsi_ctx.gsi_sqp;
 757         gsi_sah = rdev->gsi_ctx.gsi_sah;
 758
 759         ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n");
 760         bnxt_qplib_destroy_ah(&rdev->qplib_res,
 761                               &gsi_sah->qplib_ah,
 762                               true);
 763         bnxt_qplib_clean_qp(&qp->qplib_qp);
 764
 765         ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n");
 766         rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
 767         if (rc) {
 768                 ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed");
 769                 goto fail;
 770         }
 771         bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
 772
 773         /* remove from active qp list */
 774         mutex_lock(&rdev->qp_lock);
 775         list_del(&gsi_sqp->list);
 776         mutex_unlock(&rdev->qp_lock);
 777         atomic_dec(&rdev->qp_count);
 778
 779         kfree(rdev->gsi_ctx.sqp_tbl);
 780         kfree(gsi_sah);
 781         kfree(gsi_sqp);
 782         rdev->gsi_ctx.gsi_sqp = NULL;
 783         rdev->gsi_ctx.gsi_sah = NULL;
 784         rdev->gsi_ctx.sqp_tbl = NULL;
 785
 786         return 0;
 787 fail:
 788         return rc;
 789 }
 790
 791 /* Queue Pairs */
 792 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 793 {
 794         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 795         struct bnxt_re_dev *rdev = qp->rdev;
 796         unsigned int flags;
 797         int rc;
 798
 799         bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
 800
 801         rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
 802         if (rc) {
 803                 ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
 804                 return rc;
 805         }
 806
 807         if (rdma_is_kernel_res(&qp->ib_qp.res)) {
 808                 flags = bnxt_re_lock_cqs(qp);
 809                 bnxt_qplib_clean_qp(&qp->qplib_qp);
 810                 bnxt_re_unlock_cqs(qp, flags);
 811         }
 812
 813         bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
 814
 815         if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) {
 816                 rc = bnxt_re_destroy_gsi_sqp(qp);
 817                 if (rc)
 818                         return rc;
 819         }
 820
 821         mutex_lock(&rdev->qp_lock);
 822         list_del(&qp->list);
 823         mutex_unlock(&rdev->qp_lock);
 824         atomic_dec(&rdev->qp_count);
 825
 826         ib_umem_release(qp->rumem);
 827         ib_umem_release(qp->sumem);
 828
 829         return 0;
 830 }
 831
 832 static u8 __from_ib_qp_type(enum ib_qp_type type)
 833 {
 834         switch (type) {
 835         case IB_QPT_GSI:
 836                 return CMDQ_CREATE_QP1_TYPE_GSI;
 837         case IB_QPT_RC:
 838                 return CMDQ_CREATE_QP_TYPE_RC;
 839         case IB_QPT_UD:
 840                 return CMDQ_CREATE_QP_TYPE_UD;
 841         default:
 842                 return IB_QPT_MAX;
 843         }
 844 }
 845
 846 static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
 847                                    int rsge, int max)
 848 {
 849         if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
 850                 rsge = max;
 851         return bnxt_re_get_rwqe_size(rsge);
 852 }
 853
 854 static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
 855 {
 856         u16 wqe_size, calc_ils;
 857
 858         wqe_size = bnxt_re_get_swqe_size(nsge);
 859         if (ilsize) {
 860                 calc_ils = sizeof(struct sq_send_hdr) + ilsize;
 861                 wqe_size = max_t(u16, calc_ils, wqe_size);
 862                 wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
 863         }
 864         return wqe_size;
 865 }
 866
 867 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
 868                                    struct ib_qp_init_attr *init_attr)
 869 {
 870         struct bnxt_qplib_dev_attr *dev_attr;
 871         struct bnxt_qplib_qp *qplqp;
 872         struct bnxt_re_dev *rdev;
 873         struct bnxt_qplib_q *sq;
 874         int align, ilsize;
 875
 876         rdev = qp->rdev;
 877         qplqp = &qp->qplib_qp;
 878         sq = &qplqp->sq;
 879         dev_attr = &rdev->dev_attr;
 880
 881         align = sizeof(struct sq_send_hdr);
 882         ilsize = ALIGN(init_attr->cap.max_inline_data, align);
 883
 884         sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge);
 885         if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges))
 886                 return -EINVAL;
 887         /* For gen p4 and gen p5 backward compatibility mode
 888          * wqe size is fixed to 128 bytes
 889          */
 890         if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) &&
 891                         qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
 892                 sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges);
 893
 894         if (init_attr->cap.max_inline_data) {
 895                 qplqp->max_inline_data = sq->wqe_size -
 896                         sizeof(struct sq_send_hdr);
 897                 init_attr->cap.max_inline_data = qplqp->max_inline_data;
 898                 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
 899                         sq->max_sge = qplqp->max_inline_data /
 900                                 sizeof(struct sq_sge);
 901         }
 902
 903         return 0;
 904 }
 905
 906 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
 907                                 struct bnxt_re_qp *qp, struct ib_udata *udata)
 908 {
 909         struct bnxt_qplib_qp *qplib_qp;
 910         struct bnxt_re_ucontext *cntx;
 911         struct bnxt_re_qp_req ureq;
 912         int bytes = 0, psn_sz;
 913         struct ib_umem *umem;
 914         int psn_nume;
 915
 916         qplib_qp = &qp->qplib_qp;
 917         cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
 918                                          ib_uctx);
 919         if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
 920                 return -EFAULT;
 921
 922         bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
 923         /* Consider mapping PSN search memory only for RC QPs. */
 924         if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
 925                 psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
 926                                                    sizeof(struct sq_psn_search_ext) :
 927                                                    sizeof(struct sq_psn_search);
 928                 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
 929                             qplib_qp->sq.max_wqe :
 930                             ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
 931                               sizeof(struct bnxt_qplib_sge));
 932                 bytes += (psn_nume * psn_sz);
 933         }
 934
 935         bytes = PAGE_ALIGN(bytes);
 936         umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes,
 937                            IB_ACCESS_LOCAL_WRITE);
 938         if (IS_ERR(umem))
 939                 return PTR_ERR(umem);
 940
 941         qp->sumem = umem;
 942         qplib_qp->sq.sg_info.umem = umem;
 943         qplib_qp->sq.sg_info.pgsize = PAGE_SIZE;
 944         qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT;
 945         qplib_qp->qp_handle = ureq.qp_handle;
 946
 947         if (!qp->qplib_qp.srq) {
 948                 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
 949                 bytes = PAGE_ALIGN(bytes);
 950                 umem = ib_umem_get(&rdev->ibdev, ureq.qprva, bytes,
 951                                    IB_ACCESS_LOCAL_WRITE);
 952                 if (IS_ERR(umem))
 953                         goto rqfail;
 954                 qp->rumem = umem;
 955                 qplib_qp->rq.sg_info.umem = umem;
 956                 qplib_qp->rq.sg_info.pgsize = PAGE_SIZE;
 957                 qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT;
 958         }
 959
 960         qplib_qp->dpi = &cntx->dpi;
 961         return 0;
 962 rqfail:
 963         ib_umem_release(qp->sumem);
 964         qp->sumem = NULL;
 965         memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
 966
 967         return PTR_ERR(umem);
 968 }
 969
 970 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
 971                                 (struct bnxt_re_pd *pd,
 972                                  struct bnxt_qplib_res *qp1_res,
 973                                  struct bnxt_qplib_qp *qp1_qp)
 974 {
 975         struct bnxt_re_dev *rdev = pd->rdev;
 976         struct bnxt_re_ah *ah;
 977         union ib_gid sgid;
 978         int rc;
 979
 980         ah = kzalloc(sizeof(*ah), GFP_KERNEL);
 981         if (!ah)
 982                 return NULL;
 983
 984         ah->rdev = rdev;
 985         ah->qplib_ah.pd = &pd->qplib_pd;
 986
 987         rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
 988         if (rc)
 989                 goto fail;
 990
 991         /* supply the dgid data same as sgid */
 992         memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
 993                sizeof(union ib_gid));
 994         ah->qplib_ah.sgid_index = 0;
 995
 996         ah->qplib_ah.traffic_class = 0;
 997         ah->qplib_ah.flow_label = 0;
 998         ah->qplib_ah.hop_limit = 1;
 999         ah->qplib_ah.sl = 0;
1000         /* Have DMAC same as SMAC */
1001         ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
1002
1003         rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false);
1004         if (rc) {
1005                 ibdev_err(&rdev->ibdev,
1006                           "Failed to allocate HW AH for Shadow QP");
1007                 goto fail;
1008         }
1009
1010         return ah;
1011
1012 fail:
1013         kfree(ah);
1014         return NULL;
1015 }
1016
1017 static struct bnxt_re_qp *bnxt_re_create_shadow_qp
1018                                 (struct bnxt_re_pd *pd,
1019                                  struct bnxt_qplib_res *qp1_res,
1020                                  struct bnxt_qplib_qp *qp1_qp)
1021 {
1022         struct bnxt_re_dev *rdev = pd->rdev;
1023         struct bnxt_re_qp *qp;
1024         int rc;
1025
1026         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1027         if (!qp)
1028                 return NULL;
1029
1030         qp->rdev = rdev;
1031
1032         /* Initialize the shadow QP structure from the QP1 values */
1033         ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
1034
1035         qp->qplib_qp.pd = &pd->qplib_pd;
1036         qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
1037         qp->qplib_qp.type = IB_QPT_UD;
1038
1039         qp->qplib_qp.max_inline_data = 0;
1040         qp->qplib_qp.sig_type = true;
1041
1042         /* Shadow QP SQ depth should be same as QP1 RQ depth */
1043         qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);
1044         qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
1045         qp->qplib_qp.sq.max_sge = 2;
1046         /* Q full delta can be 1 since it is internal QP */
1047         qp->qplib_qp.sq.q_full_delta = 1;
1048         qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE;
1049         qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT;
1050
1051         qp->qplib_qp.scq = qp1_qp->scq;
1052         qp->qplib_qp.rcq = qp1_qp->rcq;
1053
1054         qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);
1055         qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
1056         qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
1057         /* Q full delta can be 1 since it is internal QP */
1058         qp->qplib_qp.rq.q_full_delta = 1;
1059         qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE;
1060         qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT;
1061
1062         qp->qplib_qp.mtu = qp1_qp->mtu;
1063
1064         qp->qplib_qp.sq_hdr_buf_size = 0;
1065         qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
1066         qp->qplib_qp.dpi = &rdev->dpi_privileged;
1067
1068         rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
1069         if (rc)
1070                 goto fail;
1071
1072         spin_lock_init(&qp->sq_lock);
1073         INIT_LIST_HEAD(&qp->list);
1074         mutex_lock(&rdev->qp_lock);
1075         list_add_tail(&qp->list, &rdev->qp_list);
1076         atomic_inc(&rdev->qp_count);
1077         mutex_unlock(&rdev->qp_lock);
1078         return qp;
1079 fail:
1080         kfree(qp);
1081         return NULL;
1082 }
1083
1084 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
1085                                 struct ib_qp_init_attr *init_attr)
1086 {
1087         struct bnxt_qplib_dev_attr *dev_attr;
1088         struct bnxt_qplib_qp *qplqp;
1089         struct bnxt_re_dev *rdev;
1090         struct bnxt_qplib_q *rq;
1091         int entries;
1092
1093         rdev = qp->rdev;
1094         qplqp = &qp->qplib_qp;
1095         rq = &qplqp->rq;
1096         dev_attr = &rdev->dev_attr;
1097
1098         if (init_attr->srq) {
1099                 struct bnxt_re_srq *srq;
1100
1101                 srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq);
1102                 qplqp->srq = &srq->qplib_srq;
1103                 rq->max_wqe = 0;
1104         } else {
1105                 rq->max_sge = init_attr->cap.max_recv_sge;
1106                 if (rq->max_sge > dev_attr->max_qp_sges)
1107                         rq->max_sge = dev_attr->max_qp_sges;
1108                 init_attr->cap.max_recv_sge = rq->max_sge;
1109                 rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
1110                                                        dev_attr->max_qp_sges);
1111                 /* Allocate 1 more than what's provided so posting max doesn't
1112                  * mean empty.
1113                  */
1114                 entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1);
1115                 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1116                 rq->q_full_delta = 0;
1117                 rq->sg_info.pgsize = PAGE_SIZE;
1118                 rq->sg_info.pgshft = PAGE_SHIFT;
1119         }
1120
1121         return 0;
1122 }
1123
1124 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
1125 {
1126         struct bnxt_qplib_dev_attr *dev_attr;
1127         struct bnxt_qplib_qp *qplqp;
1128         struct bnxt_re_dev *rdev;
1129
1130         rdev = qp->rdev;
1131         qplqp = &qp->qplib_qp;
1132         dev_attr = &rdev->dev_attr;
1133
1134         if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1135                 qplqp->rq.max_sge = dev_attr->max_qp_sges;
1136                 if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
1137                         qplqp->rq.max_sge = dev_attr->max_qp_sges;
1138                 qplqp->rq.max_sge = 6;
1139         }
1140 }
1141
1142 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
1143                                 struct ib_qp_init_attr *init_attr,
1144                                 struct ib_udata *udata)
1145 {
1146         struct bnxt_qplib_dev_attr *dev_attr;
1147         struct bnxt_qplib_qp *qplqp;
1148         struct bnxt_re_dev *rdev;
1149         struct bnxt_qplib_q *sq;
1150         int entries;
1151         int diff;
1152         int rc;
1153
1154         rdev = qp->rdev;
1155         qplqp = &qp->qplib_qp;
1156         sq = &qplqp->sq;
1157         dev_attr = &rdev->dev_attr;
1158
1159         sq->max_sge = init_attr->cap.max_send_sge;
1160         if (sq->max_sge > dev_attr->max_qp_sges) {
1161                 sq->max_sge = dev_attr->max_qp_sges;
1162                 init_attr->cap.max_send_sge = sq->max_sge;
1163         }
1164
1165         rc = bnxt_re_setup_swqe_size(qp, init_attr);
1166         if (rc)
1167                 return rc;
1168
1169         entries = init_attr->cap.max_send_wr;
1170         /* Allocate 128 + 1 more than what's provided */
1171         diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
1172                 0 : BNXT_QPLIB_RESERVED_QP_WRS;
1173         entries = roundup_pow_of_two(entries + diff + 1);
1174         sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
1175         sq->q_full_delta = diff + 1;
1176         /*
1177          * Reserving one slot for Phantom WQE. Application can
1178          * post one extra entry in this case. But allowing this to avoid
1179          * unexpected Queue full condition
1180          */
1181         qplqp->sq.q_full_delta -= 1;
1182         qplqp->sq.sg_info.pgsize = PAGE_SIZE;
1183         qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
1184
1185         return 0;
1186 }
1187
1188 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
1189                                        struct ib_qp_init_attr *init_attr)
1190 {
1191         struct bnxt_qplib_dev_attr *dev_attr;
1192         struct bnxt_qplib_qp *qplqp;
1193         struct bnxt_re_dev *rdev;
1194         int entries;
1195
1196         rdev = qp->rdev;
1197         qplqp = &qp->qplib_qp;
1198         dev_attr = &rdev->dev_attr;
1199
1200         if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1201                 entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1);
1202                 qplqp->sq.max_wqe = min_t(u32, entries,
1203                                           dev_attr->max_qp_wqes + 1);
1204                 qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
1205                         init_attr->cap.max_send_wr;
1206                 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
1207                 if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
1208                         qplqp->sq.max_sge = dev_attr->max_qp_sges;
1209         }
1210 }
1211
1212 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
1213                                 struct ib_qp_init_attr *init_attr)
1214 {
1215         struct bnxt_qplib_chip_ctx *chip_ctx;
1216         int qptype;
1217
1218         chip_ctx = rdev->chip_ctx;
1219
1220         qptype = __from_ib_qp_type(init_attr->qp_type);
1221         if (qptype == IB_QPT_MAX) {
1222                 ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype);
1223                 qptype = -EOPNOTSUPP;
1224                 goto out;
1225         }
1226
1227         if (bnxt_qplib_is_chip_gen_p5(chip_ctx) &&
1228             init_attr->qp_type == IB_QPT_GSI)
1229                 qptype = CMDQ_CREATE_QP_TYPE_GSI;
1230 out:
1231         return qptype;
1232 }
1233
1234 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1235                                 struct ib_qp_init_attr *init_attr,
1236                                 struct ib_udata *udata)
1237 {
1238         struct bnxt_qplib_dev_attr *dev_attr;
1239         struct bnxt_qplib_qp *qplqp;
1240         struct bnxt_re_dev *rdev;
1241         struct bnxt_re_cq *cq;
1242         int rc = 0, qptype;
1243
1244         rdev = qp->rdev;
1245         qplqp = &qp->qplib_qp;
1246         dev_attr = &rdev->dev_attr;
1247
1248         /* Setup misc params */
1249         ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr);
1250         qplqp->pd = &pd->qplib_pd;
1251         qplqp->qp_handle = (u64)qplqp;
1252         qplqp->max_inline_data = init_attr->cap.max_inline_data;
1253         qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ?
1254                             true : false);
1255         qptype = bnxt_re_init_qp_type(rdev, init_attr);
1256         if (qptype < 0) {
1257                 rc = qptype;
1258                 goto out;
1259         }
1260         qplqp->type = (u8)qptype;
1261         qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode;
1262
1263         if (init_attr->qp_type == IB_QPT_RC) {
1264                 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
1265                 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1266         }
1267         qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1268         qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
1269         if (init_attr->create_flags) {
1270                 ibdev_dbg(&rdev->ibdev,
1271                           "QP create flags 0x%x not supported",
1272                           init_attr->create_flags);
1273                 return -EOPNOTSUPP;
1274         }
1275
1276         /* Setup CQs */
1277         if (init_attr->send_cq) {
1278                 cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq);
1279                 qplqp->scq = &cq->qplib_cq;
1280                 qp->scq = cq;
1281         }
1282
1283         if (init_attr->recv_cq) {
1284                 cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq);
1285                 qplqp->rcq = &cq->qplib_cq;
1286                 qp->rcq = cq;
1287         }
1288
1289         /* Setup RQ/SRQ */
1290         rc = bnxt_re_init_rq_attr(qp, init_attr);
1291         if (rc)
1292                 goto out;
1293         if (init_attr->qp_type == IB_QPT_GSI)
1294                 bnxt_re_adjust_gsi_rq_attr(qp);
1295
1296         /* Setup SQ */
1297         rc = bnxt_re_init_sq_attr(qp, init_attr, udata);
1298         if (rc)
1299                 goto out;
1300         if (init_attr->qp_type == IB_QPT_GSI)
1301                 bnxt_re_adjust_gsi_sq_attr(qp, init_attr);
1302
1303         if (udata) /* This will update DPI and qp_handle */
1304                 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
1305 out:
1306         return rc;
1307 }
1308
1309 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
1310                                      struct bnxt_re_pd *pd)
1311 {
1312         struct bnxt_re_sqp_entries *sqp_tbl;
1313         struct bnxt_re_dev *rdev;
1314         struct bnxt_re_qp *sqp;
1315         struct bnxt_re_ah *sah;
1316         int rc = 0;
1317
1318         rdev = qp->rdev;
1319         /* Create a shadow QP to handle the QP1 traffic */
1320         sqp_tbl = kcalloc(BNXT_RE_MAX_GSI_SQP_ENTRIES, sizeof(*sqp_tbl),
1321                           GFP_KERNEL);
1322         if (!sqp_tbl)
1323                 return -ENOMEM;
1324         rdev->gsi_ctx.sqp_tbl = sqp_tbl;
1325
1326         sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
1327         if (!sqp) {
1328                 rc = -ENODEV;
1329                 ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1");
1330                 goto out;
1331         }
1332         rdev->gsi_ctx.gsi_sqp = sqp;
1333
1334         sqp->rcq = qp->rcq;
1335         sqp->scq = qp->scq;
1336         sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
1337                                           &qp->qplib_qp);
1338         if (!sah) {
1339                 bnxt_qplib_destroy_qp(&rdev->qplib_res,
1340                                       &sqp->qplib_qp);
1341                 rc = -ENODEV;
1342                 ibdev_err(&rdev->ibdev,
1343                           "Failed to create AH entry for ShadowQP");
1344                 goto out;
1345         }
1346         rdev->gsi_ctx.gsi_sah = sah;
1347
1348         return 0;
1349 out:
1350         kfree(sqp_tbl);
1351         return rc;
1352 }
1353
1354 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1355                                  struct ib_qp_init_attr *init_attr)
1356 {
1357         struct bnxt_re_dev *rdev;
1358         struct bnxt_qplib_qp *qplqp;
1359         int rc = 0;
1360
1361         rdev = qp->rdev;
1362         qplqp = &qp->qplib_qp;
1363
1364         qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1365         qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
1366
1367         rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp);
1368         if (rc) {
1369                 ibdev_err(&rdev->ibdev, "create HW QP1 failed!");
1370                 goto out;
1371         }
1372
1373         rc = bnxt_re_create_shadow_gsi(qp, pd);
1374 out:
1375         return rc;
1376 }
1377
1378 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
1379                                    struct ib_qp_init_attr *init_attr,
1380                                    struct bnxt_qplib_dev_attr *dev_attr)
1381 {
1382         bool rc = true;
1383
1384         if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes ||
1385             init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes ||
1386             init_attr->cap.max_send_sge > dev_attr->max_qp_sges ||
1387             init_attr->cap.max_recv_sge > dev_attr->max_qp_sges ||
1388             init_attr->cap.max_inline_data > dev_attr->max_inline_data) {
1389                 ibdev_err(&rdev->ibdev,
1390                           "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",
1391                           init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
1392                           init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
1393                           init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
1394                           init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
1395                           init_attr->cap.max_inline_data,
1396                           dev_attr->max_inline_data);
1397                 rc = false;
1398         }
1399         return rc;
1400 }
1401
1402 int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
1403                       struct ib_udata *udata)
1404 {
1405         struct ib_pd *ib_pd = ib_qp->pd;
1406         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1407         struct bnxt_re_dev *rdev = pd->rdev;
1408         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1409         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1410         int rc;
1411
1412         rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
1413         if (!rc) {
1414                 rc = -EINVAL;
1415                 goto fail;
1416         }
1417
1418         qp->rdev = rdev;
1419         rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata);
1420         if (rc)
1421                 goto fail;
1422
1423         if (qp_init_attr->qp_type == IB_QPT_GSI &&
1424             !(bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))) {
1425                 rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr);
1426                 if (rc == -ENODEV)
1427                         goto qp_destroy;
1428                 if (rc)
1429                         goto fail;
1430         } else {
1431                 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1432                 if (rc) {
1433                         ibdev_err(&rdev->ibdev, "Failed to create HW QP");
1434                         goto free_umem;
1435                 }
1436                 if (udata) {
1437                         struct bnxt_re_qp_resp resp;
1438
1439                         resp.qpid = qp->qplib_qp.id;
1440                         resp.rsvd = 0;
1441                         rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1442                         if (rc) {
1443                                 ibdev_err(&rdev->ibdev, "Failed to copy QP udata");
1444                                 goto qp_destroy;
1445                         }
1446                 }
1447         }
1448
1449         qp->ib_qp.qp_num = qp->qplib_qp.id;
1450         if (qp_init_attr->qp_type == IB_QPT_GSI)
1451                 rdev->gsi_ctx.gsi_qp = qp;
1452         spin_lock_init(&qp->sq_lock);
1453         spin_lock_init(&qp->rq_lock);
1454         INIT_LIST_HEAD(&qp->list);
1455         mutex_lock(&rdev->qp_lock);
1456         list_add_tail(&qp->list, &rdev->qp_list);
1457         mutex_unlock(&rdev->qp_lock);
1458         atomic_inc(&rdev->qp_count);
1459
1460         return 0;
1461 qp_destroy:
1462         bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1463 free_umem:
1464         ib_umem_release(qp->rumem);
1465         ib_umem_release(qp->sumem);
1466 fail:
1467         return rc;
1468 }
1469
1470 static u8 __from_ib_qp_state(enum ib_qp_state state)
1471 {
1472         switch (state) {
1473         case IB_QPS_RESET:
1474                 return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1475         case IB_QPS_INIT:
1476                 return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1477         case IB_QPS_RTR:
1478                 return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1479         case IB_QPS_RTS:
1480                 return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1481         case IB_QPS_SQD:
1482                 return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1483         case IB_QPS_SQE:
1484                 return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1485         case IB_QPS_ERR:
1486         default:
1487                 return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1488         }
1489 }
1490
1491 static enum ib_qp_state __to_ib_qp_state(u8 state)
1492 {
1493         switch (state) {
1494         case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1495                 return IB_QPS_RESET;
1496         case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1497                 return IB_QPS_INIT;
1498         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1499                 return IB_QPS_RTR;
1500         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1501                 return IB_QPS_RTS;
1502         case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1503                 return IB_QPS_SQD;
1504         case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1505                 return IB_QPS_SQE;
1506         case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1507         default:
1508                 return IB_QPS_ERR;
1509         }
1510 }
1511
1512 static u32 __from_ib_mtu(enum ib_mtu mtu)
1513 {
1514         switch (mtu) {
1515         case IB_MTU_256:
1516                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1517         case IB_MTU_512:
1518                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1519         case IB_MTU_1024:
1520                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1521         case IB_MTU_2048:
1522                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1523         case IB_MTU_4096:
1524                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1525         default:
1526                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1527         }
1528 }
1529
1530 static enum ib_mtu __to_ib_mtu(u32 mtu)
1531 {
1532         switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1533         case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1534                 return IB_MTU_256;
1535         case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1536                 return IB_MTU_512;
1537         case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1538                 return IB_MTU_1024;
1539         case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1540                 return IB_MTU_2048;
1541         case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1542                 return IB_MTU_4096;
1543         default:
1544                 return IB_MTU_2048;
1545         }
1546 }
1547
1548 /* Shared Receive Queues */
1549 int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
1550 {
1551         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1552                                                ib_srq);
1553         struct bnxt_re_dev *rdev = srq->rdev;
1554         struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1555         struct bnxt_qplib_nq *nq = NULL;
1556
1557         if (qplib_srq->cq)
1558                 nq = qplib_srq->cq->nq;
1559         bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
1560         ib_umem_release(srq->umem);
1561         atomic_dec(&rdev->srq_count);
1562         if (nq)
1563                 nq->budget--;
1564         return 0;
1565 }
1566
1567 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
1568                                  struct bnxt_re_pd *pd,
1569                                  struct bnxt_re_srq *srq,
1570                                  struct ib_udata *udata)
1571 {
1572         struct bnxt_re_srq_req ureq;
1573         struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1574         struct ib_umem *umem;
1575         int bytes = 0;
1576         struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
1577                 udata, struct bnxt_re_ucontext, ib_uctx);
1578
1579         if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1580                 return -EFAULT;
1581
1582         bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
1583         bytes = PAGE_ALIGN(bytes);
1584         umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes,
1585                            IB_ACCESS_LOCAL_WRITE);
1586         if (IS_ERR(umem))
1587                 return PTR_ERR(umem);
1588
1589         srq->umem = umem;
1590         qplib_srq->sg_info.umem = umem;
1591         qplib_srq->sg_info.pgsize = PAGE_SIZE;
1592         qplib_srq->sg_info.pgshft = PAGE_SHIFT;
1593         qplib_srq->srq_handle = ureq.srq_handle;
1594         qplib_srq->dpi = &cntx->dpi;
1595
1596         return 0;
1597 }
1598
1599 int bnxt_re_create_srq(struct ib_srq *ib_srq,
1600                        struct ib_srq_init_attr *srq_init_attr,
1601                        struct ib_udata *udata)
1602 {
1603         struct bnxt_qplib_dev_attr *dev_attr;
1604         struct bnxt_qplib_nq *nq = NULL;
1605         struct bnxt_re_dev *rdev;
1606         struct bnxt_re_srq *srq;
1607         struct bnxt_re_pd *pd;
1608         struct ib_pd *ib_pd;
1609         int rc, entries;
1610
1611         ib_pd = ib_srq->pd;
1612         pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1613         rdev = pd->rdev;
1614         dev_attr = &rdev->dev_attr;
1615         srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);
1616
1617         if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
1618                 ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded");
1619                 rc = -EINVAL;
1620                 goto exit;
1621         }
1622
1623         if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
1624                 rc = -EOPNOTSUPP;
1625                 goto exit;
1626         }
1627
1628         srq->rdev = rdev;
1629         srq->qplib_srq.pd = &pd->qplib_pd;
1630         srq->qplib_srq.dpi = &rdev->dpi_privileged;
1631         /* Allocate 1 more than what's provided so posting max doesn't
1632          * mean empty
1633          */
1634         entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
1635         if (entries > dev_attr->max_srq_wqes + 1)
1636                 entries = dev_attr->max_srq_wqes + 1;
1637         srq->qplib_srq.max_wqe = entries;
1638
1639         srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
1640          /* 128 byte wqe size for SRQ . So use max sges */
1641         srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);
1642         srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
1643         srq->srq_limit = srq_init_attr->attr.srq_limit;
1644         srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
1645         nq = &rdev->nq[0];
1646
1647         if (udata) {
1648                 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
1649                 if (rc)
1650                         goto fail;
1651         }
1652
1653         rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
1654         if (rc) {
1655                 ibdev_err(&rdev->ibdev, "Create HW SRQ failed!");
1656                 goto fail;
1657         }
1658
1659         if (udata) {
1660                 struct bnxt_re_srq_resp resp;
1661
1662                 resp.srqid = srq->qplib_srq.id;
1663                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1664                 if (rc) {
1665                         ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!");
1666                         bnxt_qplib_destroy_srq(&rdev->qplib_res,
1667                                                &srq->qplib_srq);
1668                         goto fail;
1669                 }
1670         }
1671         if (nq)
1672                 nq->budget++;
1673         atomic_inc(&rdev->srq_count);
1674         spin_lock_init(&srq->lock);
1675
1676         return 0;
1677
1678 fail:
1679         ib_umem_release(srq->umem);
1680 exit:
1681         return rc;
1682 }
1683
1684 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
1685                        enum ib_srq_attr_mask srq_attr_mask,
1686                        struct ib_udata *udata)
1687 {
1688         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1689                                                ib_srq);
1690         struct bnxt_re_dev *rdev = srq->rdev;
1691         int rc;
1692
1693         switch (srq_attr_mask) {
1694         case IB_SRQ_MAX_WR:
1695                 /* SRQ resize is not supported */
1696                 break;
1697         case IB_SRQ_LIMIT:
1698                 /* Change the SRQ threshold */
1699                 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
1700                         return -EINVAL;
1701
1702                 srq->qplib_srq.threshold = srq_attr->srq_limit;
1703                 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
1704                 if (rc) {
1705                         ibdev_err(&rdev->ibdev, "Modify HW SRQ failed!");
1706                         return rc;
1707                 }
1708                 /* On success, update the shadow */
1709                 srq->srq_limit = srq_attr->srq_limit;
1710                 /* No need to Build and send response back to udata */
1711                 break;
1712         default:
1713                 ibdev_err(&rdev->ibdev,
1714                           "Unsupported srq_attr_mask 0x%x", srq_attr_mask);
1715                 return -EINVAL;
1716         }
1717         return 0;
1718 }
1719
1720 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
1721 {
1722         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1723                                                ib_srq);
1724         struct bnxt_re_srq tsrq;
1725         struct bnxt_re_dev *rdev = srq->rdev;
1726         int rc;
1727
1728         /* Get live SRQ attr */
1729         tsrq.qplib_srq.id = srq->qplib_srq.id;
1730         rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq);
1731         if (rc) {
1732                 ibdev_err(&rdev->ibdev, "Query HW SRQ failed!");
1733                 return rc;
1734         }
1735         srq_attr->max_wr = srq->qplib_srq.max_wqe;
1736         srq_attr->max_sge = srq->qplib_srq.max_sge;
1737         srq_attr->srq_limit = tsrq.qplib_srq.threshold;
1738
1739         return 0;
1740 }
1741
1742 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
1743                           const struct ib_recv_wr **bad_wr)
1744 {
1745         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1746                                                ib_srq);
1747         struct bnxt_qplib_swqe wqe;
1748         unsigned long flags;
1749         int rc = 0;
1750
1751         spin_lock_irqsave(&srq->lock, flags);
1752         while (wr) {
1753                 /* Transcribe each ib_recv_wr to qplib_swqe */
1754                 wqe.num_sge = wr->num_sge;
1755                 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
1756                 wqe.wr_id = wr->wr_id;
1757                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
1758
1759                 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
1760                 if (rc) {
1761                         *bad_wr = wr;
1762                         break;
1763                 }
1764                 wr = wr->next;
1765         }
1766         spin_unlock_irqrestore(&srq->lock, flags);
1767
1768         return rc;
1769 }
1770 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
1771                                     struct bnxt_re_qp *qp1_qp,
1772                                     int qp_attr_mask)
1773 {
1774         struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
1775         int rc = 0;
1776
1777         if (qp_attr_mask & IB_QP_STATE) {
1778                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1779                 qp->qplib_qp.state = qp1_qp->qplib_qp.state;
1780         }
1781         if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1782                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1783                 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
1784         }
1785
1786         if (qp_attr_mask & IB_QP_QKEY) {
1787                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1788                 /* Using a Random  QKEY */
1789                 qp->qplib_qp.qkey = 0x81818181;
1790         }
1791         if (qp_attr_mask & IB_QP_SQ_PSN) {
1792                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1793                 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
1794         }
1795
1796         rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1797         if (rc)
1798                 ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1");
1799         return rc;
1800 }
1801
1802 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1803                       int qp_attr_mask, struct ib_udata *udata)
1804 {
1805         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1806         struct bnxt_re_dev *rdev = qp->rdev;
1807         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1808         enum ib_qp_state curr_qp_state, new_qp_state;
1809         int rc, entries;
1810         unsigned int flags;
1811         u8 nw_type;
1812
1813         if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1814                 return -EOPNOTSUPP;
1815
1816         qp->qplib_qp.modify_flags = 0;
1817         if (qp_attr_mask & IB_QP_STATE) {
1818                 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
1819                 new_qp_state = qp_attr->qp_state;
1820                 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
1821                                         ib_qp->qp_type, qp_attr_mask)) {
1822                         ibdev_err(&rdev->ibdev,
1823                                   "Invalid attribute mask: %#x specified ",
1824                                   qp_attr_mask);
1825                         ibdev_err(&rdev->ibdev,
1826                                   "for qpn: %#x type: %#x",
1827                                   ib_qp->qp_num, ib_qp->qp_type);
1828                         ibdev_err(&rdev->ibdev,
1829                                   "curr_qp_state=0x%x, new_qp_state=0x%x\n",
1830                                   curr_qp_state, new_qp_state);
1831                         return -EINVAL;
1832                 }
1833                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1834                 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
1835
1836                 if (!qp->sumem &&
1837                     qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1838                         ibdev_dbg(&rdev->ibdev,
1839                                   "Move QP = %p to flush list\n", qp);
1840                         flags = bnxt_re_lock_cqs(qp);
1841                         bnxt_qplib_add_flush_qp(&qp->qplib_qp);
1842                         bnxt_re_unlock_cqs(qp, flags);
1843                 }
1844                 if (!qp->sumem &&
1845                     qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
1846                         ibdev_dbg(&rdev->ibdev,
1847                                   "Move QP = %p out of flush list\n", qp);
1848                         flags = bnxt_re_lock_cqs(qp);
1849                         bnxt_qplib_clean_qp(&qp->qplib_qp);
1850                         bnxt_re_unlock_cqs(qp, flags);
1851                 }
1852         }
1853         if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
1854                 qp->qplib_qp.modify_flags |=
1855                                 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
1856                 qp->qplib_qp.en_sqd_async_notify = true;
1857         }
1858         if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
1859                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
1860                 qp->qplib_qp.access =
1861                         __from_ib_access_flags(qp_attr->qp_access_flags);
1862                 /* LOCAL_WRITE access must be set to allow RC receive */
1863                 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1864                 /* Temp: Set all params on QP as of now */
1865                 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
1866                 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
1867         }
1868         if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1869                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1870                 qp->qplib_qp.pkey_index = qp_attr->pkey_index;
1871         }
1872         if (qp_attr_mask & IB_QP_QKEY) {
1873                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1874                 qp->qplib_qp.qkey = qp_attr->qkey;
1875         }
1876         if (qp_attr_mask & IB_QP_AV) {
1877                 const struct ib_global_route *grh =
1878                         rdma_ah_read_grh(&qp_attr->ah_attr);
1879                 const struct ib_gid_attr *sgid_attr;
1880                 struct bnxt_re_gid_ctx *ctx;
1881
1882                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1883                                      CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1884                                      CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1885                                      CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1886                                      CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1887                                      CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1888                                      CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1889                 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
1890                        sizeof(qp->qplib_qp.ah.dgid.data));
1891                 qp->qplib_qp.ah.flow_label = grh->flow_label;
1892                 sgid_attr = grh->sgid_attr;
1893                 /* Get the HW context of the GID. The reference
1894                  * of GID table entry is already taken by the caller.
1895                  */
1896                 ctx = rdma_read_gid_hw_context(sgid_attr);
1897                 qp->qplib_qp.ah.sgid_index = ctx->idx;
1898                 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
1899                 qp->qplib_qp.ah.hop_limit = grh->hop_limit;
1900                 qp->qplib_qp.ah.traffic_class = grh->traffic_class;
1901                 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
1902                 ether_addr_copy(qp->qplib_qp.ah.dmac,
1903                                 qp_attr->ah_attr.roce.dmac);
1904
1905                 rc = rdma_read_gid_l2_fields(sgid_attr, NULL,
1906                                              &qp->qplib_qp.smac[0]);
1907                 if (rc)
1908                         return rc;
1909
1910                 nw_type = rdma_gid_attr_network_type(sgid_attr);
1911                 switch (nw_type) {
1912                 case RDMA_NETWORK_IPV4:
1913                         qp->qplib_qp.nw_type =
1914                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
1915                         break;
1916                 case RDMA_NETWORK_IPV6:
1917                         qp->qplib_qp.nw_type =
1918                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
1919                         break;
1920                 default:
1921                         qp->qplib_qp.nw_type =
1922                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
1923                         break;
1924                 }
1925         }
1926
1927         if (qp_attr_mask & IB_QP_PATH_MTU) {
1928                 qp->qplib_qp.modify_flags |=
1929                                 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1930                 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu);
1931                 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu);
1932         } else if (qp_attr->qp_state == IB_QPS_RTR) {
1933                 qp->qplib_qp.modify_flags |=
1934                         CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1935                 qp->qplib_qp.path_mtu =
1936                         __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu));
1937                 qp->qplib_qp.mtu =
1938                         ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1939         }
1940
1941         if (qp_attr_mask & IB_QP_TIMEOUT) {
1942                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
1943                 qp->qplib_qp.timeout = qp_attr->timeout;
1944         }
1945         if (qp_attr_mask & IB_QP_RETRY_CNT) {
1946                 qp->qplib_qp.modify_flags |=
1947                                 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
1948                 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
1949         }
1950         if (qp_attr_mask & IB_QP_RNR_RETRY) {
1951                 qp->qplib_qp.modify_flags |=
1952                                 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
1953                 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
1954         }
1955         if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
1956                 qp->qplib_qp.modify_flags |=
1957                                 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
1958                 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
1959         }
1960         if (qp_attr_mask & IB_QP_RQ_PSN) {
1961                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
1962                 qp->qplib_qp.rq.psn = qp_attr->rq_psn;
1963         }
1964         if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1965                 qp->qplib_qp.modify_flags |=
1966                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
1967                 /* Cap the max_rd_atomic to device max */
1968                 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
1969                                                    dev_attr->max_qp_rd_atom);
1970         }
1971         if (qp_attr_mask & IB_QP_SQ_PSN) {
1972                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1973                 qp->qplib_qp.sq.psn = qp_attr->sq_psn;
1974         }
1975         if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1976                 if (qp_attr->max_dest_rd_atomic >
1977                     dev_attr->max_qp_init_rd_atom) {
1978                         ibdev_err(&rdev->ibdev,
1979                                   "max_dest_rd_atomic requested%d is > dev_max%d",
1980                                   qp_attr->max_dest_rd_atomic,
1981                                   dev_attr->max_qp_init_rd_atom);
1982                         return -EINVAL;
1983                 }
1984
1985                 qp->qplib_qp.modify_flags |=
1986                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
1987                 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
1988         }
1989         if (qp_attr_mask & IB_QP_CAP) {
1990                 qp->qplib_qp.modify_flags |=
1991                                 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
1992                                 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
1993                                 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
1994                                 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
1995                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
1996                 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
1997                     (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
1998                     (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
1999                     (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
2000                     (qp_attr->cap.max_inline_data >=
2001                                                 dev_attr->max_inline_data)) {
2002                         ibdev_err(&rdev->ibdev,
2003                                   "Create QP failed - max exceeded");
2004                         return -EINVAL;
2005                 }
2006                 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
2007                 qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
2008                                                 dev_attr->max_qp_wqes + 1);
2009                 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
2010                                                 qp_attr->cap.max_send_wr;
2011                 /*
2012                  * Reserving one slot for Phantom WQE. Some application can
2013                  * post one extra entry in this case. Allowing this to avoid
2014                  * unexpected Queue full condition
2015                  */
2016                 qp->qplib_qp.sq.q_full_delta -= 1;
2017                 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
2018                 if (qp->qplib_qp.rq.max_wqe) {
2019                         entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
2020                         qp->qplib_qp.rq.max_wqe =
2021                                 min_t(u32, entries, dev_attr->max_qp_wqes + 1);
2022                         qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
2023                                                        qp_attr->cap.max_recv_wr;
2024                         qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
2025                 } else {
2026                         /* SRQ was used prior, just ignore the RQ caps */
2027                 }
2028         }
2029         if (qp_attr_mask & IB_QP_DEST_QPN) {
2030                 qp->qplib_qp.modify_flags |=
2031                                 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
2032                 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
2033         }
2034         rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2035         if (rc) {
2036                 ibdev_err(&rdev->ibdev, "Failed to modify HW QP");
2037                 return rc;
2038         }
2039         if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
2040                 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
2041         return rc;
2042 }
2043
2044 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2045                      int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2046 {
2047         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2048         struct bnxt_re_dev *rdev = qp->rdev;
2049         struct bnxt_qplib_qp *qplib_qp;
2050         int rc;
2051
2052         qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL);
2053         if (!qplib_qp)
2054                 return -ENOMEM;
2055
2056         qplib_qp->id = qp->qplib_qp.id;
2057         qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
2058
2059         rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
2060         if (rc) {
2061                 ibdev_err(&rdev->ibdev, "Failed to query HW QP");
2062                 goto out;
2063         }
2064         qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
2065         qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
2066         qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
2067         qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access);
2068         qp_attr->pkey_index = qplib_qp->pkey_index;
2069         qp_attr->qkey = qplib_qp->qkey;
2070         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2071         rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label,
2072                         qplib_qp->ah.host_sgid_index,
2073                         qplib_qp->ah.hop_limit,
2074                         qplib_qp->ah.traffic_class);
2075         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data);
2076         rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl);
2077         ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac);
2078         qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
2079         qp_attr->timeout = qplib_qp->timeout;
2080         qp_attr->retry_cnt = qplib_qp->retry_cnt;
2081         qp_attr->rnr_retry = qplib_qp->rnr_retry;
2082         qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
2083         qp_attr->rq_psn = qplib_qp->rq.psn;
2084         qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
2085         qp_attr->sq_psn = qplib_qp->sq.psn;
2086         qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
2087         qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
2088                                                          IB_SIGNAL_REQ_WR;
2089         qp_attr->dest_qp_num = qplib_qp->dest_qpn;
2090
2091         qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
2092         qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
2093         qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
2094         qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
2095         qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
2096         qp_init_attr->cap = qp_attr->cap;
2097
2098 out:
2099         kfree(qplib_qp);
2100         return rc;
2101 }
2102
2103 /* Routine for sending QP1 packets for RoCE V1 an V2
2104  */
2105 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
2106                                      const struct ib_send_wr *wr,
2107                                      struct bnxt_qplib_swqe *wqe,
2108                                      int payload_size)
2109 {
2110         struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
2111                                              ib_ah);
2112         struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
2113         const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr;
2114         struct bnxt_qplib_sge sge;
2115         u8 nw_type;
2116         u16 ether_type;
2117         union ib_gid dgid;
2118         bool is_eth = false;
2119         bool is_vlan = false;
2120         bool is_grh = false;
2121         bool is_udp = false;
2122         u8 ip_version = 0;
2123         u16 vlan_id = 0xFFFF;
2124         void *buf;
2125         int i, rc = 0;
2126
2127         memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
2128
2129         rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL);
2130         if (rc)
2131                 return rc;
2132
2133         /* Get network header type for this GID */
2134         nw_type = rdma_gid_attr_network_type(sgid_attr);
2135         switch (nw_type) {
2136         case RDMA_NETWORK_IPV4:
2137                 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
2138                 break;
2139         case RDMA_NETWORK_IPV6:
2140                 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
2141                 break;
2142         default:
2143                 nw_type = BNXT_RE_ROCE_V1_PACKET;
2144                 break;
2145         }
2146         memcpy(&dgid.raw, &qplib_ah->dgid, 16);
2147         is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2148         if (is_udp) {
2149                 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) {
2150                         ip_version = 4;
2151                         ether_type = ETH_P_IP;
2152                 } else {
2153                         ip_version = 6;
2154                         ether_type = ETH_P_IPV6;
2155                 }
2156                 is_grh = false;
2157         } else {
2158                 ether_type = ETH_P_IBOE;
2159                 is_grh = true;
2160         }
2161
2162         is_eth = true;
2163         is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false;
2164
2165         ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
2166                           ip_version, is_udp, 0, &qp->qp1_hdr);
2167
2168         /* ETH */
2169         ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
2170         ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
2171
2172         /* For vlan, check the sgid for vlan existence */
2173
2174         if (!is_vlan) {
2175                 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
2176         } else {
2177                 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
2178                 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
2179         }
2180
2181         if (is_grh || (ip_version == 6)) {
2182                 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw,
2183                        sizeof(sgid_attr->gid));
2184                 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
2185                        sizeof(sgid_attr->gid));
2186                 qp->qp1_hdr.grh.hop_limit     = qplib_ah->hop_limit;
2187         }
2188
2189         if (ip_version == 4) {
2190                 qp->qp1_hdr.ip4.tos = 0;
2191                 qp->qp1_hdr.ip4.id = 0;
2192                 qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
2193                 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
2194
2195                 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4);
2196                 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
2197                 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
2198         }
2199
2200         if (is_udp) {
2201                 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
2202                 qp->qp1_hdr.udp.sport = htons(0x8CD1);
2203                 qp->qp1_hdr.udp.csum = 0;
2204         }
2205
2206         /* BTH */
2207         if (wr->opcode == IB_WR_SEND_WITH_IMM) {
2208                 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2209                 qp->qp1_hdr.immediate_present = 1;
2210         } else {
2211                 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2212         }
2213         if (wr->send_flags & IB_SEND_SOLICITED)
2214                 qp->qp1_hdr.bth.solicited_event = 1;
2215         /* pad_count */
2216         qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
2217
2218         /* P_key for QP1 is for all members */
2219         qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
2220         qp->qp1_hdr.bth.destination_qpn = IB_QP1;
2221         qp->qp1_hdr.bth.ack_req = 0;
2222         qp->send_psn++;
2223         qp->send_psn &= BTH_PSN_MASK;
2224         qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
2225         /* DETH */
2226         /* Use the priviledged Q_Key for QP1 */
2227         qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
2228         qp->qp1_hdr.deth.source_qpn = IB_QP1;
2229
2230         /* Pack the QP1 to the transmit buffer */
2231         buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
2232         if (buf) {
2233                 ib_ud_header_pack(&qp->qp1_hdr, buf);
2234                 for (i = wqe->num_sge; i; i--) {
2235                         wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
2236                         wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
2237                         wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
2238                 }
2239
2240                 /*
2241                  * Max Header buf size for IPV6 RoCE V2 is 86,
2242                  * which is same as the QP1 SQ header buffer.
2243                  * Header buf size for IPV4 RoCE V2 can be 66.
2244                  * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
2245                  * Subtract 20 bytes from QP1 SQ header buf size
2246                  */
2247                 if (is_udp && ip_version == 4)
2248                         sge.size -= 20;
2249                 /*
2250                  * Max Header buf size for RoCE V1 is 78.
2251                  * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
2252                  * Subtract 8 bytes from QP1 SQ header buf size
2253                  */
2254                 if (!is_udp)
2255                         sge.size -= 8;
2256
2257                 /* Subtract 4 bytes for non vlan packets */
2258                 if (!is_vlan)
2259                         sge.size -= 4;
2260
2261                 wqe->sg_list[0].addr = sge.addr;
2262                 wqe->sg_list[0].lkey = sge.lkey;
2263                 wqe->sg_list[0].size = sge.size;
2264                 wqe->num_sge++;
2265
2266         } else {
2267                 ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!");
2268                 rc = -ENOMEM;
2269         }
2270         return rc;
2271 }
2272
2273 /* For the MAD layer, it only provides the recv SGE the size of
2274  * ib_grh + MAD datagram.  No Ethernet headers, Ethertype, BTH, DETH,
2275  * nor RoCE iCRC.  The Cu+ solution must provide buffer for the entire
2276  * receive packet (334 bytes) with no VLAN and then copy the GRH
2277  * and the MAD datagram out to the provided SGE.
2278  */
2279 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
2280                                             const struct ib_recv_wr *wr,
2281                                             struct bnxt_qplib_swqe *wqe,
2282                                             int payload_size)
2283 {
2284         struct bnxt_re_sqp_entries *sqp_entry;
2285         struct bnxt_qplib_sge ref, sge;
2286         struct bnxt_re_dev *rdev;
2287         u32 rq_prod_index;
2288
2289         rdev = qp->rdev;
2290
2291         rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
2292
2293         if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
2294                 return -ENOMEM;
2295
2296         /* Create 1 SGE to receive the entire
2297          * ethernet packet
2298          */
2299         /* Save the reference from ULP */
2300         ref.addr = wqe->sg_list[0].addr;
2301         ref.lkey = wqe->sg_list[0].lkey;
2302         ref.size = wqe->sg_list[0].size;
2303
2304         sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
2305
2306         /* SGE 1 */
2307         wqe->sg_list[0].addr = sge.addr;
2308         wqe->sg_list[0].lkey = sge.lkey;
2309         wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
2310         sge.size -= wqe->sg_list[0].size;
2311
2312         sqp_entry->sge.addr = ref.addr;
2313         sqp_entry->sge.lkey = ref.lkey;
2314         sqp_entry->sge.size = ref.size;
2315         /* Store the wrid for reporting completion */
2316         sqp_entry->wrid = wqe->wr_id;
2317         /* change the wqe->wrid to table index */
2318         wqe->wr_id = rq_prod_index;
2319         return 0;
2320 }
2321
2322 static int is_ud_qp(struct bnxt_re_qp *qp)
2323 {
2324         return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
2325                 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
2326 }
2327
2328 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
2329                                   const struct ib_send_wr *wr,
2330                                   struct bnxt_qplib_swqe *wqe)
2331 {
2332         struct bnxt_re_ah *ah = NULL;
2333
2334         if (is_ud_qp(qp)) {
2335                 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
2336                 wqe->send.q_key = ud_wr(wr)->remote_qkey;
2337                 wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
2338                 wqe->send.avid = ah->qplib_ah.id;
2339         }
2340         switch (wr->opcode) {
2341         case IB_WR_SEND:
2342                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
2343                 break;
2344         case IB_WR_SEND_WITH_IMM:
2345                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
2346                 wqe->send.imm_data = wr->ex.imm_data;
2347                 break;
2348         case IB_WR_SEND_WITH_INV:
2349                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
2350                 wqe->send.inv_key = wr->ex.invalidate_rkey;
2351                 break;
2352         default:
2353                 return -EINVAL;
2354         }
2355         if (wr->send_flags & IB_SEND_SIGNALED)
2356                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2357         if (wr->send_flags & IB_SEND_FENCE)
2358                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2359         if (wr->send_flags & IB_SEND_SOLICITED)
2360                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2361         if (wr->send_flags & IB_SEND_INLINE)
2362                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2363
2364         return 0;
2365 }
2366
2367 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
2368                                   struct bnxt_qplib_swqe *wqe)
2369 {
2370         switch (wr->opcode) {
2371         case IB_WR_RDMA_WRITE:
2372                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
2373                 break;
2374         case IB_WR_RDMA_WRITE_WITH_IMM:
2375                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
2376                 wqe->rdma.imm_data = wr->ex.imm_data;
2377                 break;
2378         case IB_WR_RDMA_READ:
2379                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
2380                 wqe->rdma.inv_key = wr->ex.invalidate_rkey;
2381                 break;
2382         default:
2383                 return -EINVAL;
2384         }
2385         wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
2386         wqe->rdma.r_key = rdma_wr(wr)->rkey;
2387         if (wr->send_flags & IB_SEND_SIGNALED)
2388                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2389         if (wr->send_flags & IB_SEND_FENCE)
2390                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2391         if (wr->send_flags & IB_SEND_SOLICITED)
2392                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2393         if (wr->send_flags & IB_SEND_INLINE)
2394                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2395
2396         return 0;
2397 }
2398
2399 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
2400                                     struct bnxt_qplib_swqe *wqe)
2401 {
2402         switch (wr->opcode) {
2403         case IB_WR_ATOMIC_CMP_AND_SWP:
2404                 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
2405                 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2406                 wqe->atomic.swap_data = atomic_wr(wr)->swap;
2407                 break;
2408         case IB_WR_ATOMIC_FETCH_AND_ADD:
2409                 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
2410                 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2411                 break;
2412         default:
2413                 return -EINVAL;
2414         }
2415         wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
2416         wqe->atomic.r_key = atomic_wr(wr)->rkey;
2417         if (wr->send_flags & IB_SEND_SIGNALED)
2418                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2419         if (wr->send_flags & IB_SEND_FENCE)
2420                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2421         if (wr->send_flags & IB_SEND_SOLICITED)
2422                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2423         return 0;
2424 }
2425
2426 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
2427                                  struct bnxt_qplib_swqe *wqe)
2428 {
2429         wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
2430         wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
2431
2432         /* Need unconditional fence for local invalidate
2433          * opcode to work as expected.
2434          */
2435         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2436
2437         if (wr->send_flags & IB_SEND_SIGNALED)
2438                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2439         if (wr->send_flags & IB_SEND_SOLICITED)
2440                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2441
2442         return 0;
2443 }
2444
2445 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
2446                                  struct bnxt_qplib_swqe *wqe)
2447 {
2448         struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
2449         struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
2450         int access = wr->access;
2451
2452         wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
2453         wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
2454         wqe->frmr.page_list = mr->pages;
2455         wqe->frmr.page_list_len = mr->npages;
2456         wqe->frmr.levels = qplib_frpl->hwq.level;
2457         wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
2458
2459         /* Need unconditional fence for reg_mr
2460          * opcode to function as expected.
2461          */
2462
2463         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2464
2465         if (wr->wr.send_flags & IB_SEND_SIGNALED)
2466                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2467
2468         if (access & IB_ACCESS_LOCAL_WRITE)
2469                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2470         if (access & IB_ACCESS_REMOTE_READ)
2471                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
2472         if (access & IB_ACCESS_REMOTE_WRITE)
2473                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
2474         if (access & IB_ACCESS_REMOTE_ATOMIC)
2475                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
2476         if (access & IB_ACCESS_MW_BIND)
2477                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
2478
2479         wqe->frmr.l_key = wr->key;
2480         wqe->frmr.length = wr->mr->length;
2481         wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1;
2482         wqe->frmr.va = wr->mr->iova;
2483         return 0;
2484 }
2485
2486 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
2487                                     const struct ib_send_wr *wr,
2488                                     struct bnxt_qplib_swqe *wqe)
2489 {
2490         /*  Copy the inline data to the data  field */
2491         u8 *in_data;
2492         u32 i, sge_len;
2493         void *sge_addr;
2494
2495         in_data = wqe->inline_data;
2496         for (i = 0; i < wr->num_sge; i++) {
2497                 sge_addr = (void *)(unsigned long)
2498                                 wr->sg_list[i].addr;
2499                 sge_len = wr->sg_list[i].length;
2500
2501                 if ((sge_len + wqe->inline_len) >
2502                     BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
2503                         ibdev_err(&rdev->ibdev,
2504                                   "Inline data size requested > supported value");
2505                         return -EINVAL;
2506                 }
2507                 sge_len = wr->sg_list[i].length;
2508
2509                 memcpy(in_data, sge_addr, sge_len);
2510                 in_data += wr->sg_list[i].length;
2511                 wqe->inline_len += wr->sg_list[i].length;
2512         }
2513         return wqe->inline_len;
2514 }
2515
2516 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
2517                                    const struct ib_send_wr *wr,
2518                                    struct bnxt_qplib_swqe *wqe)
2519 {
2520         int payload_sz = 0;
2521
2522         if (wr->send_flags & IB_SEND_INLINE)
2523                 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
2524         else
2525                 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
2526                                                wqe->num_sge);
2527
2528         return payload_sz;
2529 }
2530
2531 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
2532 {
2533         if ((qp->ib_qp.qp_type == IB_QPT_UD ||
2534              qp->ib_qp.qp_type == IB_QPT_GSI ||
2535              qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
2536              qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
2537                 int qp_attr_mask;
2538                 struct ib_qp_attr qp_attr;
2539
2540                 qp_attr_mask = IB_QP_STATE;
2541                 qp_attr.qp_state = IB_QPS_RTS;
2542                 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
2543                 qp->qplib_qp.wqe_cnt = 0;
2544         }
2545 }
2546
2547 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
2548                                        struct bnxt_re_qp *qp,
2549                                        const struct ib_send_wr *wr)
2550 {
2551         int rc = 0, payload_sz = 0;
2552         unsigned long flags;
2553
2554         spin_lock_irqsave(&qp->sq_lock, flags);
2555         while (wr) {
2556                 struct bnxt_qplib_swqe wqe = {};
2557
2558                 /* Common */
2559                 wqe.num_sge = wr->num_sge;
2560                 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2561                         ibdev_err(&rdev->ibdev,
2562                                   "Limit exceeded for Send SGEs");
2563                         rc = -EINVAL;
2564                         goto bad;
2565                 }
2566
2567                 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2568                 if (payload_sz < 0) {
2569                         rc = -EINVAL;
2570                         goto bad;
2571                 }
2572                 wqe.wr_id = wr->wr_id;
2573
2574                 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
2575
2576                 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2577                 if (!rc)
2578                         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2579 bad:
2580                 if (rc) {
2581                         ibdev_err(&rdev->ibdev,
2582                                   "Post send failed opcode = %#x rc = %d",
2583                                   wr->opcode, rc);
2584                         break;
2585                 }
2586                 wr = wr->next;
2587         }
2588         bnxt_qplib_post_send_db(&qp->qplib_qp);
2589         bnxt_ud_qp_hw_stall_workaround(qp);
2590         spin_unlock_irqrestore(&qp->sq_lock, flags);
2591         return rc;
2592 }
2593
2594 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
2595                       const struct ib_send_wr **bad_wr)
2596 {
2597         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2598         struct bnxt_qplib_swqe wqe;
2599         int rc = 0, payload_sz = 0;
2600         unsigned long flags;
2601
2602         spin_lock_irqsave(&qp->sq_lock, flags);
2603         while (wr) {
2604                 /* House keeping */
2605                 memset(&wqe, 0, sizeof(wqe));
2606
2607                 /* Common */
2608                 wqe.num_sge = wr->num_sge;
2609                 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2610                         ibdev_err(&qp->rdev->ibdev,
2611                                   "Limit exceeded for Send SGEs");
2612                         rc = -EINVAL;
2613                         goto bad;
2614                 }
2615
2616                 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2617                 if (payload_sz < 0) {
2618                         rc = -EINVAL;
2619                         goto bad;
2620                 }
2621                 wqe.wr_id = wr->wr_id;
2622
2623                 switch (wr->opcode) {
2624                 case IB_WR_SEND:
2625                 case IB_WR_SEND_WITH_IMM:
2626                         if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) {
2627                                 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
2628                                                                payload_sz);
2629                                 if (rc)
2630                                         goto bad;
2631                                 wqe.rawqp1.lflags |=
2632                                         SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
2633                         }
2634                         switch (wr->send_flags) {
2635                         case IB_SEND_IP_CSUM:
2636                                 wqe.rawqp1.lflags |=
2637                                         SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
2638                                 break;
2639                         default:
2640                                 break;
2641                         }
2642                         fallthrough;
2643                 case IB_WR_SEND_WITH_INV:
2644                         rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2645                         break;
2646                 case IB_WR_RDMA_WRITE:
2647                 case IB_WR_RDMA_WRITE_WITH_IMM:
2648                 case IB_WR_RDMA_READ:
2649                         rc = bnxt_re_build_rdma_wqe(wr, &wqe);
2650                         break;
2651                 case IB_WR_ATOMIC_CMP_AND_SWP:
2652                 case IB_WR_ATOMIC_FETCH_AND_ADD:
2653                         rc = bnxt_re_build_atomic_wqe(wr, &wqe);
2654                         break;
2655                 case IB_WR_RDMA_READ_WITH_INV:
2656                         ibdev_err(&qp->rdev->ibdev,
2657                                   "RDMA Read with Invalidate is not supported");
2658                         rc = -EINVAL;
2659                         goto bad;
2660                 case IB_WR_LOCAL_INV:
2661                         rc = bnxt_re_build_inv_wqe(wr, &wqe);
2662                         break;
2663                 case IB_WR_REG_MR:
2664                         rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2665                         break;
2666                 default:
2667                         /* Unsupported WRs */
2668                         ibdev_err(&qp->rdev->ibdev,
2669                                   "WR (%#x) is not supported", wr->opcode);
2670                         rc = -EINVAL;
2671                         goto bad;
2672                 }
2673                 if (!rc)
2674                         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2675 bad:
2676                 if (rc) {
2677                         ibdev_err(&qp->rdev->ibdev,
2678                                   "post_send failed op:%#x qps = %#x rc = %d\n",
2679                                   wr->opcode, qp->qplib_qp.state, rc);
2680                         *bad_wr = wr;
2681                         break;
2682                 }
2683                 wr = wr->next;
2684         }
2685         bnxt_qplib_post_send_db(&qp->qplib_qp);
2686         bnxt_ud_qp_hw_stall_workaround(qp);
2687         spin_unlock_irqrestore(&qp->sq_lock, flags);
2688
2689         return rc;
2690 }
2691
2692 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2693                                        struct bnxt_re_qp *qp,
2694                                        const struct ib_recv_wr *wr)
2695 {
2696         struct bnxt_qplib_swqe wqe;
2697         int rc = 0;
2698
2699         memset(&wqe, 0, sizeof(wqe));
2700         while (wr) {
2701                 /* House keeping */
2702                 memset(&wqe, 0, sizeof(wqe));
2703
2704                 /* Common */
2705                 wqe.num_sge = wr->num_sge;
2706                 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2707                         ibdev_err(&rdev->ibdev,
2708                                   "Limit exceeded for Receive SGEs");
2709                         rc = -EINVAL;
2710                         break;
2711                 }
2712                 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
2713                 wqe.wr_id = wr->wr_id;
2714                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2715
2716                 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2717                 if (rc)
2718                         break;
2719
2720                 wr = wr->next;
2721         }
2722         if (!rc)
2723                 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2724         return rc;
2725 }
2726
2727 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
2728                       const struct ib_recv_wr **bad_wr)
2729 {
2730         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2731         struct bnxt_qplib_swqe wqe;
2732         int rc = 0, payload_sz = 0;
2733         unsigned long flags;
2734         u32 count = 0;
2735
2736         spin_lock_irqsave(&qp->rq_lock, flags);
2737         while (wr) {
2738                 /* House keeping */
2739                 memset(&wqe, 0, sizeof(wqe));
2740
2741                 /* Common */
2742                 wqe.num_sge = wr->num_sge;
2743                 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2744                         ibdev_err(&qp->rdev->ibdev,
2745                                   "Limit exceeded for Receive SGEs");
2746                         rc = -EINVAL;
2747                         *bad_wr = wr;
2748                         break;
2749                 }
2750
2751                 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2752                                                wr->num_sge);
2753                 wqe.wr_id = wr->wr_id;
2754                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2755
2756                 if (ib_qp->qp_type == IB_QPT_GSI &&
2757                     qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI)
2758                         rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
2759                                                               payload_sz);
2760                 if (!rc)
2761                         rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2762                 if (rc) {
2763                         *bad_wr = wr;
2764                         break;
2765                 }
2766
2767                 /* Ring DB if the RQEs posted reaches a threshold value */
2768                 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
2769                         bnxt_qplib_post_recv_db(&qp->qplib_qp);
2770                         count = 0;
2771                 }
2772
2773                 wr = wr->next;
2774         }
2775
2776         if (count)
2777                 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2778
2779         spin_unlock_irqrestore(&qp->rq_lock, flags);
2780
2781         return rc;
2782 }
2783
2784 /* Completion Queues */
2785 int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
2786 {
2787         struct bnxt_re_cq *cq;
2788         struct bnxt_qplib_nq *nq;
2789         struct bnxt_re_dev *rdev;
2790
2791         cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2792         rdev = cq->rdev;
2793         nq = cq->qplib_cq.nq;
2794
2795         bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2796         ib_umem_release(cq->umem);
2797
2798         atomic_dec(&rdev->cq_count);
2799         nq->budget--;
2800         kfree(cq->cql);
2801         return 0;
2802 }
2803
2804 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
2805                       struct ib_udata *udata)
2806 {
2807         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
2808         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
2809         struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
2810         int rc, entries;
2811         int cqe = attr->cqe;
2812         struct bnxt_qplib_nq *nq = NULL;
2813         unsigned int nq_alloc_cnt;
2814
2815         if (attr->flags)
2816                 return -EOPNOTSUPP;
2817
2818         /* Validate CQ fields */
2819         if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
2820                 ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded");
2821                 return -EINVAL;
2822         }
2823
2824         cq->rdev = rdev;
2825         cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
2826
2827         entries = roundup_pow_of_two(cqe + 1);
2828         if (entries > dev_attr->max_cq_wqes + 1)
2829                 entries = dev_attr->max_cq_wqes + 1;
2830
2831         cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
2832         cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
2833         if (udata) {
2834                 struct bnxt_re_cq_req req;
2835                 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
2836                         udata, struct bnxt_re_ucontext, ib_uctx);
2837                 if (ib_copy_from_udata(&req, udata, sizeof(req))) {
2838                         rc = -EFAULT;
2839                         goto fail;
2840                 }
2841
2842                 cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
2843                                        entries * sizeof(struct cq_base),
2844                                        IB_ACCESS_LOCAL_WRITE);
2845                 if (IS_ERR(cq->umem)) {
2846                         rc = PTR_ERR(cq->umem);
2847                         goto fail;
2848                 }
2849                 cq->qplib_cq.sg_info.umem = cq->umem;
2850                 cq->qplib_cq.dpi = &uctx->dpi;
2851         } else {
2852                 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
2853                 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
2854                                   GFP_KERNEL);
2855                 if (!cq->cql) {
2856                         rc = -ENOMEM;
2857                         goto fail;
2858                 }
2859
2860                 cq->qplib_cq.dpi = &rdev->dpi_privileged;
2861         }
2862         /*
2863          * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
2864          * used for getting the NQ index.
2865          */
2866         nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt);
2867         nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)];
2868         cq->qplib_cq.max_wqe = entries;
2869         cq->qplib_cq.cnq_hw_ring_id = nq->ring_id;
2870         cq->qplib_cq.nq = nq;
2871
2872         rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
2873         if (rc) {
2874                 ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
2875                 goto fail;
2876         }
2877
2878         cq->ib_cq.cqe = entries;
2879         cq->cq_period = cq->qplib_cq.period;
2880         nq->budget++;
2881
2882         atomic_inc(&rdev->cq_count);
2883         spin_lock_init(&cq->cq_lock);
2884
2885         if (udata) {
2886                 struct bnxt_re_cq_resp resp;
2887
2888                 resp.cqid = cq->qplib_cq.id;
2889                 resp.tail = cq->qplib_cq.hwq.cons;
2890                 resp.phase = cq->qplib_cq.period;
2891                 resp.rsvd = 0;
2892                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
2893                 if (rc) {
2894                         ibdev_err(&rdev->ibdev, "Failed to copy CQ udata");
2895                         bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2896                         goto c2fail;
2897                 }
2898         }
2899
2900         return 0;
2901
2902 c2fail:
2903         ib_umem_release(cq->umem);
2904 fail:
2905         kfree(cq->cql);
2906         return rc;
2907 }
2908
2909 static u8 __req_to_ib_wc_status(u8 qstatus)
2910 {
2911         switch (qstatus) {
2912         case CQ_REQ_STATUS_OK:
2913                 return IB_WC_SUCCESS;
2914         case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
2915                 return IB_WC_BAD_RESP_ERR;
2916         case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
2917                 return IB_WC_LOC_LEN_ERR;
2918         case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
2919                 return IB_WC_LOC_QP_OP_ERR;
2920         case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
2921                 return IB_WC_LOC_PROT_ERR;
2922         case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
2923                 return IB_WC_GENERAL_ERR;
2924         case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
2925                 return IB_WC_REM_INV_REQ_ERR;
2926         case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
2927                 return IB_WC_REM_ACCESS_ERR;
2928         case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
2929                 return IB_WC_REM_OP_ERR;
2930         case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
2931                 return IB_WC_RNR_RETRY_EXC_ERR;
2932         case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
2933                 return IB_WC_RETRY_EXC_ERR;
2934         case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
2935                 return IB_WC_WR_FLUSH_ERR;
2936         default:
2937                 return IB_WC_GENERAL_ERR;
2938         }
2939         return 0;
2940 }
2941
2942 static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
2943 {
2944         switch (qstatus) {
2945         case CQ_RES_RAWETH_QP1_STATUS_OK:
2946                 return IB_WC_SUCCESS;
2947         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
2948                 return IB_WC_LOC_ACCESS_ERR;
2949         case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
2950                 return IB_WC_LOC_LEN_ERR;
2951         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
2952                 return IB_WC_LOC_PROT_ERR;
2953         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
2954                 return IB_WC_LOC_QP_OP_ERR;
2955         case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
2956                 return IB_WC_GENERAL_ERR;
2957         case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
2958                 return IB_WC_WR_FLUSH_ERR;
2959         case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
2960                 return IB_WC_WR_FLUSH_ERR;
2961         default:
2962                 return IB_WC_GENERAL_ERR;
2963         }
2964 }
2965
2966 static u8 __rc_to_ib_wc_status(u8 qstatus)
2967 {
2968         switch (qstatus) {
2969         case CQ_RES_RC_STATUS_OK:
2970                 return IB_WC_SUCCESS;
2971         case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
2972                 return IB_WC_LOC_ACCESS_ERR;
2973         case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
2974                 return IB_WC_LOC_LEN_ERR;
2975         case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
2976                 return IB_WC_LOC_PROT_ERR;
2977         case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
2978                 return IB_WC_LOC_QP_OP_ERR;
2979         case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
2980                 return IB_WC_GENERAL_ERR;
2981         case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
2982                 return IB_WC_REM_INV_REQ_ERR;
2983         case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
2984                 return IB_WC_WR_FLUSH_ERR;
2985         case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
2986                 return IB_WC_WR_FLUSH_ERR;
2987         default:
2988                 return IB_WC_GENERAL_ERR;
2989         }
2990 }
2991
2992 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
2993 {
2994         switch (cqe->type) {
2995         case BNXT_QPLIB_SWQE_TYPE_SEND:
2996                 wc->opcode = IB_WC_SEND;
2997                 break;
2998         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
2999                 wc->opcode = IB_WC_SEND;
3000                 wc->wc_flags |= IB_WC_WITH_IMM;
3001                 break;
3002         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
3003                 wc->opcode = IB_WC_SEND;
3004                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3005                 break;
3006         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
3007                 wc->opcode = IB_WC_RDMA_WRITE;
3008                 break;
3009         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
3010                 wc->opcode = IB_WC_RDMA_WRITE;
3011                 wc->wc_flags |= IB_WC_WITH_IMM;
3012                 break;
3013         case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
3014                 wc->opcode = IB_WC_RDMA_READ;
3015                 break;
3016         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
3017                 wc->opcode = IB_WC_COMP_SWAP;
3018                 break;
3019         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
3020                 wc->opcode = IB_WC_FETCH_ADD;
3021                 break;
3022         case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
3023                 wc->opcode = IB_WC_LOCAL_INV;
3024                 break;
3025         case BNXT_QPLIB_SWQE_TYPE_REG_MR:
3026                 wc->opcode = IB_WC_REG_MR;
3027                 break;
3028         default:
3029                 wc->opcode = IB_WC_SEND;
3030                 break;
3031         }
3032
3033         wc->status = __req_to_ib_wc_status(cqe->status);
3034 }
3035
3036 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
3037                                      u16 raweth_qp1_flags2)
3038 {
3039         bool is_ipv6 = false, is_ipv4 = false;
3040
3041         /* raweth_qp1_flags Bit 9-6 indicates itype */
3042         if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3043             != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3044                 return -1;
3045
3046         if (raweth_qp1_flags2 &
3047             CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&