drivers/infiniband/hw/bnxt_re/ib_verbs.c

   1 /*
   2  * Broadcom NetXtreme-E RoCE driver.
   3  *
   4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * BSD license below:
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in
  21  *    the documentation and/or other materials provided with the
  22  *    distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Description: IB Verbs interpreter
  37  */
  38
  39 #include <linux/interrupt.h>
  40 #include <linux/types.h>
  41 #include <linux/pci.h>
  42 #include <linux/netdevice.h>
  43 #include <linux/if_ether.h>
  44
  45 #include <rdma/ib_verbs.h>
  46 #include <rdma/ib_user_verbs.h>
  47 #include <rdma/ib_umem.h>
  48 #include <rdma/ib_addr.h>
  49 #include <rdma/ib_mad.h>
  50 #include <rdma/ib_cache.h>
  51 #include <rdma/uverbs_ioctl.h>
  52
  53 #include "bnxt_ulp.h"
  54
  55 #include "roce_hsi.h"
  56 #include "qplib_res.h"
  57 #include "qplib_sp.h"
  58 #include "qplib_fp.h"
  59 #include "qplib_rcfw.h"
  60
  61 #include "bnxt_re.h"
  62 #include "ib_verbs.h"
  63 #include <rdma/bnxt_re-abi.h>
  64
  65 static int __from_ib_access_flags(int iflags)
  66 {
  67         int qflags = 0;
  68
  69         if (iflags & IB_ACCESS_LOCAL_WRITE)
  70                 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
  71         if (iflags & IB_ACCESS_REMOTE_READ)
  72                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
  73         if (iflags & IB_ACCESS_REMOTE_WRITE)
  74                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
  75         if (iflags & IB_ACCESS_REMOTE_ATOMIC)
  76                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
  77         if (iflags & IB_ACCESS_MW_BIND)
  78                 qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
  79         if (iflags & IB_ZERO_BASED)
  80                 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
  81         if (iflags & IB_ACCESS_ON_DEMAND)
  82                 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
  83         return qflags;
  84 };
  85
  86 static enum ib_access_flags __to_ib_access_flags(int qflags)
  87 {
  88         enum ib_access_flags iflags = 0;
  89
  90         if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
  91                 iflags |= IB_ACCESS_LOCAL_WRITE;
  92         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
  93                 iflags |= IB_ACCESS_REMOTE_WRITE;
  94         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
  95                 iflags |= IB_ACCESS_REMOTE_READ;
  96         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
  97                 iflags |= IB_ACCESS_REMOTE_ATOMIC;
  98         if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
  99                 iflags |= IB_ACCESS_MW_BIND;
 100         if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
 101                 iflags |= IB_ZERO_BASED;
 102         if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
 103                 iflags |= IB_ACCESS_ON_DEMAND;
 104         return iflags;
 105 };
 106
 107 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
 108                              struct bnxt_qplib_sge *sg_list, int num)
 109 {
 110         int i, total = 0;
 111
 112         for (i = 0; i < num; i++) {
 113                 sg_list[i].addr = ib_sg_list[i].addr;
 114                 sg_list[i].lkey = ib_sg_list[i].lkey;
 115                 sg_list[i].size = ib_sg_list[i].length;
 116                 total += sg_list[i].size;
 117         }
 118         return total;
 119 }
 120
 121 /* Device */
 122 int bnxt_re_query_device(struct ib_device *ibdev,
 123                          struct ib_device_attr *ib_attr,
 124                          struct ib_udata *udata)
 125 {
 126         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 127         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
 128
 129         memset(ib_attr, 0, sizeof(*ib_attr));
 130         memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
 131                min(sizeof(dev_attr->fw_ver),
 132                    sizeof(ib_attr->fw_ver)));
 133         bnxt_qplib_get_guid(rdev->netdev->dev_addr,
 134                             (u8 *)&ib_attr->sys_image_guid);
 135         ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
 136         ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M;
 137
 138         ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
 139         ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
 140         ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
 141         ib_attr->max_qp = dev_attr->max_qp;
 142         ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
 143         ib_attr->device_cap_flags =
 144                                     IB_DEVICE_CURR_QP_STATE_MOD
 145                                     | IB_DEVICE_RC_RNR_NAK_GEN
 146                                     | IB_DEVICE_SHUTDOWN_PORT
 147                                     | IB_DEVICE_SYS_IMAGE_GUID
 148                                     | IB_DEVICE_LOCAL_DMA_LKEY
 149                                     | IB_DEVICE_RESIZE_MAX_WR
 150                                     | IB_DEVICE_PORT_ACTIVE_EVENT
 151                                     | IB_DEVICE_N_NOTIFY_CQ
 152                                     | IB_DEVICE_MEM_WINDOW
 153                                     | IB_DEVICE_MEM_WINDOW_TYPE_2B
 154                                     | IB_DEVICE_MEM_MGT_EXTENSIONS;
 155         ib_attr->max_send_sge = dev_attr->max_qp_sges;
 156         ib_attr->max_recv_sge = dev_attr->max_qp_sges;
 157         ib_attr->max_sge_rd = dev_attr->max_qp_sges;
 158         ib_attr->max_cq = dev_attr->max_cq;
 159         ib_attr->max_cqe = dev_attr->max_cq_wqes;
 160         ib_attr->max_mr = dev_attr->max_mr;
 161         ib_attr->max_pd = dev_attr->max_pd;
 162         ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
 163         ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
 164         ib_attr->atomic_cap = IB_ATOMIC_NONE;
 165         ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
 166         if (dev_attr->is_atomic) {
 167                 ib_attr->atomic_cap = IB_ATOMIC_GLOB;
 168                 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
 169         }
 170
 171         ib_attr->max_ee_rd_atom = 0;
 172         ib_attr->max_res_rd_atom = 0;
 173         ib_attr->max_ee_init_rd_atom = 0;
 174         ib_attr->max_ee = 0;
 175         ib_attr->max_rdd = 0;
 176         ib_attr->max_mw = dev_attr->max_mw;
 177         ib_attr->max_raw_ipv6_qp = 0;
 178         ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
 179         ib_attr->max_mcast_grp = 0;
 180         ib_attr->max_mcast_qp_attach = 0;
 181         ib_attr->max_total_mcast_qp_attach = 0;
 182         ib_attr->max_ah = dev_attr->max_ah;
 183
 184         ib_attr->max_srq = dev_attr->max_srq;
 185         ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
 186         ib_attr->max_srq_sge = dev_attr->max_srq_sges;
 187
 188         ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
 189
 190         ib_attr->max_pkeys = 1;
 191         ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
 192         return 0;
 193 }
 194
 195 /* Port */
 196 int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,
 197                        struct ib_port_attr *port_attr)
 198 {
 199         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 200         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
 201
 202         memset(port_attr, 0, sizeof(*port_attr));
 203
 204         if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
 205                 port_attr->state = IB_PORT_ACTIVE;
 206                 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 207         } else {
 208                 port_attr->state = IB_PORT_DOWN;
 209                 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 210         }
 211         port_attr->max_mtu = IB_MTU_4096;
 212         port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
 213         port_attr->gid_tbl_len = dev_attr->max_sgid;
 214         port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 215                                     IB_PORT_DEVICE_MGMT_SUP |
 216                                     IB_PORT_VENDOR_CLASS_SUP;
 217         port_attr->ip_gids = true;
 218
 219         port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
 220         port_attr->bad_pkey_cntr = 0;
 221         port_attr->qkey_viol_cntr = 0;
 222         port_attr->pkey_tbl_len = dev_attr->max_pkey;
 223         port_attr->lid = 0;
 224         port_attr->sm_lid = 0;
 225         port_attr->lmc = 0;
 226         port_attr->max_vl_num = 4;
 227         port_attr->sm_sl = 0;
 228         port_attr->subnet_timeout = 0;
 229         port_attr->init_type_reply = 0;
 230         port_attr->active_speed = rdev->active_speed;
 231         port_attr->active_width = rdev->active_width;
 232
 233         return 0;
 234 }
 235
 236 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
 237                                struct ib_port_immutable *immutable)
 238 {
 239         struct ib_port_attr port_attr;
 240
 241         if (bnxt_re_query_port(ibdev, port_num, &port_attr))
 242                 return -EINVAL;
 243
 244         immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
 245         immutable->gid_tbl_len = port_attr.gid_tbl_len;
 246         immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
 247         immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
 248         immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 249         return 0;
 250 }
 251
 252 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)
 253 {
 254         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 255
 256         snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d",
 257                  rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1],
 258                  rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]);
 259 }
 260
 261 int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,
 262                        u16 index, u16 *pkey)
 263 {
 264         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 265
 266         /* Ignore port_num */
 267
 268         memset(pkey, 0, sizeof(*pkey));
 269         return bnxt_qplib_get_pkey(&rdev->qplib_res,
 270                                    &rdev->qplib_res.pkey_tbl, index, pkey);
 271 }
 272
 273 int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,
 274                       int index, union ib_gid *gid)
 275 {
 276         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 277         int rc = 0;
 278
 279         /* Ignore port_num */
 280         memset(gid, 0, sizeof(*gid));
 281         rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
 282                                  &rdev->qplib_res.sgid_tbl, index,
 283                                  (struct bnxt_qplib_gid *)gid);
 284         return rc;
 285 }
 286
 287 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)
 288 {
 289         int rc = 0;
 290         struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
 291         struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 292         struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 293         struct bnxt_qplib_gid *gid_to_del;
 294         u16 vlan_id = 0xFFFF;
 295
 296         /* Delete the entry from the hardware */
 297         ctx = *context;
 298         if (!ctx)
 299                 return -EINVAL;
 300
 301         if (sgid_tbl && sgid_tbl->active) {
 302                 if (ctx->idx >= sgid_tbl->max)
 303                         return -EINVAL;
 304                 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
 305                 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
 306                 /* DEL_GID is called in WQ context(netdevice_event_work_handler)
 307                  * or via the ib_unregister_device path. In the former case QP1
 308                  * may not be destroyed yet, in which case just return as FW
 309                  * needs that entry to be present and will fail it's deletion.
 310                  * We could get invoked again after QP1 is destroyed OR get an
 311                  * ADD_GID call with a different GID value for the same index
 312                  * where we issue MODIFY_GID cmd to update the GID entry -- TBD
 313                  */
 314                 if (ctx->idx == 0 &&
 315                     rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
 316                     ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) {
 317                         ibdev_dbg(&rdev->ibdev,
 318                                   "Trying to delete GID0 while QP1 is alive\n");
 319                         return -EFAULT;
 320                 }
 321                 ctx->refcnt--;
 322                 if (!ctx->refcnt) {
 323                         rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
 324                                                  vlan_id,  true);
 325                         if (rc) {
 326                                 ibdev_err(&rdev->ibdev,
 327                                           "Failed to remove GID: %#x", rc);
 328                         } else {
 329                                 ctx_tbl = sgid_tbl->ctx;
 330                                 ctx_tbl[ctx->idx] = NULL;
 331                                 kfree(ctx);
 332                         }
 333                 }
 334         } else {
 335                 return -EINVAL;
 336         }
 337         return rc;
 338 }
 339
 340 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)
 341 {
 342         int rc;
 343         u32 tbl_idx = 0;
 344         u16 vlan_id = 0xFFFF;
 345         struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
 346         struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 347         struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 348
 349         rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL);
 350         if (rc)
 351                 return rc;
 352
 353         rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid,
 354                                  rdev->qplib_res.netdev->dev_addr,
 355                                  vlan_id, true, &tbl_idx);
 356         if (rc == -EALREADY) {
 357                 ctx_tbl = sgid_tbl->ctx;
 358                 ctx_tbl[tbl_idx]->refcnt++;
 359                 *context = ctx_tbl[tbl_idx];
 360                 return 0;
 361         }
 362
 363         if (rc < 0) {
 364                 ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc);
 365                 return rc;
 366         }
 367
 368         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 369         if (!ctx)
 370                 return -ENOMEM;
 371         ctx_tbl = sgid_tbl->ctx;
 372         ctx->idx = tbl_idx;
 373         ctx->refcnt = 1;
 374         ctx_tbl[tbl_idx] = ctx;
 375         *context = ctx;
 376
 377         return rc;
 378 }
 379
 380 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
 381                                             u32 port_num)
 382 {
 383         return IB_LINK_LAYER_ETHERNET;
 384 }
 385
 386 #define BNXT_RE_FENCE_PBL_SIZE  DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE)
 387
 388 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)
 389 {
 390         struct bnxt_re_fence_data *fence = &pd->fence;
 391         struct ib_mr *ib_mr = &fence->mr->ib_mr;
 392         struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
 393
 394         memset(wqe, 0, sizeof(*wqe));
 395         wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
 396         wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
 397         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
 398         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
 399         wqe->bind.zero_based = false;
 400         wqe->bind.parent_l_key = ib_mr->lkey;
 401         wqe->bind.va = (u64)(unsigned long)fence->va;
 402         wqe->bind.length = fence->size;
 403         wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
 404         wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
 405
 406         /* Save the initial rkey in fence structure for now;
 407          * wqe->bind.r_key will be set at (re)bind time.
 408          */
 409         fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
 410 }
 411
 412 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
 413 {
 414         struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
 415                                              qplib_qp);
 416         struct ib_pd *ib_pd = qp->ib_qp.pd;
 417         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 418         struct bnxt_re_fence_data *fence = &pd->fence;
 419         struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
 420         struct bnxt_qplib_swqe wqe;
 421         int rc;
 422
 423         memcpy(&wqe, fence_wqe, sizeof(wqe));
 424         wqe.bind.r_key = fence->bind_rkey;
 425         fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
 426
 427         ibdev_dbg(&qp->rdev->ibdev,
 428                   "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
 429                 wqe.bind.r_key, qp->qplib_qp.id, pd);
 430         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
 431         if (rc) {
 432                 ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n");
 433                 return rc;
 434         }
 435         bnxt_qplib_post_send_db(&qp->qplib_qp);
 436
 437         return rc;
 438 }
 439
 440 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)
 441 {
 442         struct bnxt_re_fence_data *fence = &pd->fence;
 443         struct bnxt_re_dev *rdev = pd->rdev;
 444         struct device *dev = &rdev->en_dev->pdev->dev;
 445         struct bnxt_re_mr *mr = fence->mr;
 446
 447         if (fence->mw) {
 448                 bnxt_re_dealloc_mw(fence->mw);
 449                 fence->mw = NULL;
 450         }
 451         if (mr) {
 452                 if (mr->ib_mr.rkey)
 453                         bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
 454                                              true);
 455                 if (mr->ib_mr.lkey)
 456                         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
 457                 kfree(mr);
 458                 fence->mr = NULL;
 459         }
 460         if (fence->dma_addr) {
 461                 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES,
 462                                  DMA_BIDIRECTIONAL);
 463                 fence->dma_addr = 0;
 464         }
 465 }
 466
 467 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
 468 {
 469         int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
 470         struct bnxt_re_fence_data *fence = &pd->fence;
 471         struct bnxt_re_dev *rdev = pd->rdev;
 472         struct device *dev = &rdev->en_dev->pdev->dev;
 473         struct bnxt_re_mr *mr = NULL;
 474         dma_addr_t dma_addr = 0;
 475         struct ib_mw *mw;
 476         int rc;
 477
 478         dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
 479                                   DMA_BIDIRECTIONAL);
 480         rc = dma_mapping_error(dev, dma_addr);
 481         if (rc) {
 482                 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n");
 483                 rc = -EIO;
 484                 fence->dma_addr = 0;
 485                 goto fail;
 486         }
 487         fence->dma_addr = dma_addr;
 488
 489         /* Allocate a MR */
 490         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 491         if (!mr) {
 492                 rc = -ENOMEM;
 493                 goto fail;
 494         }
 495         fence->mr = mr;
 496         mr->rdev = rdev;
 497         mr->qplib_mr.pd = &pd->qplib_pd;
 498         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
 499         mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
 500         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
 501         if (rc) {
 502                 ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n");
 503                 goto fail;
 504         }
 505
 506         /* Register MR */
 507         mr->ib_mr.lkey = mr->qplib_mr.lkey;
 508         mr->qplib_mr.va = (u64)(unsigned long)fence->va;
 509         mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
 510         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
 511                                BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE);
 512         if (rc) {
 513                 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
 514                 goto fail;
 515         }
 516         mr->ib_mr.rkey = mr->qplib_mr.rkey;
 517
 518         /* Create a fence MW only for kernel consumers */
 519         mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
 520         if (IS_ERR(mw)) {
 521                 ibdev_err(&rdev->ibdev,
 522                           "Failed to create fence-MW for PD: %p\n", pd);
 523                 rc = PTR_ERR(mw);
 524                 goto fail;
 525         }
 526         fence->mw = mw;
 527
 528         bnxt_re_create_fence_wqe(pd);
 529         return 0;
 530
 531 fail:
 532         bnxt_re_destroy_fence_mr(pd);
 533         return rc;
 534 }
 535
 536 /* Protection Domains */
 537 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
 538 {
 539         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 540         struct bnxt_re_dev *rdev = pd->rdev;
 541
 542         bnxt_re_destroy_fence_mr(pd);
 543
 544         if (pd->qplib_pd.id)
 545                 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
 546                                       &pd->qplib_pd);
 547         return 0;
 548 }
 549
 550 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 551 {
 552         struct ib_device *ibdev = ibpd->device;
 553         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 554         struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(
 555                 udata, struct bnxt_re_ucontext, ib_uctx);
 556         struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);
 557         int rc;
 558
 559         pd->rdev = rdev;
 560         if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) {
 561                 ibdev_err(&rdev->ibdev, "Failed to allocate HW PD");
 562                 rc = -ENOMEM;
 563                 goto fail;
 564         }
 565
 566         if (udata) {
 567                 struct bnxt_re_pd_resp resp;
 568
 569                 if (!ucntx->dpi.dbr) {
 570                         /* Allocate DPI in alloc_pd to avoid failing of
 571                          * ibv_devinfo and family of application when DPIs
 572                          * are depleted.
 573                          */
 574                         if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl,
 575                                                  &ucntx->dpi, ucntx)) {
 576                                 rc = -ENOMEM;
 577                                 goto dbfail;
 578                         }
 579                 }
 580
 581                 resp.pdid = pd->qplib_pd.id;
 582                 /* Still allow mapping this DBR to the new user PD. */
 583                 resp.dpi = ucntx->dpi.dpi;
 584                 resp.dbr = (u64)ucntx->dpi.umdbr;
 585
 586                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
 587                 if (rc) {
 588                         ibdev_err(&rdev->ibdev,
 589                                   "Failed to copy user response\n");
 590                         goto dbfail;
 591                 }
 592         }
 593
 594         if (!udata)
 595                 if (bnxt_re_create_fence_mr(pd))
 596                         ibdev_warn(&rdev->ibdev,
 597                                    "Failed to create Fence-MR\n");
 598         return 0;
 599 dbfail:
 600         bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
 601                               &pd->qplib_pd);
 602 fail:
 603         return rc;
 604 }
 605
 606 /* Address Handles */
 607 int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
 608 {
 609         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 610         struct bnxt_re_dev *rdev = ah->rdev;
 611
 612         bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah,
 613                               !(flags & RDMA_DESTROY_AH_SLEEPABLE));
 614         return 0;
 615 }
 616
 617 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
 618 {
 619         u8 nw_type;
 620
 621         switch (ntype) {
 622         case RDMA_NETWORK_IPV4:
 623                 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
 624                 break;
 625         case RDMA_NETWORK_IPV6:
 626                 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
 627                 break;
 628         default:
 629                 nw_type = CMDQ_CREATE_AH_TYPE_V1;
 630                 break;
 631         }
 632         return nw_type;
 633 }
 634
 635 int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
 636                       struct ib_udata *udata)
 637 {
 638         struct ib_pd *ib_pd = ib_ah->pd;
 639         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 640         struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 641         const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
 642         struct bnxt_re_dev *rdev = pd->rdev;
 643         const struct ib_gid_attr *sgid_attr;
 644         struct bnxt_re_gid_ctx *ctx;
 645         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 646         u8 nw_type;
 647         int rc;
 648
 649         if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
 650                 ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set");
 651                 return -EINVAL;
 652         }
 653
 654         ah->rdev = rdev;
 655         ah->qplib_ah.pd = &pd->qplib_pd;
 656
 657         /* Supply the configuration for the HW */
 658         memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
 659                sizeof(union ib_gid));
 660         sgid_attr = grh->sgid_attr;
 661         /* Get the HW context of the GID. The reference
 662          * of GID table entry is already taken by the caller.
 663          */
 664         ctx = rdma_read_gid_hw_context(sgid_attr);
 665         ah->qplib_ah.sgid_index = ctx->idx;
 666         ah->qplib_ah.host_sgid_index = grh->sgid_index;
 667         ah->qplib_ah.traffic_class = grh->traffic_class;
 668         ah->qplib_ah.flow_label = grh->flow_label;
 669         ah->qplib_ah.hop_limit = grh->hop_limit;
 670         ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
 671
 672         /* Get network header type for this GID */
 673         nw_type = rdma_gid_attr_network_type(sgid_attr);
 674         ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type);
 675
 676         memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
 677         rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah,
 678                                   !(init_attr->flags &
 679                                     RDMA_CREATE_AH_SLEEPABLE));
 680         if (rc) {
 681                 ibdev_err(&rdev->ibdev, "Failed to allocate HW AH");
 682                 return rc;
 683         }
 684
 685         /* Write AVID to shared page. */
 686         if (udata) {
 687                 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
 688                         udata, struct bnxt_re_ucontext, ib_uctx);
 689                 unsigned long flag;
 690                 u32 *wrptr;
 691
 692                 spin_lock_irqsave(&uctx->sh_lock, flag);
 693                 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
 694                 *wrptr = ah->qplib_ah.id;
 695                 wmb(); /* make sure cache is updated. */
 696                 spin_unlock_irqrestore(&uctx->sh_lock, flag);
 697         }
 698
 699         return 0;
 700 }
 701
 702 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 703 {
 704         return 0;
 705 }
 706
 707 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 708 {
 709         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 710
 711         ah_attr->type = ib_ah->type;
 712         rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
 713         memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
 714         rdma_ah_set_grh(ah_attr, NULL, 0,
 715                         ah->qplib_ah.host_sgid_index,
 716                         0, ah->qplib_ah.traffic_class);
 717         rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data);
 718         rdma_ah_set_port_num(ah_attr, 1);
 719         rdma_ah_set_static_rate(ah_attr, 0);
 720         return 0;
 721 }
 722
 723 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
 724         __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
 725 {
 726         unsigned long flags;
 727
 728         spin_lock_irqsave(&qp->scq->cq_lock, flags);
 729         if (qp->rcq != qp->scq)
 730                 spin_lock(&qp->rcq->cq_lock);
 731         else
 732                 __acquire(&qp->rcq->cq_lock);
 733
 734         return flags;
 735 }
 736
 737 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
 738                         unsigned long flags)
 739         __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
 740 {
 741         if (qp->rcq != qp->scq)
 742                 spin_unlock(&qp->rcq->cq_lock);
 743         else
 744                 __release(&qp->rcq->cq_lock);
 745         spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
 746 }
 747
 748 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
 749 {
 750         struct bnxt_re_qp *gsi_sqp;
 751         struct bnxt_re_ah *gsi_sah;
 752         struct bnxt_re_dev *rdev;
 753         int rc = 0;
 754
 755         rdev = qp->rdev;
 756         gsi_sqp = rdev->gsi_ctx.gsi_sqp;
 757         gsi_sah = rdev->gsi_ctx.gsi_sah;
 758
 759         ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n");
 760         bnxt_qplib_destroy_ah(&rdev->qplib_res,
 761                               &gsi_sah->qplib_ah,
 762                               true);
 763         bnxt_qplib_clean_qp(&qp->qplib_qp);
 764
 765         ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n");
 766         rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
 767         if (rc) {
 768                 ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed");
 769                 goto fail;
 770         }
 771         bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
 772
 773         /* remove from active qp list */
 774         mutex_lock(&rdev->qp_lock);
 775         list_del(&gsi_sqp->list);
 776         mutex_unlock(&rdev->qp_lock);
 777         atomic_dec(&rdev->qp_count);
 778
 779         kfree(rdev->gsi_ctx.sqp_tbl);
 780         kfree(gsi_sah);
 781         kfree(gsi_sqp);
 782         rdev->gsi_ctx.gsi_sqp = NULL;
 783         rdev->gsi_ctx.gsi_sah = NULL;
 784         rdev->gsi_ctx.sqp_tbl = NULL;
 785
 786         return 0;
 787 fail:
 788         return rc;
 789 }
 790
 791 /* Queue Pairs */
 792 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 793 {
 794         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 795         struct bnxt_re_dev *rdev = qp->rdev;
 796         unsigned int flags;
 797         int rc;
 798
 799         bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
 800
 801         rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
 802         if (rc) {
 803                 ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
 804                 return rc;
 805         }
 806
 807         if (rdma_is_kernel_res(&qp->ib_qp.res)) {
 808                 flags = bnxt_re_lock_cqs(qp);
 809                 bnxt_qplib_clean_qp(&qp->qplib_qp);
 810                 bnxt_re_unlock_cqs(qp, flags);
 811         }
 812
 813         bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
 814
 815         if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) {
 816                 rc = bnxt_re_destroy_gsi_sqp(qp);
 817                 if (rc)
 818                         return rc;
 819         }
 820
 821         mutex_lock(&rdev->qp_lock);
 822         list_del(&qp->list);
 823         mutex_unlock(&rdev->qp_lock);
 824         atomic_dec(&rdev->qp_count);
 825
 826         ib_umem_release(qp->rumem);
 827         ib_umem_release(qp->sumem);
 828
 829         return 0;
 830 }
 831
 832 static u8 __from_ib_qp_type(enum ib_qp_type type)
 833 {
 834         switch (type) {
 835         case IB_QPT_GSI:
 836                 return CMDQ_CREATE_QP1_TYPE_GSI;
 837         case IB_QPT_RC:
 838                 return CMDQ_CREATE_QP_TYPE_RC;
 839         case IB_QPT_UD:
 840                 return CMDQ_CREATE_QP_TYPE_UD;
 841         default:
 842                 return IB_QPT_MAX;
 843         }
 844 }
 845
 846 static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
 847                                    int rsge, int max)
 848 {
 849         if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
 850                 rsge = max;
 851         return bnxt_re_get_rwqe_size(rsge);
 852 }
 853
 854 static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
 855 {
 856         u16 wqe_size, calc_ils;
 857
 858         wqe_size = bnxt_re_get_swqe_size(nsge);
 859         if (ilsize) {
 860                 calc_ils = sizeof(struct sq_send_hdr) + ilsize;
 861                 wqe_size = max_t(u16, calc_ils, wqe_size);
 862                 wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
 863         }
 864         return wqe_size;
 865 }
 866
 867 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
 868                                    struct ib_qp_init_attr *init_attr)
 869 {
 870         struct bnxt_qplib_dev_attr *dev_attr;
 871         struct bnxt_qplib_qp *qplqp;
 872         struct bnxt_re_dev *rdev;
 873         struct bnxt_qplib_q *sq;
 874         int align, ilsize;
 875
 876         rdev = qp->rdev;
 877         qplqp = &qp->qplib_qp;
 878         sq = &qplqp->sq;
 879         dev_attr = &rdev->dev_attr;
 880
 881         align = sizeof(struct sq_send_hdr);
 882         ilsize = ALIGN(init_attr->cap.max_inline_data, align);
 883
 884         sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge);
 885         if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges))
 886                 return -EINVAL;
 887         /* For gen p4 and gen p5 backward compatibility mode
 888          * wqe size is fixed to 128 bytes
 889          */
 890         if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) &&
 891                         qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
 892                 sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges);
 893
 894         if (init_attr->cap.max_inline_data) {
 895                 qplqp->max_inline_data = sq->wqe_size -
 896                         sizeof(struct sq_send_hdr);
 897                 init_attr->cap.max_inline_data = qplqp->max_inline_data;
 898                 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
 899                         sq->max_sge = qplqp->max_inline_data /
 900                                 sizeof(struct sq_sge);
 901         }
 902
 903         return 0;
 904 }
 905
 906 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
 907                                 struct bnxt_re_qp *qp, struct ib_udata *udata)
 908 {
 909         struct bnxt_qplib_qp *qplib_qp;
 910         struct bnxt_re_ucontext *cntx;
 911         struct bnxt_re_qp_req ureq;
 912         int bytes = 0, psn_sz;
 913         struct ib_umem *umem;
 914         int psn_nume;
 915
 916         qplib_qp = &qp->qplib_qp;
 917         cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
 918                                          ib_uctx);
 919         if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
 920                 return -EFAULT;
 921
 922         bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
 923         /* Consider mapping PSN search memory only for RC QPs. */
 924         if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
 925                 psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
 926                                                    sizeof(struct sq_psn_search_ext) :
 927                                                    sizeof(struct sq_psn_search);
 928                 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
 929                             qplib_qp->sq.max_wqe :
 930                             ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
 931                               sizeof(struct bnxt_qplib_sge));
 932                 bytes += (psn_nume * psn_sz);
 933         }
 934
 935         bytes = PAGE_ALIGN(bytes);
 936         umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes,
 937                            IB_ACCESS_LOCAL_WRITE);
 938         if (IS_ERR(umem))
 939                 return PTR_ERR(umem);
 940
 941         qp->sumem = umem;
 942         qplib_qp->sq.sg_info.umem = umem;
 943         qplib_qp->sq.sg_info.pgsize = PAGE_SIZE;
 944         qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT;
 945         qplib_qp->qp_handle = ureq.qp_handle;
 946
 947         if (!qp->qplib_qp.srq) {
 948                 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
 949                 bytes = PAGE_ALIGN(bytes);
 950                 umem = ib_umem_get(&rdev->ibdev, ureq.qprva, bytes,
 951                                    IB_ACCESS_LOCAL_WRITE);
 952                 if (IS_ERR(umem))
 953                         goto rqfail;
 954                 qp->rumem = umem;
 955                 qplib_qp->rq.sg_info.umem = umem;
 956                 qplib_qp->rq.sg_info.pgsize = PAGE_SIZE;
 957                 qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT;
 958         }
 959
 960         qplib_qp->dpi = &cntx->dpi;
 961         return 0;
 962 rqfail:
 963         ib_umem_release(qp->sumem);
 964         qp->sumem = NULL;
 965         memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
 966
 967         return PTR_ERR(umem);
 968 }
 969
 970 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
 971                                 (struct bnxt_re_pd *pd,
 972                                  struct bnxt_qplib_res *qp1_res,
 973                                  struct bnxt_qplib_qp *qp1_qp)
 974 {
 975         struct bnxt_re_dev *rdev = pd->rdev;
 976         struct bnxt_re_ah *ah;
 977         union ib_gid sgid;
 978         int rc;
 979
 980         ah = kzalloc(sizeof(*ah), GFP_KERNEL);
 981         if (!ah)
 982                 return NULL;
 983
 984         ah->rdev = rdev;
 985         ah->qplib_ah.pd = &pd->qplib_pd;
 986
 987         rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
 988         if (rc)
 989                 goto fail;
 990
 991         /* supply the dgid data same as sgid */
 992         memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
 993                sizeof(union ib_gid));
 994         ah->qplib_ah.sgid_index = 0;
 995
 996         ah->qplib_ah.traffic_class = 0;
 997         ah->qplib_ah.flow_label = 0;
 998         ah->qplib_ah.hop_limit = 1;
 999         ah->qplib_ah.sl = 0;
1000         /* Have DMAC same as SMAC */
1001         ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
1002
1003         rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false);
1004         if (rc) {
1005                 ibdev_err(&rdev->ibdev,
1006                           "Failed to allocate HW AH for Shadow QP");
1007                 goto fail;
1008         }
1009
1010         return ah;
1011
1012 fail:
1013         kfree(ah);
1014         return NULL;
1015 }
1016
1017 static struct bnxt_re_qp *bnxt_re_create_shadow_qp
1018                                 (struct bnxt_re_pd *pd,
1019                                  struct bnxt_qplib_res *qp1_res,
1020                                  struct bnxt_qplib_qp *qp1_qp)
1021 {
1022         struct bnxt_re_dev *rdev = pd->rdev;
1023         struct bnxt_re_qp *qp;
1024         int rc;
1025
1026         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1027         if (!qp)
1028                 return NULL;
1029
1030         qp->rdev = rdev;
1031
1032         /* Initialize the shadow QP structure from the QP1 values */
1033         ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
1034
1035         qp->qplib_qp.pd = &pd->qplib_pd;
1036         qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
1037         qp->qplib_qp.type = IB_QPT_UD;
1038
1039         qp->qplib_qp.max_inline_data = 0;
1040         qp->qplib_qp.sig_type = true;
1041
1042         /* Shadow QP SQ depth should be same as QP1 RQ depth */
1043         qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);
1044         qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
1045         qp->qplib_qp.sq.max_sge = 2;
1046         /* Q full delta can be 1 since it is internal QP */
1047         qp->qplib_qp.sq.q_full_delta = 1;
1048         qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE;
1049         qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT;
1050
1051         qp->qplib_qp.scq = qp1_qp->scq;
1052         qp->qplib_qp.rcq = qp1_qp->rcq;
1053
1054         qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);
1055         qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
1056         qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
1057         /* Q full delta can be 1 since it is internal QP */
1058         qp->qplib_qp.rq.q_full_delta = 1;
1059         qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE;
1060         qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT;
1061
1062         qp->qplib_qp.mtu = qp1_qp->mtu;
1063
1064         qp->qplib_qp.sq_hdr_buf_size = 0;
1065         qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
1066         qp->qplib_qp.dpi = &rdev->dpi_privileged;
1067
1068         rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
1069         if (rc)
1070                 goto fail;
1071
1072         spin_lock_init(&qp->sq_lock);
1073         INIT_LIST_HEAD(&qp->list);
1074         mutex_lock(&rdev->qp_lock);
1075         list_add_tail(&qp->list, &rdev->qp_list);
1076         atomic_inc(&rdev->qp_count);
1077         mutex_unlock(&rdev->qp_lock);
1078         return qp;
1079 fail:
1080         kfree(qp);
1081         return NULL;
1082 }
1083
1084 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
1085                                 struct ib_qp_init_attr *init_attr)
1086 {
1087         struct bnxt_qplib_dev_attr *dev_attr;
1088         struct bnxt_qplib_qp *qplqp;
1089         struct bnxt_re_dev *rdev;
1090         struct bnxt_qplib_q *rq;
1091         int entries;
1092
1093         rdev = qp->rdev;
1094         qplqp = &qp->qplib_qp;
1095         rq = &qplqp->rq;
1096         dev_attr = &rdev->dev_attr;
1097
1098         if (init_attr->srq) {
1099                 struct bnxt_re_srq *srq;
1100
1101                 srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq);
1102                 qplqp->srq = &srq->qplib_srq;
1103                 rq->max_wqe = 0;
1104         } else {
1105                 rq->max_sge = init_attr->cap.max_recv_sge;
1106                 if (rq->max_sge > dev_attr->max_qp_sges)
1107                         rq->max_sge = dev_attr->max_qp_sges;
1108                 init_attr->cap.max_recv_sge = rq->max_sge;
1109                 rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
1110                                                        dev_attr->max_qp_sges);
1111                 /* Allocate 1 more than what's provided so posting max doesn't
1112                  * mean empty.
1113                  */
1114                 entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1);
1115                 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1116                 rq->q_full_delta = 0;
1117                 rq->sg_info.pgsize = PAGE_SIZE;
1118                 rq->sg_info.pgshft = PAGE_SHIFT;
1119         }
1120
1121         return 0;
1122 }
1123
1124 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
1125 {
1126         struct bnxt_qplib_dev_attr *dev_attr;
1127         struct bnxt_qplib_qp *qplqp;
1128         struct bnxt_re_dev *rdev;
1129
1130         rdev = qp->rdev;
1131         qplqp = &qp->qplib_qp;
1132         dev_attr = &rdev->dev_attr;
1133
1134         if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1135                 qplqp->rq.max_sge = dev_attr->max_qp_sges;
1136                 if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
1137                         qplqp->rq.max_sge = dev_attr->max_qp_sges;
1138                 qplqp->rq.max_sge = 6;
1139         }
1140 }
1141
1142 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
1143                                 struct ib_qp_init_attr *init_attr,
1144                                 struct ib_udata *udata)
1145 {
1146         struct bnxt_qplib_dev_attr *dev_attr;
1147         struct bnxt_qplib_qp *qplqp;
1148         struct bnxt_re_dev *rdev;
1149         struct bnxt_qplib_q *sq;
1150         int entries;
1151         int diff;
1152         int rc;
1153
1154         rdev = qp->rdev;
1155         qplqp = &qp->qplib_qp;
1156         sq = &qplqp->sq;
1157         dev_attr = &rdev->dev_attr;
1158
1159         sq->max_sge = init_attr->cap.max_send_sge;
1160         if (sq->max_sge > dev_attr->max_qp_sges) {
1161                 sq->max_sge = dev_attr->max_qp_sges;
1162                 init_attr->cap.max_send_sge = sq->max_sge;
1163         }
1164
1165         rc = bnxt_re_setup_swqe_size(qp, init_attr);
1166         if (rc)
1167                 return rc;
1168
1169         entries = init_attr->cap.max_send_wr;
1170         /* Allocate 128 + 1 more than what's provided */
1171         diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
1172                 0 : BNXT_QPLIB_RESERVED_QP_WRS;
1173         entries = roundup_pow_of_two(entries + diff + 1);
1174         sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
1175         sq->q_full_delta = diff + 1;
1176         /*
1177          * Reserving one slot for Phantom WQE. Application can
1178          * post one extra entry in this case. But allowing this to avoid
1179          * unexpected Queue full condition
1180          */
1181         qplqp->sq.q_full_delta -= 1;
1182         qplqp->sq.sg_info.pgsize = PAGE_SIZE;
1183         qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
1184
1185         return 0;
1186 }
1187
1188 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
1189                                        struct ib_qp_init_attr *init_attr)
1190 {
1191         struct bnxt_qplib_dev_attr *dev_attr;
1192         struct bnxt_qplib_qp *qplqp;
1193         struct bnxt_re_dev *rdev;
1194         int entries;
1195
1196         rdev = qp->rdev;
1197         qplqp = &qp->qplib_qp;
1198         dev_attr = &rdev->dev_attr;
1199
1200         if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1201                 entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1);
1202                 qplqp->sq.max_wqe = min_t(u32, entries,
1203                                           dev_attr->max_qp_wqes + 1);
1204                 qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
1205                         init_attr->cap.max_send_wr;
1206                 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
1207                 if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
1208                         qplqp->sq.max_sge = dev_attr->max_qp_sges;
1209         }
1210 }
1211
1212 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
1213                                 struct ib_qp_init_attr *init_attr)
1214 {
1215         struct bnxt_qplib_chip_ctx *chip_ctx;
1216         int qptype;
1217
1218         chip_ctx = rdev->chip_ctx;
1219
1220         qptype = __from_ib_qp_type(init_attr->qp_type);
1221         if (qptype == IB_QPT_MAX) {
1222                 ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype);
1223                 qptype = -EOPNOTSUPP;
1224                 goto out;
1225         }
1226
1227         if (bnxt_qplib_is_chip_gen_p5(chip_ctx) &&
1228             init_attr->qp_type == IB_QPT_GSI)
1229                 qptype = CMDQ_CREATE_QP_TYPE_GSI;
1230 out:
1231         return qptype;
1232 }
1233
1234 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1235                                 struct ib_qp_init_attr *init_attr,
1236                                 struct ib_udata *udata)
1237 {
1238         struct bnxt_qplib_dev_attr *dev_attr;
1239         struct bnxt_qplib_qp *qplqp;
1240         struct bnxt_re_dev *rdev;
1241         struct bnxt_re_cq *cq;
1242         int rc = 0, qptype;
1243
1244         rdev = qp->rdev;
1245         qplqp = &qp->qplib_qp;
1246         dev_attr = &rdev->dev_attr;
1247
1248         /* Setup misc params */
1249         ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr);
1250         qplqp->pd = &pd->qplib_pd;
1251         qplqp->qp_handle = (u64)qplqp;
1252         qplqp->max_inline_data = init_attr->cap.max_inline_data;
1253         qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ?
1254                             true : false);
1255         qptype = bnxt_re_init_qp_type(rdev, init_attr);
1256         if (qptype < 0) {
1257                 rc = qptype;
1258                 goto out;
1259         }
1260         qplqp->type = (u8)qptype;
1261         qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode;
1262
1263         if (init_attr->qp_type == IB_QPT_RC) {
1264                 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
1265                 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1266         }
1267         qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1268         qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
1269         if (init_attr->create_flags) {
1270                 ibdev_dbg(&rdev->ibdev,
1271                           "QP create flags 0x%x not supported",
1272                           init_attr->create_flags);
1273                 return -EOPNOTSUPP;
1274         }
1275
1276         /* Setup CQs */
1277         if (init_attr->send_cq) {
1278                 cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq);
1279                 qplqp->scq = &cq->qplib_cq;
1280                 qp->scq = cq;
1281         }
1282
1283         if (init_attr->recv_cq) {
1284                 cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq);
1285                 qplqp->rcq = &cq->qplib_cq;
1286                 qp->rcq = cq;
1287         }
1288
1289         /* Setup RQ/SRQ */
1290         rc = bnxt_re_init_rq_attr(qp, init_attr);
1291         if (rc)
1292                 goto out;
1293         if (init_attr->qp_type == IB_QPT_GSI)
1294                 bnxt_re_adjust_gsi_rq_attr(qp);
1295
1296         /* Setup SQ */
1297         rc = bnxt_re_init_sq_attr(qp, init_attr, udata);
1298         if (rc)
1299                 goto out;
1300         if (init_attr->qp_type == IB_QPT_GSI)
1301                 bnxt_re_adjust_gsi_sq_attr(qp, init_attr);
1302
1303         if (udata) /* This will update DPI and qp_handle */
1304                 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
1305 out:
1306         return rc;
1307 }
1308
1309 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
1310                                      struct bnxt_re_pd *pd)
1311 {
1312         struct bnxt_re_sqp_entries *sqp_tbl;
1313         struct bnxt_re_dev *rdev;
1314         struct bnxt_re_qp *sqp;
1315         struct bnxt_re_ah *sah;
1316         int rc = 0;
1317
1318         rdev = qp->rdev;
1319         /* Create a shadow QP to handle the QP1 traffic */
1320         sqp_tbl = kcalloc(BNXT_RE_MAX_GSI_SQP_ENTRIES, sizeof(*sqp_tbl),
1321                           GFP_KERNEL);
1322         if (!sqp_tbl)
1323                 return -ENOMEM;
1324         rdev->gsi_ctx.sqp_tbl = sqp_tbl;
1325
1326         sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
1327         if (!sqp) {
1328                 rc = -ENODEV;
1329                 ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1");
1330                 goto out;
1331         }
1332         rdev->gsi_ctx.gsi_sqp = sqp;
1333
1334         sqp->rcq = qp->rcq;
1335         sqp->scq = qp->scq;
1336         sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
1337                                           &qp->qplib_qp);
1338         if (!sah) {
1339                 bnxt_qplib_destroy_qp(&rdev->qplib_res,
1340                                       &sqp->qplib_qp);
1341                 rc = -ENODEV;
1342                 ibdev_err(&rdev->ibdev,
1343                           "Failed to create AH entry for ShadowQP");
1344                 goto out;
1345         }
1346         rdev->gsi_ctx.gsi_sah = sah;
1347
1348         return 0;
1349 out:
1350         kfree(sqp_tbl);
1351         return rc;
1352 }
1353
1354 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1355                                  struct ib_qp_init_attr *init_attr)
1356 {
1357         struct bnxt_re_dev *rdev;
1358         struct bnxt_qplib_qp *qplqp;
1359         int rc = 0;
1360
1361         rdev = qp->rdev;
1362         qplqp = &qp->qplib_qp;
1363
1364         qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1365         qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
1366
1367         rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp);
1368         if (rc) {
1369                 ibdev_err(&rdev->ibdev, "create HW QP1 failed!");
1370                 goto out;
1371         }
1372
1373         rc = bnxt_re_create_shadow_gsi(qp, pd);
1374 out:
1375         return rc;
1376 }
1377
1378 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
1379                                    struct ib_qp_init_attr *init_attr,
1380                                    struct bnxt_qplib_dev_attr *dev_attr)
1381 {
1382         bool rc = true;
1383
1384         if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes ||
1385             init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes ||
1386             init_attr->cap.max_send_sge > dev_attr->max_qp_sges ||
1387             init_attr->cap.max_recv_sge > dev_attr->max_qp_sges ||
1388             init_attr->cap.max_inline_data > dev_attr->max_inline_data) {
1389                 ibdev_err(&rdev->ibdev,
1390                           "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",
1391                           init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
1392                           init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
1393                           init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
1394                           init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
1395                           init_attr->cap.max_inline_data,
1396                           dev_attr->max_inline_data);
1397                 rc = false;
1398         }
1399         return rc;
1400 }
1401
1402 int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
1403                       struct ib_udata *udata)
1404 {
1405         struct ib_pd *ib_pd = ib_qp->pd;
1406         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1407         struct bnxt_re_dev *rdev = pd->rdev;
1408         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1409         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1410         int rc;
1411
1412         rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
1413         if (!rc) {
1414                 rc = -EINVAL;
1415                 goto fail;
1416         }
1417
1418         qp->rdev = rdev;
1419         rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata);
1420         if (rc)
1421                 goto fail;
1422
1423         if (qp_init_attr->qp_type == IB_QPT_GSI &&
1424             !(bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))) {
1425                 rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr);
1426                 if (rc == -ENODEV)
1427                         goto qp_destroy;
1428                 if (rc)
1429                         goto fail;
1430         } else {
1431                 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1432                 if (rc) {
1433                         ibdev_err(&rdev->ibdev, "Failed to create HW QP");
1434                         goto free_umem;
1435                 }
1436                 if (udata) {
1437                         struct bnxt_re_qp_resp resp;
1438
1439                         resp.qpid = qp->qplib_qp.id;
1440                         resp.rsvd = 0;
1441                         rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1442                         if (rc) {
1443                                 ibdev_err(&rdev->ibdev, "Failed to copy QP udata");
1444                                 goto qp_destroy;
1445                         }
1446                 }
1447         }
1448
1449         qp->ib_qp.qp_num = qp->qplib_qp.id;
1450         if (qp_init_attr->qp_type == IB_QPT_GSI)
1451                 rdev->gsi_ctx.gsi_qp = qp;
1452         spin_lock_init(&qp->sq_lock);
1453         spin_lock_init(&qp->rq_lock);
1454         INIT_LIST_HEAD(&qp->list);
1455         mutex_lock(&rdev->qp_lock);
1456         list_add_tail(&qp->list, &rdev->qp_list);
1457         mutex_unlock(&rdev->qp_lock);
1458         atomic_inc(&rdev->qp_count);
1459
1460         return 0;
1461 qp_destroy:
1462         bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1463 free_umem:
1464         ib_umem_release(qp->rumem);
1465         ib_umem_release(qp->sumem);
1466 fail:
1467         return rc;
1468 }
1469
1470 static u8 __from_ib_qp_state(enum ib_qp_state state)
1471 {
1472         switch (state) {
1473         case IB_QPS_RESET:
1474                 return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1475         case IB_QPS_INIT:
1476                 return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1477         case IB_QPS_RTR:
1478                 return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1479         case IB_QPS_RTS:
1480                 return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1481         case IB_QPS_SQD:
1482                 return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1483         case IB_QPS_SQE:
1484                 return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1485         case IB_QPS_ERR:
1486         default:
1487                 return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1488         }
1489 }
1490
1491 static enum ib_qp_state __to_ib_qp_state(u8 state)
1492 {
1493         switch (state) {
1494         case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1495                 return IB_QPS_RESET;
1496         case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1497                 return IB_QPS_INIT;
1498         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1499                 return IB_QPS_RTR;
1500         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1501                 return IB_QPS_RTS;
1502         case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1503                 return IB_QPS_SQD;
1504         case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1505                 return IB_QPS_SQE;
1506         case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1507         default:
1508                 return IB_QPS_ERR;
1509         }
1510 }
1511
1512 static u32 __from_ib_mtu(enum ib_mtu mtu)
1513 {
1514         switch (mtu) {
1515         case IB_MTU_256:
1516                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1517         case IB_MTU_512:
1518                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1519         case IB_MTU_1024:
1520                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1521         case IB_MTU_2048:
1522                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1523         case IB_MTU_4096:
1524                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1525         default:
1526                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1527         }
1528 }
1529
1530 static enum ib_mtu __to_ib_mtu(u32 mtu)
1531 {
1532         switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1533         case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1534                 return IB_MTU_256;
1535         case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1536                 return IB_MTU_512;
1537         case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1538                 return IB_MTU_1024;
1539         case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1540                 return IB_MTU_2048;
1541         case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1542                 return IB_MTU_4096;
1543         default:
1544                 return IB_MTU_2048;
1545         }
1546 }
1547
1548 /* Shared Receive Queues */
1549 int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
1550 {
1551         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1552                                                ib_srq);
1553         struct bnxt_re_dev *rdev = srq->rdev;
1554         struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1555         struct bnxt_qplib_nq *nq = NULL;
1556
1557         if (qplib_srq->cq)
1558                 nq = qplib_srq->cq->nq;
1559         bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
1560         ib_umem_release(srq->umem);
1561         atomic_dec(&rdev->srq_count);
1562         if (nq)
1563                 nq->budget--;
1564         return 0;
1565 }
1566
1567 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
1568                                  struct bnxt_re_pd *pd,
1569                                  struct bnxt_re_srq *srq,
1570                                  struct ib_udata *udata)
1571 {
1572         struct bnxt_re_srq_req ureq;
1573         struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1574         struct ib_umem *umem;
1575         int bytes = 0;
1576         struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
1577                 udata, struct bnxt_re_ucontext, ib_uctx);
1578
1579         if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1580                 return -EFAULT;
1581
1582         bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
1583         bytes = PAGE_ALIGN(bytes);
1584         umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes,
1585                            IB_ACCESS_LOCAL_WRITE);
1586         if (IS_ERR(umem))
1587                 return PTR_ERR(umem);
1588
1589         srq->umem = umem;
1590         qplib_srq->sg_info.umem = umem;
1591         qplib_srq->sg_info.pgsize = PAGE_SIZE;
1592         qplib_srq->sg_info.pgshft = PAGE_SHIFT;
1593         qplib_srq->srq_handle = ureq.srq_handle;
1594         qplib_srq->dpi = &cntx->dpi;
1595
1596         return 0;
1597 }
1598
1599 int bnxt_re_create_srq(struct ib_srq *ib_srq,
1600                        struct ib_srq_init_attr *srq_init_attr,
1601                        struct ib_udata *udata)
1602 {
1603         struct bnxt_qplib_dev_attr *dev_attr;
1604         struct bnxt_qplib_nq *nq = NULL;
1605         struct bnxt_re_dev *rdev;
1606         struct bnxt_re_srq *srq;
1607         struct bnxt_re_pd *pd;
1608         struct ib_pd *ib_pd;
1609         int rc, entries;
1610
1611         ib_pd = ib_srq->pd;
1612         pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1613         rdev = pd->rdev;
1614         dev_attr = &rdev->dev_attr;
1615         srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);
1616
1617         if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
1618                 ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded");
1619                 rc = -EINVAL;
1620                 goto exit;
1621         }
1622
1623         if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
1624                 rc = -EOPNOTSUPP;
1625                 goto exit;
1626         }
1627
1628         srq->rdev = rdev;
1629         srq->qplib_srq.pd = &pd->qplib_pd;
1630         srq->qplib_srq.dpi = &rdev->dpi_privileged;
1631         /* Allocate 1 more than what's provided so posting max doesn't
1632          * mean empty
1633          */
1634         entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
1635         if (entries > dev_attr->max_srq_wqes + 1)
1636                 entries = dev_attr->max_srq_wqes + 1;
1637         srq->qplib_srq.max_wqe = entries;
1638
1639         srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
1640          /* 128 byte wqe size for SRQ . So use max sges */
1641         srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);
1642         srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
1643         srq->srq_limit = srq_init_attr->attr.srq_limit;
1644         srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
1645         nq = &rdev->nq[0];
1646
1647         if (udata) {
1648                 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
1649                 if (rc)
1650                         goto fail;
1651         }
1652
1653         rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
1654         if (rc) {
1655                 ibdev_err(&rdev->ibdev, "Create HW SRQ failed!");
1656                 goto fail;
1657         }
1658
1659         if (udata) {
1660                 struct bnxt_re_srq_resp resp;
1661
1662                 resp.srqid = srq->qplib_srq.id;
1663                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1664                 if (rc) {
1665                         ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!");
1666                         bnxt_qplib_destroy_srq(&rdev->qplib_res,
1667                                                &srq->qplib_srq);
1668                         goto fail;
1669                 }
1670         }
1671         if (nq)
1672                 nq->budget++;
1673         atomic_inc(&rdev->srq_count);
1674
1675         return 0;
1676
1677 fail:
1678         ib_umem_release(srq->umem);
1679 exit:
1680         return rc;
1681 }
1682
1683 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
1684                        enum ib_srq_attr_mask srq_attr_mask,
1685                        struct ib_udata *udata)
1686 {
1687         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1688                                                ib_srq);
1689         struct bnxt_re_dev *rdev = srq->rdev;
1690         int rc;
1691
1692         switch (srq_attr_mask) {
1693         case IB_SRQ_MAX_WR:
1694                 /* SRQ resize is not supported */
1695                 break;
1696         case IB_SRQ_LIMIT:
1697                 /* Change the SRQ threshold */
1698                 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
1699                         return -EINVAL;
1700
1701                 srq->qplib_srq.threshold = srq_attr->srq_limit;
1702                 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
1703                 if (rc) {
1704                         ibdev_err(&rdev->ibdev, "Modify HW SRQ failed!");
1705                         return rc;
1706                 }
1707                 /* On success, update the shadow */
1708                 srq->srq_limit = srq_attr->srq_limit;
1709                 /* No need to Build and send response back to udata */
1710                 break;
1711         default:
1712                 ibdev_err(&rdev->ibdev,
1713                           "Unsupported srq_attr_mask 0x%x", srq_attr_mask);
1714                 return -EINVAL;
1715         }
1716         return 0;
1717 }
1718
1719 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
1720 {
1721         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1722                                                ib_srq);
1723         struct bnxt_re_srq tsrq;
1724         struct bnxt_re_dev *rdev = srq->rdev;
1725         int rc;
1726
1727         /* Get live SRQ attr */
1728         tsrq.qplib_srq.id = srq->qplib_srq.id;
1729         rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq);
1730         if (rc) {
1731                 ibdev_err(&rdev->ibdev, "Query HW SRQ failed!");
1732                 return rc;
1733         }
1734         srq_attr->max_wr = srq->qplib_srq.max_wqe;
1735         srq_attr->max_sge = srq->qplib_srq.max_sge;
1736         srq_attr->srq_limit = tsrq.qplib_srq.threshold;
1737
1738         return 0;
1739 }
1740
1741 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
1742                           const struct ib_recv_wr **bad_wr)
1743 {
1744         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1745                                                ib_srq);
1746         struct bnxt_qplib_swqe wqe;
1747         unsigned long flags;
1748         int rc = 0;
1749
1750         spin_lock_irqsave(&srq->lock, flags);
1751         while (wr) {
1752                 /* Transcribe each ib_recv_wr to qplib_swqe */
1753                 wqe.num_sge = wr->num_sge;
1754                 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
1755                 wqe.wr_id = wr->wr_id;
1756                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
1757
1758                 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
1759                 if (rc) {
1760                         *bad_wr = wr;
1761                         break;
1762                 }
1763                 wr = wr->next;
1764         }
1765         spin_unlock_irqrestore(&srq->lock, flags);
1766
1767         return rc;
1768 }
1769 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
1770                                     struct bnxt_re_qp *qp1_qp,
1771                                     int qp_attr_mask)
1772 {
1773         struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
1774         int rc = 0;
1775
1776         if (qp_attr_mask & IB_QP_STATE) {
1777                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1778                 qp->qplib_qp.state = qp1_qp->qplib_qp.state;
1779         }
1780         if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1781                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1782                 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
1783         }
1784
1785         if (qp_attr_mask & IB_QP_QKEY) {
1786                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1787                 /* Using a Random  QKEY */
1788                 qp->qplib_qp.qkey = 0x81818181;
1789         }
1790         if (qp_attr_mask & IB_QP_SQ_PSN) {
1791                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1792                 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
1793         }
1794
1795         rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1796         if (rc)
1797                 ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1");
1798         return rc;
1799 }
1800
1801 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1802                       int qp_attr_mask, struct ib_udata *udata)
1803 {
1804         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1805         struct bnxt_re_dev *rdev = qp->rdev;
1806         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1807         enum ib_qp_state curr_qp_state, new_qp_state;
1808         int rc, entries;
1809         unsigned int flags;
1810         u8 nw_type;
1811
1812         if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1813                 return -EOPNOTSUPP;
1814
1815         qp->qplib_qp.modify_flags = 0;
1816         if (qp_attr_mask & IB_QP_STATE) {
1817                 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
1818                 new_qp_state = qp_attr->qp_state;
1819                 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
1820                                         ib_qp->qp_type, qp_attr_mask)) {
1821                         ibdev_err(&rdev->ibdev,
1822                                   "Invalid attribute mask: %#x specified ",
1823                                   qp_attr_mask);
1824                         ibdev_err(&rdev->ibdev,
1825                                   "for qpn: %#x type: %#x",
1826                                   ib_qp->qp_num, ib_qp->qp_type);
1827                         ibdev_err(&rdev->ibdev,
1828                                   "curr_qp_state=0x%x, new_qp_state=0x%x\n",
1829                                   curr_qp_state, new_qp_state);
1830                         return -EINVAL;
1831                 }
1832                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1833                 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
1834
1835                 if (!qp->sumem &&
1836                     qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1837                         ibdev_dbg(&rdev->ibdev,
1838                                   "Move QP = %p to flush list\n", qp);
1839                         flags = bnxt_re_lock_cqs(qp);
1840                         bnxt_qplib_add_flush_qp(&qp->qplib_qp);
1841                         bnxt_re_unlock_cqs(qp, flags);
1842                 }
1843                 if (!qp->sumem &&
1844                     qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
1845                         ibdev_dbg(&rdev->ibdev,
1846                                   "Move QP = %p out of flush list\n", qp);
1847                         flags = bnxt_re_lock_cqs(qp);
1848                         bnxt_qplib_clean_qp(&qp->qplib_qp);
1849                         bnxt_re_unlock_cqs(qp, flags);
1850                 }
1851         }
1852         if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
1853                 qp->qplib_qp.modify_flags |=
1854                                 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
1855                 qp->qplib_qp.en_sqd_async_notify = true;
1856         }
1857         if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
1858                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
1859                 qp->qplib_qp.access =
1860                         __from_ib_access_flags(qp_attr->qp_access_flags);
1861                 /* LOCAL_WRITE access must be set to allow RC receive */
1862                 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1863                 /* Temp: Set all params on QP as of now */
1864                 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
1865                 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
1866         }
1867         if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1868                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1869                 qp->qplib_qp.pkey_index = qp_attr->pkey_index;
1870         }
1871         if (qp_attr_mask & IB_QP_QKEY) {
1872                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1873                 qp->qplib_qp.qkey = qp_attr->qkey;
1874         }
1875         if (qp_attr_mask & IB_QP_AV) {
1876                 const struct ib_global_route *grh =
1877                         rdma_ah_read_grh(&qp_attr->ah_attr);
1878                 const struct ib_gid_attr *sgid_attr;
1879                 struct bnxt_re_gid_ctx *ctx;
1880
1881                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1882                                      CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1883                                      CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1884                                      CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1885                                      CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1886                                      CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1887                                      CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1888                 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
1889                        sizeof(qp->qplib_qp.ah.dgid.data));
1890                 qp->qplib_qp.ah.flow_label = grh->flow_label;
1891                 sgid_attr = grh->sgid_attr;
1892                 /* Get the HW context of the GID. The reference
1893                  * of GID table entry is already taken by the caller.
1894                  */
1895                 ctx = rdma_read_gid_hw_context(sgid_attr);
1896                 qp->qplib_qp.ah.sgid_index = ctx->idx;
1897                 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
1898                 qp->qplib_qp.ah.hop_limit = grh->hop_limit;
1899                 qp->qplib_qp.ah.traffic_class = grh->traffic_class;
1900                 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
1901                 ether_addr_copy(qp->qplib_qp.ah.dmac,
1902                                 qp_attr->ah_attr.roce.dmac);
1903
1904                 rc = rdma_read_gid_l2_fields(sgid_attr, NULL,
1905                                              &qp->qplib_qp.smac[0]);
1906                 if (rc)
1907                         return rc;
1908
1909                 nw_type = rdma_gid_attr_network_type(sgid_attr);
1910                 switch (nw_type) {
1911                 case RDMA_NETWORK_IPV4:
1912                         qp->qplib_qp.nw_type =
1913                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
1914                         break;
1915                 case RDMA_NETWORK_IPV6:
1916                         qp->qplib_qp.nw_type =
1917                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
1918                         break;
1919                 default:
1920                         qp->qplib_qp.nw_type =
1921                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
1922                         break;
1923                 }
1924         }
1925
1926         if (qp_attr_mask & IB_QP_PATH_MTU) {
1927                 qp->qplib_qp.modify_flags |=
1928                                 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1929                 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu);
1930                 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu);
1931         } else if (qp_attr->qp_state == IB_QPS_RTR) {
1932                 qp->qplib_qp.modify_flags |=
1933                         CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1934                 qp->qplib_qp.path_mtu =
1935                         __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu));
1936                 qp->qplib_qp.mtu =
1937                         ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1938         }
1939
1940         if (qp_attr_mask & IB_QP_TIMEOUT) {
1941                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
1942                 qp->qplib_qp.timeout = qp_attr->timeout;
1943         }
1944         if (qp_attr_mask & IB_QP_RETRY_CNT) {
1945                 qp->qplib_qp.modify_flags |=
1946                                 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
1947                 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
1948         }
1949         if (qp_attr_mask & IB_QP_RNR_RETRY) {
1950                 qp->qplib_qp.modify_flags |=
1951                                 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
1952                 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
1953         }
1954         if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
1955                 qp->qplib_qp.modify_flags |=
1956                                 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
1957                 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
1958         }
1959         if (qp_attr_mask & IB_QP_RQ_PSN) {
1960                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
1961                 qp->qplib_qp.rq.psn = qp_attr->rq_psn;
1962         }
1963         if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1964                 qp->qplib_qp.modify_flags |=
1965                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
1966                 /* Cap the max_rd_atomic to device max */
1967                 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
1968                                                    dev_attr->max_qp_rd_atom);
1969         }
1970         if (qp_attr_mask & IB_QP_SQ_PSN) {
1971                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1972                 qp->qplib_qp.sq.psn = qp_attr->sq_psn;
1973         }
1974         if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1975                 if (qp_attr->max_dest_rd_atomic >
1976                     dev_attr->max_qp_init_rd_atom) {
1977                         ibdev_err(&rdev->ibdev,
1978                                   "max_dest_rd_atomic requested%d is > dev_max%d",
1979                                   qp_attr->max_dest_rd_atomic,
1980                                   dev_attr->max_qp_init_rd_atom);
1981                         return -EINVAL;
1982                 }
1983
1984                 qp->qplib_qp.modify_flags |=
1985                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
1986                 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
1987         }
1988         if (qp_attr_mask & IB_QP_CAP) {
1989                 qp->qplib_qp.modify_flags |=
1990                                 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
1991                                 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
1992                                 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
1993                                 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
1994                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
1995                 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
1996                     (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
1997                     (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
1998                     (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
1999                     (qp_attr->cap.max_inline_data >=
2000                                                 dev_attr->max_inline_data)) {
2001                         ibdev_err(&rdev->ibdev,
2002                                   "Create QP failed - max exceeded");
2003                         return -EINVAL;
2004                 }
2005                 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
2006                 qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
2007                                                 dev_attr->max_qp_wqes + 1);
2008                 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
2009                                                 qp_attr->cap.max_send_wr;
2010                 /*
2011                  * Reserving one slot for Phantom WQE. Some application can
2012                  * post one extra entry in this case. Allowing this to avoid
2013                  * unexpected Queue full condition
2014                  */
2015                 qp->qplib_qp.sq.q_full_delta -= 1;
2016                 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
2017                 if (qp->qplib_qp.rq.max_wqe) {
2018                         entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
2019                         qp->qplib_qp.rq.max_wqe =
2020                                 min_t(u32, entries, dev_attr->max_qp_wqes + 1);
2021                         qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
2022                                                        qp_attr->cap.max_recv_wr;
2023                         qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
2024                 } else {
2025                         /* SRQ was used prior, just ignore the RQ caps */
2026                 }
2027         }
2028         if (qp_attr_mask & IB_QP_DEST_QPN) {
2029                 qp->qplib_qp.modify_flags |=
2030                                 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
2031                 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
2032         }
2033         rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2034         if (rc) {
2035                 ibdev_err(&rdev->ibdev, "Failed to modify HW QP");
2036                 return rc;
2037         }
2038         if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
2039                 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
2040         return rc;
2041 }
2042
2043 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2044                      int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2045 {
2046         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2047         struct bnxt_re_dev *rdev = qp->rdev;
2048         struct bnxt_qplib_qp *qplib_qp;
2049         int rc;
2050
2051         qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL);
2052         if (!qplib_qp)
2053                 return -ENOMEM;
2054
2055         qplib_qp->id = qp->qplib_qp.id;
2056         qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
2057
2058         rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
2059         if (rc) {
2060                 ibdev_err(&rdev->ibdev, "Failed to query HW QP");
2061                 goto out;
2062         }
2063         qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
2064         qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
2065         qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
2066         qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access);
2067         qp_attr->pkey_index = qplib_qp->pkey_index;
2068         qp_attr->qkey = qplib_qp->qkey;
2069         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2070         rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label,
2071                         qplib_qp->ah.host_sgid_index,
2072                         qplib_qp->ah.hop_limit,
2073                         qplib_qp->ah.traffic_class);
2074         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data);
2075         rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl);
2076         ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac);
2077         qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
2078         qp_attr->timeout = qplib_qp->timeout;
2079         qp_attr->retry_cnt = qplib_qp->retry_cnt;
2080         qp_attr->rnr_retry = qplib_qp->rnr_retry;
2081         qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
2082         qp_attr->rq_psn = qplib_qp->rq.psn;
2083         qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
2084         qp_attr->sq_psn = qplib_qp->sq.psn;
2085         qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
2086         qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
2087                                                          IB_SIGNAL_REQ_WR;
2088         qp_attr->dest_qp_num = qplib_qp->dest_qpn;
2089
2090         qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
2091         qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
2092         qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
2093         qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
2094         qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
2095         qp_init_attr->cap = qp_attr->cap;
2096
2097 out:
2098         kfree(qplib_qp);
2099         return rc;
2100 }
2101
2102 /* Routine for sending QP1 packets for RoCE V1 an V2
2103  */
2104 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
2105                                      const struct ib_send_wr *wr,
2106                                      struct bnxt_qplib_swqe *wqe,
2107                                      int payload_size)
2108 {
2109         struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
2110                                              ib_ah);
2111         struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
2112         const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr;
2113         struct bnxt_qplib_sge sge;
2114         u8 nw_type;
2115         u16 ether_type;
2116         union ib_gid dgid;
2117         bool is_eth = false;
2118         bool is_vlan = false;
2119         bool is_grh = false;
2120         bool is_udp = false;
2121         u8 ip_version = 0;
2122         u16 vlan_id = 0xFFFF;
2123         void *buf;
2124         int i, rc = 0;
2125
2126         memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
2127
2128         rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL);
2129         if (rc)
2130                 return rc;
2131
2132         /* Get network header type for this GID */
2133         nw_type = rdma_gid_attr_network_type(sgid_attr);
2134         switch (nw_type) {
2135         case RDMA_NETWORK_IPV4:
2136                 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
2137                 break;
2138         case RDMA_NETWORK_IPV6:
2139                 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
2140                 break;
2141         default:
2142                 nw_type = BNXT_RE_ROCE_V1_PACKET;
2143                 break;
2144         }
2145         memcpy(&dgid.raw, &qplib_ah->dgid, 16);
2146         is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2147         if (is_udp) {
2148                 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) {
2149                         ip_version = 4;
2150                         ether_type = ETH_P_IP;
2151                 } else {
2152                         ip_version = 6;
2153                         ether_type = ETH_P_IPV6;
2154                 }
2155                 is_grh = false;
2156         } else {
2157                 ether_type = ETH_P_IBOE;
2158                 is_grh = true;
2159         }
2160
2161         is_eth = true;
2162         is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false;
2163
2164         ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
2165                           ip_version, is_udp, 0, &qp->qp1_hdr);
2166
2167         /* ETH */
2168         ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
2169         ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
2170
2171         /* For vlan, check the sgid for vlan existence */
2172
2173         if (!is_vlan) {
2174                 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
2175         } else {
2176                 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
2177                 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
2178         }
2179
2180         if (is_grh || (ip_version == 6)) {
2181                 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw,
2182                        sizeof(sgid_attr->gid));
2183                 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
2184                        sizeof(sgid_attr->gid));
2185                 qp->qp1_hdr.grh.hop_limit     = qplib_ah->hop_limit;
2186         }
2187
2188         if (ip_version == 4) {
2189                 qp->qp1_hdr.ip4.tos = 0;
2190                 qp->qp1_hdr.ip4.id = 0;
2191                 qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
2192                 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
2193
2194                 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4);
2195                 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
2196                 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
2197         }
2198
2199         if (is_udp) {
2200                 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
2201                 qp->qp1_hdr.udp.sport = htons(0x8CD1);
2202                 qp->qp1_hdr.udp.csum = 0;
2203         }
2204
2205         /* BTH */
2206         if (wr->opcode == IB_WR_SEND_WITH_IMM) {
2207                 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2208                 qp->qp1_hdr.immediate_present = 1;
2209         } else {
2210                 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2211         }
2212         if (wr->send_flags & IB_SEND_SOLICITED)
2213                 qp->qp1_hdr.bth.solicited_event = 1;
2214         /* pad_count */
2215         qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
2216
2217         /* P_key for QP1 is for all members */
2218         qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
2219         qp->qp1_hdr.bth.destination_qpn = IB_QP1;
2220         qp->qp1_hdr.bth.ack_req = 0;
2221         qp->send_psn++;
2222         qp->send_psn &= BTH_PSN_MASK;
2223         qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
2224         /* DETH */
2225         /* Use the priviledged Q_Key for QP1 */
2226         qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
2227         qp->qp1_hdr.deth.source_qpn = IB_QP1;
2228
2229         /* Pack the QP1 to the transmit buffer */
2230         buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
2231         if (buf) {
2232                 ib_ud_header_pack(&qp->qp1_hdr, buf);
2233                 for (i = wqe->num_sge; i; i--) {
2234                         wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
2235                         wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
2236                         wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
2237                 }
2238
2239                 /*
2240                  * Max Header buf size for IPV6 RoCE V2 is 86,
2241                  * which is same as the QP1 SQ header buffer.
2242                  * Header buf size for IPV4 RoCE V2 can be 66.
2243                  * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
2244                  * Subtract 20 bytes from QP1 SQ header buf size
2245                  */
2246                 if (is_udp && ip_version == 4)
2247                         sge.size -= 20;
2248                 /*
2249                  * Max Header buf size for RoCE V1 is 78.
2250                  * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
2251                  * Subtract 8 bytes from QP1 SQ header buf size
2252                  */
2253                 if (!is_udp)
2254                         sge.size -= 8;
2255
2256                 /* Subtract 4 bytes for non vlan packets */
2257                 if (!is_vlan)
2258                         sge.size -= 4;
2259
2260                 wqe->sg_list[0].addr = sge.addr;
2261                 wqe->sg_list[0].lkey = sge.lkey;
2262                 wqe->sg_list[0].size = sge.size;
2263                 wqe->num_sge++;
2264
2265         } else {
2266                 ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!");
2267                 rc = -ENOMEM;
2268         }
2269         return rc;
2270 }
2271
2272 /* For the MAD layer, it only provides the recv SGE the size of
2273  * ib_grh + MAD datagram.  No Ethernet headers, Ethertype, BTH, DETH,
2274  * nor RoCE iCRC.  The Cu+ solution must provide buffer for the entire
2275  * receive packet (334 bytes) with no VLAN and then copy the GRH
2276  * and the MAD datagram out to the provided SGE.
2277  */
2278 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
2279                                             const struct ib_recv_wr *wr,
2280                                             struct bnxt_qplib_swqe *wqe,
2281                                             int payload_size)
2282 {
2283         struct bnxt_re_sqp_entries *sqp_entry;
2284         struct bnxt_qplib_sge ref, sge;
2285         struct bnxt_re_dev *rdev;
2286         u32 rq_prod_index;
2287
2288         rdev = qp->rdev;
2289
2290         rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
2291
2292         if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
2293                 return -ENOMEM;
2294
2295         /* Create 1 SGE to receive the entire
2296          * ethernet packet
2297          */
2298         /* Save the reference from ULP */
2299         ref.addr = wqe->sg_list[0].addr;
2300         ref.lkey = wqe->sg_list[0].lkey;
2301         ref.size = wqe->sg_list[0].size;
2302
2303         sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
2304
2305         /* SGE 1 */
2306         wqe->sg_list[0].addr = sge.addr;
2307         wqe->sg_list[0].lkey = sge.lkey;
2308         wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
2309         sge.size -= wqe->sg_list[0].size;
2310
2311         sqp_entry->sge.addr = ref.addr;
2312         sqp_entry->sge.lkey = ref.lkey;
2313         sqp_entry->sge.size = ref.size;
2314         /* Store the wrid for reporting completion */
2315         sqp_entry->wrid = wqe->wr_id;
2316         /* change the wqe->wrid to table index */
2317         wqe->wr_id = rq_prod_index;
2318         return 0;
2319 }
2320
2321 static int is_ud_qp(struct bnxt_re_qp *qp)
2322 {
2323         return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
2324                 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
2325 }
2326
2327 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
2328                                   const struct ib_send_wr *wr,
2329                                   struct bnxt_qplib_swqe *wqe)
2330 {
2331         struct bnxt_re_ah *ah = NULL;
2332
2333         if (is_ud_qp(qp)) {
2334                 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
2335                 wqe->send.q_key = ud_wr(wr)->remote_qkey;
2336                 wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
2337                 wqe->send.avid = ah->qplib_ah.id;
2338         }
2339         switch (wr->opcode) {
2340         case IB_WR_SEND:
2341                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
2342                 break;
2343         case IB_WR_SEND_WITH_IMM:
2344                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
2345                 wqe->send.imm_data = wr->ex.imm_data;
2346                 break;
2347         case IB_WR_SEND_WITH_INV:
2348                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
2349                 wqe->send.inv_key = wr->ex.invalidate_rkey;
2350                 break;
2351         default:
2352                 return -EINVAL;
2353         }
2354         if (wr->send_flags & IB_SEND_SIGNALED)
2355                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2356         if (wr->send_flags & IB_SEND_FENCE)
2357                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2358         if (wr->send_flags & IB_SEND_SOLICITED)
2359                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2360         if (wr->send_flags & IB_SEND_INLINE)
2361                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2362
2363         return 0;
2364 }
2365
2366 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
2367                                   struct bnxt_qplib_swqe *wqe)
2368 {
2369         switch (wr->opcode) {
2370         case IB_WR_RDMA_WRITE:
2371                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
2372                 break;
2373         case IB_WR_RDMA_WRITE_WITH_IMM:
2374                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
2375                 wqe->rdma.imm_data = wr->ex.imm_data;
2376                 break;
2377         case IB_WR_RDMA_READ:
2378                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
2379                 wqe->rdma.inv_key = wr->ex.invalidate_rkey;
2380                 break;
2381         default:
2382                 return -EINVAL;
2383         }
2384         wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
2385         wqe->rdma.r_key = rdma_wr(wr)->rkey;
2386         if (wr->send_flags & IB_SEND_SIGNALED)
2387                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2388         if (wr->send_flags & IB_SEND_FENCE)
2389                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2390         if (wr->send_flags & IB_SEND_SOLICITED)
2391                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2392         if (wr->send_flags & IB_SEND_INLINE)
2393                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2394
2395         return 0;
2396 }
2397
2398 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
2399                                     struct bnxt_qplib_swqe *wqe)
2400 {
2401         switch (wr->opcode) {
2402         case IB_WR_ATOMIC_CMP_AND_SWP:
2403                 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
2404                 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2405                 wqe->atomic.swap_data = atomic_wr(wr)->swap;
2406                 break;
2407         case IB_WR_ATOMIC_FETCH_AND_ADD:
2408                 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
2409                 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2410                 break;
2411         default:
2412                 return -EINVAL;
2413         }
2414         wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
2415         wqe->atomic.r_key = atomic_wr(wr)->rkey;
2416         if (wr->send_flags & IB_SEND_SIGNALED)
2417                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2418         if (wr->send_flags & IB_SEND_FENCE)
2419                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2420         if (wr->send_flags & IB_SEND_SOLICITED)
2421                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2422         return 0;
2423 }
2424
2425 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
2426                                  struct bnxt_qplib_swqe *wqe)
2427 {
2428         wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
2429         wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
2430
2431         /* Need unconditional fence for local invalidate
2432          * opcode to work as expected.
2433          */
2434         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2435
2436         if (wr->send_flags & IB_SEND_SIGNALED)
2437                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2438         if (wr->send_flags & IB_SEND_SOLICITED)
2439                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2440
2441         return 0;
2442 }
2443
2444 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
2445                                  struct bnxt_qplib_swqe *wqe)
2446 {
2447         struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
2448         struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
2449         int access = wr->access;
2450
2451         wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
2452         wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
2453         wqe->frmr.page_list = mr->pages;
2454         wqe->frmr.page_list_len = mr->npages;
2455         wqe->frmr.levels = qplib_frpl->hwq.level;
2456         wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
2457
2458         /* Need unconditional fence for reg_mr
2459          * opcode to function as expected.
2460          */
2461
2462         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2463
2464         if (wr->wr.send_flags & IB_SEND_SIGNALED)
2465                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2466
2467         if (access & IB_ACCESS_LOCAL_WRITE)
2468                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2469         if (access & IB_ACCESS_REMOTE_READ)
2470                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
2471         if (access & IB_ACCESS_REMOTE_WRITE)
2472                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
2473         if (access & IB_ACCESS_REMOTE_ATOMIC)
2474                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
2475         if (access & IB_ACCESS_MW_BIND)
2476                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
2477
2478         wqe->frmr.l_key = wr->key;
2479         wqe->frmr.length = wr->mr->length;
2480         wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1;
2481         wqe->frmr.va = wr->mr->iova;
2482         return 0;
2483 }
2484
2485 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
2486                                     const struct ib_send_wr *wr,
2487                                     struct bnxt_qplib_swqe *wqe)
2488 {
2489         /*  Copy the inline data to the data  field */
2490         u8 *in_data;
2491         u32 i, sge_len;
2492         void *sge_addr;
2493
2494         in_data = wqe->inline_data;
2495         for (i = 0; i < wr->num_sge; i++) {
2496                 sge_addr = (void *)(unsigned long)
2497                                 wr->sg_list[i].addr;
2498                 sge_len = wr->sg_list[i].length;
2499
2500                 if ((sge_len + wqe->inline_len) >
2501                     BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
2502                         ibdev_err(&rdev->ibdev,
2503                                   "Inline data size requested > supported value");
2504                         return -EINVAL;
2505                 }
2506                 sge_len = wr->sg_list[i].length;
2507
2508                 memcpy(in_data, sge_addr, sge_len);
2509                 in_data += wr->sg_list[i].length;
2510                 wqe->inline_len += wr->sg_list[i].length;
2511         }
2512         return wqe->inline_len;
2513 }
2514
2515 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
2516                                    const struct ib_send_wr *wr,
2517                                    struct bnxt_qplib_swqe *wqe)
2518 {
2519         int payload_sz = 0;
2520
2521         if (wr->send_flags & IB_SEND_INLINE)
2522                 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
2523         else
2524                 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
2525                                                wqe->num_sge);
2526
2527         return payload_sz;
2528 }
2529
2530 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
2531 {
2532         if ((qp->ib_qp.qp_type == IB_QPT_UD ||
2533              qp->ib_qp.qp_type == IB_QPT_GSI ||
2534              qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
2535              qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
2536                 int qp_attr_mask;
2537                 struct ib_qp_attr qp_attr;
2538
2539                 qp_attr_mask = IB_QP_STATE;
2540                 qp_attr.qp_state = IB_QPS_RTS;
2541                 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
2542                 qp->qplib_qp.wqe_cnt = 0;
2543         }
2544 }
2545
2546 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
2547                                        struct bnxt_re_qp *qp,
2548                                        const struct ib_send_wr *wr)
2549 {
2550         int rc = 0, payload_sz = 0;
2551         unsigned long flags;
2552
2553         spin_lock_irqsave(&qp->sq_lock, flags);
2554         while (wr) {
2555                 struct bnxt_qplib_swqe wqe = {};
2556
2557                 /* Common */
2558                 wqe.num_sge = wr->num_sge;
2559                 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2560                         ibdev_err(&rdev->ibdev,
2561                                   "Limit exceeded for Send SGEs");
2562                         rc = -EINVAL;
2563                         goto bad;
2564                 }
2565
2566                 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2567                 if (payload_sz < 0) {
2568                         rc = -EINVAL;
2569                         goto bad;
2570                 }
2571                 wqe.wr_id = wr->wr_id;
2572
2573                 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
2574
2575                 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2576                 if (!rc)
2577                         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2578 bad:
2579                 if (rc) {
2580                         ibdev_err(&rdev->ibdev,
2581                                   "Post send failed opcode = %#x rc = %d",
2582                                   wr->opcode, rc);
2583                         break;
2584                 }
2585                 wr = wr->next;
2586         }
2587         bnxt_qplib_post_send_db(&qp->qplib_qp);
2588         bnxt_ud_qp_hw_stall_workaround(qp);
2589         spin_unlock_irqrestore(&qp->sq_lock, flags);
2590         return rc;
2591 }
2592
2593 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
2594                       const struct ib_send_wr **bad_wr)
2595 {
2596         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2597         struct bnxt_qplib_swqe wqe;
2598         int rc = 0, payload_sz = 0;
2599         unsigned long flags;
2600
2601         spin_lock_irqsave(&qp->sq_lock, flags);
2602         while (wr) {
2603                 /* House keeping */
2604                 memset(&wqe, 0, sizeof(wqe));
2605
2606                 /* Common */
2607                 wqe.num_sge = wr->num_sge;
2608                 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2609                         ibdev_err(&qp->rdev->ibdev,
2610                                   "Limit exceeded for Send SGEs");
2611                         rc = -EINVAL;
2612                         goto bad;
2613                 }
2614
2615                 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2616                 if (payload_sz < 0) {
2617                         rc = -EINVAL;
2618                         goto bad;
2619                 }
2620                 wqe.wr_id = wr->wr_id;
2621
2622                 switch (wr->opcode) {
2623                 case IB_WR_SEND:
2624                 case IB_WR_SEND_WITH_IMM:
2625                         if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) {
2626                                 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
2627                                                                payload_sz);
2628                                 if (rc)
2629                                         goto bad;
2630                                 wqe.rawqp1.lflags |=
2631                                         SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
2632                         }
2633                         switch (wr->send_flags) {
2634                         case IB_SEND_IP_CSUM:
2635                                 wqe.rawqp1.lflags |=
2636                                         SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
2637                                 break;
2638                         default:
2639                                 break;
2640                         }
2641                         fallthrough;
2642                 case IB_WR_SEND_WITH_INV:
2643                         rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2644                         break;
2645                 case IB_WR_RDMA_WRITE:
2646                 case IB_WR_RDMA_WRITE_WITH_IMM:
2647                 case IB_WR_RDMA_READ:
2648                         rc = bnxt_re_build_rdma_wqe(wr, &wqe);
2649                         break;
2650                 case IB_WR_ATOMIC_CMP_AND_SWP:
2651                 case IB_WR_ATOMIC_FETCH_AND_ADD:
2652                         rc = bnxt_re_build_atomic_wqe(wr, &wqe);
2653                         break;
2654                 case IB_WR_RDMA_READ_WITH_INV:
2655                         ibdev_err(&qp->rdev->ibdev,
2656                                   "RDMA Read with Invalidate is not supported");
2657                         rc = -EINVAL;
2658                         goto bad;
2659                 case IB_WR_LOCAL_INV:
2660                         rc = bnxt_re_build_inv_wqe(wr, &wqe);
2661                         break;
2662                 case IB_WR_REG_MR:
2663                         rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2664                         break;
2665                 default:
2666                         /* Unsupported WRs */
2667                         ibdev_err(&qp->rdev->ibdev,
2668                                   "WR (%#x) is not supported", wr->opcode);
2669                         rc = -EINVAL;
2670                         goto bad;
2671                 }
2672                 if (!rc)
2673                         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2674 bad:
2675                 if (rc) {
2676                         ibdev_err(&qp->rdev->ibdev,
2677                                   "post_send failed op:%#x qps = %#x rc = %d\n",
2678                                   wr->opcode, qp->qplib_qp.state, rc);
2679                         *bad_wr = wr;
2680                         break;
2681                 }
2682                 wr = wr->next;
2683         }
2684         bnxt_qplib_post_send_db(&qp->qplib_qp);
2685         bnxt_ud_qp_hw_stall_workaround(qp);
2686         spin_unlock_irqrestore(&qp->sq_lock, flags);
2687
2688         return rc;
2689 }
2690
2691 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2692                                        struct bnxt_re_qp *qp,
2693                                        const struct ib_recv_wr *wr)
2694 {
2695         struct bnxt_qplib_swqe wqe;
2696         int rc = 0;
2697
2698         memset(&wqe, 0, sizeof(wqe));
2699         while (wr) {
2700                 /* House keeping */
2701                 memset(&wqe, 0, sizeof(wqe));
2702
2703                 /* Common */
2704                 wqe.num_sge = wr->num_sge;
2705                 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2706                         ibdev_err(&rdev->ibdev,
2707                                   "Limit exceeded for Receive SGEs");
2708                         rc = -EINVAL;
2709                         break;
2710                 }
2711                 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
2712                 wqe.wr_id = wr->wr_id;
2713                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2714
2715                 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2716                 if (rc)
2717                         break;
2718
2719                 wr = wr->next;
2720         }
2721         if (!rc)
2722                 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2723         return rc;
2724 }
2725
2726 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
2727                       const struct ib_recv_wr **bad_wr)
2728 {
2729         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2730         struct bnxt_qplib_swqe wqe;
2731         int rc = 0, payload_sz = 0;
2732         unsigned long flags;
2733         u32 count = 0;
2734
2735         spin_lock_irqsave(&qp->rq_lock, flags);
2736         while (wr) {
2737                 /* House keeping */
2738                 memset(&wqe, 0, sizeof(wqe));
2739
2740                 /* Common */
2741                 wqe.num_sge = wr->num_sge;
2742                 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2743                         ibdev_err(&qp->rdev->ibdev,
2744                                   "Limit exceeded for Receive SGEs");
2745                         rc = -EINVAL;
2746                         *bad_wr = wr;
2747                         break;
2748                 }
2749
2750                 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2751                                                wr->num_sge);
2752                 wqe.wr_id = wr->wr_id;
2753                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2754
2755                 if (ib_qp->qp_type == IB_QPT_GSI &&
2756                     qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI)
2757                         rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
2758                                                               payload_sz);
2759                 if (!rc)
2760                         rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2761                 if (rc) {
2762                         *bad_wr = wr;
2763                         break;
2764                 }
2765
2766                 /* Ring DB if the RQEs posted reaches a threshold value */
2767                 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
2768                         bnxt_qplib_post_recv_db(&qp->qplib_qp);
2769                         count = 0;
2770                 }
2771
2772                 wr = wr->next;
2773         }
2774
2775         if (count)
2776                 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2777
2778         spin_unlock_irqrestore(&qp->rq_lock, flags);
2779
2780         return rc;
2781 }
2782
2783 /* Completion Queues */
2784 int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
2785 {
2786         struct bnxt_re_cq *cq;
2787         struct bnxt_qplib_nq *nq;
2788         struct bnxt_re_dev *rdev;
2789
2790         cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2791         rdev = cq->rdev;
2792         nq = cq->qplib_cq.nq;
2793
2794         bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2795         ib_umem_release(cq->umem);
2796
2797         atomic_dec(&rdev->cq_count);
2798         nq->budget--;
2799         kfree(cq->cql);
2800         return 0;
2801 }
2802
2803 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
2804                       struct ib_udata *udata)
2805 {
2806         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
2807         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
2808         struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
2809         int rc, entries;
2810         int cqe = attr->cqe;
2811         struct bnxt_qplib_nq *nq = NULL;
2812         unsigned int nq_alloc_cnt;
2813
2814         if (attr->flags)
2815                 return -EOPNOTSUPP;
2816
2817         /* Validate CQ fields */
2818         if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
2819                 ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded");
2820                 return -EINVAL;
2821         }
2822
2823         cq->rdev = rdev;
2824         cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
2825
2826         entries = roundup_pow_of_two(cqe + 1);
2827         if (entries > dev_attr->max_cq_wqes + 1)
2828                 entries = dev_attr->max_cq_wqes + 1;
2829
2830         cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
2831         cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
2832         if (udata) {
2833                 struct bnxt_re_cq_req req;
2834                 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
2835                         udata, struct bnxt_re_ucontext, ib_uctx);
2836                 if (ib_copy_from_udata(&req, udata, sizeof(req))) {
2837                         rc = -EFAULT;
2838                         goto fail;
2839                 }
2840
2841                 cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
2842                                        entries * sizeof(struct cq_base),
2843                                        IB_ACCESS_LOCAL_WRITE);
2844                 if (IS_ERR(cq->umem)) {
2845                         rc = PTR_ERR(cq->umem);
2846                         goto fail;
2847                 }
2848                 cq->qplib_cq.sg_info.umem = cq->umem;
2849                 cq->qplib_cq.dpi = &uctx->dpi;
2850         } else {
2851                 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
2852                 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
2853                                   GFP_KERNEL);
2854                 if (!cq->cql) {
2855                         rc = -ENOMEM;
2856                         goto fail;
2857                 }
2858
2859                 cq->qplib_cq.dpi = &rdev->dpi_privileged;
2860         }
2861         /*
2862          * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
2863          * used for getting the NQ index.
2864          */
2865         nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt);
2866         nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)];
2867         cq->qplib_cq.max_wqe = entries;
2868         cq->qplib_cq.cnq_hw_ring_id = nq->ring_id;
2869         cq->qplib_cq.nq = nq;
2870
2871         rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
2872         if (rc) {
2873                 ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
2874                 goto fail;
2875         }
2876
2877         cq->ib_cq.cqe = entries;
2878         cq->cq_period = cq->qplib_cq.period;
2879         nq->budget++;
2880
2881         atomic_inc(&rdev->cq_count);
2882         spin_lock_init(&cq->cq_lock);
2883
2884         if (udata) {
2885                 struct bnxt_re_cq_resp resp;
2886
2887                 resp.cqid = cq->qplib_cq.id;
2888                 resp.tail = cq->qplib_cq.hwq.cons;
2889                 resp.phase = cq->qplib_cq.period;
2890                 resp.rsvd = 0;
2891                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
2892                 if (rc) {
2893                         ibdev_err(&rdev->ibdev, "Failed to copy CQ udata");
2894                         bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2895                         goto c2fail;
2896                 }
2897         }
2898
2899         return 0;
2900
2901 c2fail:
2902         ib_umem_release(cq->umem);
2903 fail:
2904         kfree(cq->cql);
2905         return rc;
2906 }
2907
2908 static u8 __req_to_ib_wc_status(u8 qstatus)
2909 {
2910         switch (qstatus) {
2911         case CQ_REQ_STATUS_OK:
2912                 return IB_WC_SUCCESS;
2913         case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
2914                 return IB_WC_BAD_RESP_ERR;
2915         case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
2916                 return IB_WC_LOC_LEN_ERR;
2917         case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
2918                 return IB_WC_LOC_QP_OP_ERR;
2919         case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
2920                 return IB_WC_LOC_PROT_ERR;
2921         case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
2922                 return IB_WC_GENERAL_ERR;
2923         case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
2924                 return IB_WC_REM_INV_REQ_ERR;
2925         case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
2926                 return IB_WC_REM_ACCESS_ERR;
2927         case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
2928                 return IB_WC_REM_OP_ERR;
2929         case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
2930                 return IB_WC_RNR_RETRY_EXC_ERR;
2931         case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
2932                 return IB_WC_RETRY_EXC_ERR;
2933         case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
2934                 return IB_WC_WR_FLUSH_ERR;
2935         default:
2936                 return IB_WC_GENERAL_ERR;
2937         }
2938         return 0;
2939 }
2940
2941 static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
2942 {
2943         switch (qstatus) {
2944         case CQ_RES_RAWETH_QP1_STATUS_OK:
2945                 return IB_WC_SUCCESS;
2946         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
2947                 return IB_WC_LOC_ACCESS_ERR;
2948         case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
2949                 return IB_WC_LOC_LEN_ERR;
2950         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
2951                 return IB_WC_LOC_PROT_ERR;
2952         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
2953                 return IB_WC_LOC_QP_OP_ERR;
2954         case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
2955                 return IB_WC_GENERAL_ERR;
2956         case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
2957                 return IB_WC_WR_FLUSH_ERR;
2958         case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
2959                 return IB_WC_WR_FLUSH_ERR;
2960         default:
2961                 return IB_WC_GENERAL_ERR;
2962         }
2963 }
2964
2965 static u8 __rc_to_ib_wc_status(u8 qstatus)
2966 {
2967         switch (qstatus) {
2968         case CQ_RES_RC_STATUS_OK:
2969                 return IB_WC_SUCCESS;
2970         case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
2971                 return IB_WC_LOC_ACCESS_ERR;
2972         case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
2973                 return IB_WC_LOC_LEN_ERR;
2974         case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
2975                 return IB_WC_LOC_PROT_ERR;
2976         case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
2977                 return IB_WC_LOC_QP_OP_ERR;
2978         case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
2979                 return IB_WC_GENERAL_ERR;
2980         case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
2981                 return IB_WC_REM_INV_REQ_ERR;
2982         case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
2983                 return IB_WC_WR_FLUSH_ERR;
2984         case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
2985                 return IB_WC_WR_FLUSH_ERR;
2986         default:
2987                 return IB_WC_GENERAL_ERR;
2988         }
2989 }
2990
2991 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
2992 {
2993         switch (cqe->type) {
2994         case BNXT_QPLIB_SWQE_TYPE_SEND:
2995                 wc->opcode = IB_WC_SEND;
2996                 break;
2997         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
2998                 wc->opcode = IB_WC_SEND;
2999                 wc->wc_flags |= IB_WC_WITH_IMM;
3000                 break;
3001         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
3002                 wc->opcode = IB_WC_SEND;
3003                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3004                 break;
3005         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
3006                 wc->opcode = IB_WC_RDMA_WRITE;
3007                 break;
3008         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
3009                 wc->opcode = IB_WC_RDMA_WRITE;
3010                 wc->wc_flags |= IB_WC_WITH_IMM;
3011                 break;
3012         case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
3013                 wc->opcode = IB_WC_RDMA_READ;
3014                 break;
3015         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
3016                 wc->opcode = IB_WC_COMP_SWAP;
3017                 break;
3018         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
3019                 wc->opcode = IB_WC_FETCH_ADD;
3020                 break;
3021         case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
3022                 wc->opcode = IB_WC_LOCAL_INV;
3023                 break;
3024         case BNXT_QPLIB_SWQE_TYPE_REG_MR:
3025                 wc->opcode = IB_WC_REG_MR;
3026                 break;
3027         default:
3028                 wc->opcode = IB_WC_SEND;
3029                 break;
3030         }
3031
3032         wc->status = __req_to_ib_wc_status(cqe->status);
3033 }
3034
3035 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
3036                                      u16 raweth_qp1_flags2)
3037 {
3038         bool is_ipv6 = false, is_ipv4 = false;
3039
3040         /* raweth_qp1_flags Bit 9-6 indicates itype */
3041         if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3042             != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3043                 return -1;
3044
3045         if (raweth_qp1_flags2 &
3046             CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
3047             raweth_qp1_flags2 &
3048             CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
3049                 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
3050                 (raweth_qp1_flags2 &
3051                  CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
3052                         (is_ipv6 = true) : (is_ipv4 = true);
3053                 return ((is_ipv6) ?
3054                          BNXT_RE_ROCEV2_IPV6_PACKET :
3055                          BNXT_RE_ROCEV2_IPV4_PACKET);
3056         } else {
3057                 return BNXT_RE_ROCE_V1_PACKET;
3058         }
3059 }
3060
3061 static int bnxt_re_to_ib_nw_type(int nw_type)
3062 {
3063         u8 nw_hdr_type = 0xFF;
3064
3065         switch (nw_type) {
3066         case BNXT_RE_ROCE_V1_PACKET:
3067                 nw_hdr_type = RDMA_NETWORK_ROCE_V1;
3068                 break;
3069         case BNXT_RE_ROCEV2_IPV4_PACKET:
3070                 nw_hdr_type = RDMA_NETWORK_IPV4;
3071                 break;
3072         case BNXT_RE_ROCEV2_IPV6_PACKET:
3073                 nw_hdr_type = RDMA_NETWORK_IPV6;
3074                 break;
3075         }
3076         return nw_hdr_type;
3077 }
3078
3079 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
3080                                        void *rq_hdr_buf)
3081 {
3082         u8 *tmp_buf = NULL;
3083         struct ethhdr *eth_hdr;
3084         u16 eth_type;
3085         bool rc = false;
3086
3087         tmp_buf = (u8 *)rq_hdr_buf;
3088         /*
3089          * If dest mac is not same as I/F mac, this could be a
3090          * loopback address or multicast address, check whether
3091          * it is a loopback packet
3092          */
3093         if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
3094                 tmp_buf += 4;
3095                 /* Check the  ether type */
3096                 eth_hdr = (struct ethhdr *)tmp_buf;
3097                 eth_type = ntohs(eth_hdr->h_proto);
3098                 switch (eth_type) {
3099                 case ETH_P_IBOE:
3100                         rc = true;
3101                         break;
3102                 case ETH_P_IP:
3103                 case ETH_P_IPV6: {
3104                         u32 len;
3105                         struct udphdr *udp_hdr;
3106
3107                         len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
3108                                                       sizeof(struct ipv6hdr));
3109                         tmp_buf += sizeof(struct ethhdr) + len;
3110                         udp_hdr = (struct udphdr *)tmp_buf;
3111                         if (ntohs(udp_hdr->dest) ==
3112                                     ROCE_V2_UDP_DPORT)
3113                                 rc = true;
3114                         break;
3115                         }
3116                 default:
3117                         break;
3118                 }
3119         }
3120
3121         return rc;
3122 }
3123
3124 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp,
3125                                          struct bnxt_qplib_cqe *cqe)
3126 {
3127         struct bnxt_re_dev *rdev = gsi_qp->rdev;
3128         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3129         struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp;
3130         struct bnxt_re_ah *gsi_sah;
3131         struct ib_send_wr *swr;
3132         struct ib_ud_wr udwr;
3133         struct ib_recv_wr rwr;
3134         int pkt_type = 0;
3135         u32 tbl_idx;
3136         void *rq_hdr_buf;
3137         dma_addr_t rq_hdr_buf_map;
3138         dma_addr_t shrq_hdr_buf_map;
3139         u32 offset = 0;
3140         u32 skip_bytes = 0;
3141         struct ib_sge s_sge[2];
3142         struct ib_sge r_sge[2];
3143         int rc;
3144
3145         memset(&udwr, 0, sizeof(udwr));
3146         memset(&rwr, 0, sizeof(rwr));
3147         memset(&s_sge, 0, sizeof(s_sge));
3148         memset(&r_sge, 0, sizeof(r_sge));
3149
3150         swr = &udwr.wr;
3151         tbl_idx = cqe->wr_id;
3152
3153         rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf +
3154                         (tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size);
3155         rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3156                                                           tbl_idx);
3157
3158         /* Shadow QP header buffer */
3159         shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3160                                                             tbl_idx);
3161         sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3162
3163         /* Store this cqe */
3164         memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
3165         sqp_entry->qp1_qp = gsi_qp;
3166
3167         /* Find packet type from the cqe */
3168
3169         pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
3170                                              cqe->raweth_qp1_flags2);
3171         if (pkt_type < 0) {
3172                 ibdev_err(&rdev->ibdev, "Invalid packet\n");
3173                 return -EINVAL;
3174         }
3175
3176         /* Adjust the offset for the user buffer and post in the rq */
3177
3178         if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
3179                 offset = 20;
3180
3181         /*
3182          * QP1 loopback packet has 4 bytes of internal header before
3183          * ether header. Skip these four bytes.
3184          */
3185         if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
3186                 skip_bytes = 4;
3187
3188         /* First send SGE . Skip the ether header*/
3189         s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
3190                         + skip_bytes;
3191         s_sge[0].lkey = 0xFFFFFFFF;
3192         s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
3193                                 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
3194
3195         /* Second Send SGE */
3196         s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
3197                         BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
3198         if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
3199                 s_sge[1].addr += 8;
3200         s_sge[1].lkey = 0xFFFFFFFF;
3201         s_sge[1].length = 256;
3202
3203         /* First recv SGE */
3204
3205         r_sge[0].addr = shrq_hdr_buf_map;
3206         r_sge[0].lkey = 0xFFFFFFFF;
3207         r_sge[0].length = 40;
3208
3209         r_sge[1].addr = sqp_entry->sge.addr + offset;
3210         r_sge[1].lkey = sqp_entry->sge.lkey;
3211         r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
3212
3213         /* Create receive work request */
3214         rwr.num_sge = 2;
3215         rwr.sg_list = r_sge;
3216         rwr.wr_id = tbl_idx;
3217         rwr.next = NULL;
3218
3219         rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr);
3220         if (rc) {
3221                 ibdev_err(&rdev->ibdev,
3222                           "Failed to post Rx buffers to shadow QP");
3223                 return -ENOMEM;
3224         }
3225
3226         swr->num_sge = 2;
3227         swr->sg_list = s_sge;
3228         swr->wr_id = tbl_idx;
3229         swr->opcode = IB_WR_SEND;
3230         swr->next = NULL;
3231         gsi_sah = rdev->gsi_ctx.gsi_sah;
3232         udwr.ah = &gsi_sah->ib_ah;
3233         udwr.remote_qpn = gsi_sqp->qplib_qp.id;
3234         udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
3235
3236         /* post data received  in the send queue */
3237         rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);
3238
3239         return 0;
3240 }
3241
3242 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
3243                                           struct bnxt_qplib_cqe *cqe)
3244 {
3245         wc->opcode = IB_WC_RECV;
3246         wc->status = __rawqp1_to_ib_wc_status(cqe->status);
3247         wc->wc_flags |= IB_WC_GRH;
3248 }
3249
3250 static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,
3251                                         u16 vlan_id)
3252 {
3253         /*
3254          * Check if the vlan is configured in the host.  If not configured, it
3255          * can be a transparent VLAN. So dont report the vlan id.
3256          */
3257         if (!__vlan_find_dev_deep_rcu(rdev->netdev,
3258                                       htons(ETH_P_8021Q), vlan_id))
3259                 return false;
3260         return true;
3261 }
3262
3263 static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe,
3264                                 u16 *vid, u8 *sl)
3265 {
3266         bool ret = false;
3267         u32 metadata;
3268         u16 tpid;
3269
3270         metadata = orig_cqe->raweth_qp1_metadata;
3271         if (orig_cqe->raweth_qp1_flags2 &
3272                 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
3273                 tpid = ((metadata &
3274                          CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
3275                          CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
3276                 if (tpid == ETH_P_8021Q) {
3277                         *vid = metadata &
3278                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
3279                         *sl = (metadata &
3280                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
3281                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
3282                         ret = true;
3283                 }
3284         }
3285
3286         return ret;
3287 }
3288
3289 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
3290                                       struct bnxt_qplib_cqe *cqe)
3291 {
3292         wc->opcode = IB_WC_RECV;
3293         wc->status = __rc_to_ib_wc_status(cqe->status);
3294
3295         if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3296                 wc->wc_flags |= IB_WC_WITH_IMM;
3297         if (cqe->flags & CQ_RES_RC_FLAGS_INV)
3298                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3299         if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
3300             (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
3301                 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3302 }
3303
3304 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
3305                                              struct ib_wc *wc,
3306                                              struct bnxt_qplib_cqe *cqe)
3307 {
3308         struct bnxt_re_dev *rdev = gsi_sqp->rdev;
3309         struct bnxt_re_qp *gsi_qp = NULL;
3310         struct bnxt_qplib_cqe *orig_cqe = NULL;
3311         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3312         int nw_type;
3313         u32 tbl_idx;
3314         u16 vlan_id;
3315         u8 sl;
3316
3317         tbl_idx = cqe->wr_id;
3318
3319         sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3320         gsi_qp = sqp_entry->qp1_qp;
3321         orig_cqe = &sqp_entry->cqe;
3322
3323         wc->wr_id = sqp_entry->wrid;
3324         wc->byte_len = orig_cqe->length;
3325         wc->qp = &gsi_qp->ib_qp;
3326
3327         wc->ex.imm_data = orig_cqe->immdata;
3328         wc->src_qp = orig_cqe->src_qp;
3329         memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
3330         if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) {
3331                 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3332                         wc->vlan_id = vlan_id;
3333                         wc->sl = sl;
3334                         wc->wc_flags |= IB_WC_WITH_VLAN;
3335                 }
3336         }
3337         wc->port_num = 1;
3338         wc->vendor_err = orig_cqe->status;
3339
3340         wc->opcode = IB_WC_RECV;
3341         wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
3342         wc->wc_flags |= IB_WC_GRH;
3343
3344         nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
3345                                             orig_cqe->raweth_qp1_flags2);
3346         if (nw_type >= 0) {
3347                 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3348                 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3349         }
3350 }
3351
3352 static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,
3353                                       struct ib_wc *wc,
3354                                       struct bnxt_qplib_cqe *cqe)
3355 {
3356         u8 nw_type;
3357
3358         wc->opcode = IB_WC_RECV;
3359         wc->status = __rc_to_ib_wc_status(cqe->status);
3360
3361         if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
3362                 wc->wc_flags |= IB_WC_WITH_IMM;
3363         /* report only on GSI QP for Thor */
3364         if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) {
3365                 wc->wc_flags |= IB_WC_GRH;
3366                 memcpy(wc->smac, cqe->smac, ETH_ALEN);
3367                 wc->wc_flags |= IB_WC_WITH_SMAC;
3368                 if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
3369                         wc->vlan_id = (cqe->cfa_meta & 0xFFF);
3370                         if (wc->vlan_id < 0x1000)
3371                                 wc->wc_flags |= IB_WC_WITH_VLAN;
3372                 }
3373                 nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >>
3374                            CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT;
3375                 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3376                 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3377         }
3378
3379 }
3380
3381 static int send_phantom_wqe(struct bnxt_re_qp *qp)
3382 {
3383         struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
3384         unsigned long flags;
3385         int rc = 0;
3386
3387         spin_lock_irqsave(&qp->sq_lock, flags);
3388
3389         rc = bnxt_re_bind_fence_mw(lib_qp);
3390         if (!rc) {
3391                 lib_qp->sq.phantom_wqe_cnt++;
3392                 ibdev_dbg(&qp->rdev->ibdev,
3393                           "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
3394                           lib_qp->id, lib_qp->sq.hwq.prod,
3395                           HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
3396                           lib_qp->sq.phantom_wqe_cnt);
3397         }
3398
3399         spin_unlock_irqrestore(&qp->sq_lock, flags);
3400         return rc;
3401 }
3402
3403 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
3404 {
3405         struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3406         struct bnxt_re_qp *qp, *sh_qp;
3407         struct bnxt_qplib_cqe *cqe;
3408         int i, ncqe, budget;
3409         struct bnxt_qplib_q *sq;
3410         struct bnxt_qplib_qp *lib_qp;
3411         u32 tbl_idx;
3412         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3413         unsigned long flags;
3414
3415         spin_lock_irqsave(&cq->cq_lock, flags);
3416         budget = min_t(u32, num_entries, cq->max_cql);
3417         num_entries = budget;
3418         if (!cq->cql) {
3419                 ibdev_err(&cq->rdev->ibdev, "POLL CQ : no CQL to use");
3420                 goto exit;
3421         }
3422         cqe = &cq->cql[0];
3423         while (budget) {
3424                 lib_qp = NULL;
3425                 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
3426                 if (lib_qp) {
3427                         sq = &lib_qp->sq;
3428                         if (sq->send_phantom) {
3429                                 qp = container_of(lib_qp,
3430                                                   struct bnxt_re_qp, qplib_qp);
3431                                 if (send_phantom_wqe(qp) == -ENOMEM)
3432                                         ibdev_err(&cq->rdev->ibdev,
3433                                                   "Phantom failed! Scheduled to send again\n");
3434                                 else
3435                                         sq->send_phantom = false;
3436                         }
3437                 }
3438                 if (ncqe < budget)
3439                         ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
3440                                                               cqe + ncqe,
3441                                                               budget - ncqe);
3442
3443                 if (!ncqe)
3444                         break;
3445
3446                 for (i = 0; i < ncqe; i++, cqe++) {
3447                         /* Transcribe each qplib_wqe back to ib_wc */
3448                         memset(wc, 0, sizeof(*wc));
3449
3450                         wc->wr_id = cqe->wr_id;
3451                         wc->byte_len = cqe->length;
3452                         qp = container_of
3453                                 ((struct bnxt_qplib_qp *)
3454                                  (unsigned long)(cqe->qp_handle),
3455                                  struct bnxt_re_qp, qplib_qp);
3456                         wc->qp = &qp->ib_qp;
3457                         wc->ex.imm_data = cqe->immdata;
3458                         wc->src_qp = cqe->src_qp;
3459                         memcpy(wc->smac, cqe->smac, ETH_ALEN);
3460                         wc->port_num = 1;
3461                         wc->vendor_err = cqe->status;
3462
3463                         switch (cqe->opcode) {
3464                         case CQ_BASE_CQE_TYPE_REQ:
3465                                 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3466                                 if (sh_qp &&
3467                                     qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3468                                         /* Handle this completion with
3469                                          * the stored completion
3470                                          */
3471                                         memset(wc, 0, sizeof(*wc));
3472                                         continue;
3473                                 }
3474                                 bnxt_re_process_req_wc(wc, cqe);
3475                                 break;
3476                         case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
3477                                 if (!cqe->status) {
3478                                         int rc = 0;
3479
3480                                         rc = bnxt_re_process_raw_qp_pkt_rx
3481                                                                 (qp, cqe);
3482                                         if (!rc) {
3483                                                 memset(wc, 0, sizeof(*wc));
3484                                                 continue;
3485                                         }
3486                                         cqe->status = -1;
3487                                 }
3488                                 /* Errors need not be looped back.
3489                                  * But change the wr_id to the one
3490                                  * stored in the table
3491                                  */
3492                                 tbl_idx = cqe->wr_id;
3493                                 sqp_entry = &cq->rdev->gsi_ctx.sqp_tbl[tbl_idx];
3494                                 wc->wr_id = sqp_entry->wrid;
3495                                 bnxt_re_process_res_rawqp1_wc(wc, cqe);
3496                                 break;
3497                         case CQ_BASE_CQE_TYPE_RES_RC:
3498                                 bnxt_re_process_res_rc_wc(wc, cqe);
3499                                 break;
3500                         case CQ_BASE_CQE_TYPE_RES_UD:
3501                                 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3502                                 if (sh_qp &&
3503                                     qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3504                                         /* Handle this completion with
3505                                          * the stored completion
3506                                          */
3507                                         if (cqe->status) {
3508                                                 continue;
3509                                         } else {
3510                                                 bnxt_re_process_res_shadow_qp_wc
3511                                                                 (qp, wc, cqe);
3512                                                 break;
3513                                         }
3514                                 }
3515                                 bnxt_re_process_res_ud_wc(qp, wc, cqe);
3516                                 break;
3517                         default:
3518                                 ibdev_err(&cq->rdev->ibdev,
3519                                           "POLL CQ : type 0x%x not handled",
3520                                           cqe->opcode);
3521                                 continue;
3522                         }
3523                         wc++;
3524                         budget--;
3525                 }
3526         }
3527 exit:
3528         spin_unlock_irqrestore(&cq->cq_lock, flags);
3529         return num_entries - budget;
3530 }
3531
3532 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
3533                           enum ib_cq_notify_flags ib_cqn_flags)
3534 {
3535         struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3536         int type = 0, rc = 0;
3537         unsigned long flags;
3538
3539         spin_lock_irqsave(&cq->cq_lock, flags);
3540         /* Trigger on the very next completion */
3541         if (ib_cqn_flags & IB_CQ_NEXT_COMP)
3542                 type = DBC_DBC_TYPE_CQ_ARMALL;
3543         /* Trigger on the next solicited completion */
3544         else if (ib_cqn_flags & IB_CQ_SOLICITED)
3545                 type = DBC_DBC_TYPE_CQ_ARMSE;
3546
3547         /* Poll to see if there are missed events */
3548         if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
3549             !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) {
3550                 rc = 1;
3551                 goto exit;
3552         }
3553         bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
3554
3555 exit:
3556         spin_unlock_irqrestore(&cq->cq_lock, flags);
3557         return rc;
3558 }
3559
3560 /* Memory Regions */
3561 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
3562 {
3563         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3564         struct bnxt_re_dev *rdev = pd->rdev;
3565         struct bnxt_re_mr *mr;
3566         int rc;
3567
3568         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3569         if (!mr)
3570                 return ERR_PTR(-ENOMEM);
3571
3572         mr->rdev = rdev;
3573         mr->qplib_mr.pd = &pd->qplib_pd;
3574         mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
3575         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
3576
3577         /* Allocate and register 0 as the address */
3578         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3579         if (rc)
3580                 goto fail;
3581
3582         mr->qplib_mr.hwq.level = PBL_LVL_MAX;
3583         mr->qplib_mr.total_size = -1; /* Infinte length */
3584         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0,
3585                                PAGE_SIZE);
3586         if (rc)
3587                 goto fail_mr;
3588
3589         mr->ib_mr.lkey = mr->qplib_mr.lkey;
3590         if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
3591                                IB_ACCESS_REMOTE_ATOMIC))
3592                 mr->ib_mr.rkey = mr->ib_mr.lkey;
3593         atomic_inc(&rdev->mr_count);
3594
3595         return &mr->ib_mr;
3596
3597 fail_mr:
3598         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3599 fail:
3600         kfree(mr);
3601         return ERR_PTR(rc);
3602 }
3603
3604 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3605 {
3606         struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3607         struct bnxt_re_dev *rdev = mr->rdev;
3608         int rc;
3609
3610         rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3611         if (rc) {
3612                 ibdev_err(&rdev->ibdev, "Dereg MR failed: %#x\n", rc);
3613                 return rc;
3614         }
3615
3616         if (mr->pages) {
3617                 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
3618                                                         &mr->qplib_frpl);
3619                 kfree(mr->pages);
3620                 mr->npages = 0;
3621                 mr->pages = NULL;
3622         }
3623         ib_umem_release(mr->ib_umem);
3624
3625         kfree(mr);
3626         atomic_dec(&rdev->mr_count);
3627         return rc;
3628 }
3629
3630 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
3631 {
3632         struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3633
3634         if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
3635                 return -ENOMEM;
3636
3637         mr->pages[mr->npages++] = addr;
3638         return 0;
3639 }
3640
3641 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
3642                       unsigned int *sg_offset)
3643 {
3644         struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3645
3646         mr->npages = 0;
3647         return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page);
3648 }
3649
3650 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
3651                                u32 max_num_sg)
3652 {
3653         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3654         struct bnxt_re_dev *rdev = pd->rdev;
3655         struct bnxt_re_mr *mr = NULL;
3656         int rc;
3657
3658         if (type != IB_MR_TYPE_MEM_REG) {
3659                 ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported", type);
3660                 return ERR_PTR(-EINVAL);
3661         }
3662         if (max_num_sg > MAX_PBL_LVL_1_PGS)
3663                 return ERR_PTR(-EINVAL);
3664
3665         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3666         if (!mr)
3667                 return ERR_PTR(-ENOMEM);
3668
3669         mr->rdev = rdev;
3670         mr->qplib_mr.pd = &pd->qplib_pd;
3671         mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR;
3672         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
3673
3674         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3675         if (rc)
3676                 goto bail;
3677
3678         mr->ib_mr.lkey = mr->qplib_mr.lkey;
3679         mr->ib_mr.rkey = mr->ib_mr.lkey;
3680
3681         mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
3682         if (!mr->pages) {
3683                 rc = -ENOMEM;
3684                 goto fail;
3685         }
3686         rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
3687                                                  &mr->qplib_frpl, max_num_sg);
3688         if (rc) {
3689                 ibdev_err(&rdev->ibdev,
3690                           "Failed to allocate HW FR page list");
3691                 goto fail_mr;
3692         }
3693
3694         atomic_inc(&rdev->mr_count);
3695         return &mr->ib_mr;
3696
3697 fail_mr:
3698         kfree(mr->pages);
3699 fail:
3700         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3701 bail:
3702         kfree(mr);
3703         return ERR_PTR(rc);
3704 }
3705
3706 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
3707                                struct ib_udata *udata)
3708 {
3709         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3710         struct bnxt_re_dev *rdev = pd->rdev;
3711         struct bnxt_re_mw *mw;
3712         int rc;
3713
3714         mw = kzalloc(sizeof(*mw), GFP_KERNEL);
3715         if (!mw)
3716                 return ERR_PTR(-ENOMEM);
3717         mw->rdev = rdev;
3718         mw->qplib_mw.pd = &pd->qplib_pd;
3719
3720         mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
3721                                CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
3722                                CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
3723         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw);
3724         if (rc) {
3725                 ibdev_err(&rdev->ibdev, "Allocate MW failed!");
3726                 goto fail;
3727         }
3728         mw->ib_mw.rkey = mw->qplib_mw.rkey;
3729
3730         atomic_inc(&rdev->mw_count);
3731         return &mw->ib_mw;
3732
3733 fail:
3734         kfree(mw);
3735         return ERR_PTR(rc);
3736 }
3737
3738 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
3739 {
3740         struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw);
3741         struct bnxt_re_dev *rdev = mw->rdev;
3742         int rc;
3743
3744         rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw);
3745         if (rc) {
3746                 ibdev_err(&rdev->ibdev, "Free MW failed: %#x\n", rc);
3747                 return rc;
3748         }
3749
3750         kfree(mw);
3751         atomic_dec(&rdev->mw_count);
3752         return rc;
3753 }
3754
3755 /* uverbs */
3756 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
3757                                   u64 virt_addr, int mr_access_flags,
3758                                   struct ib_udata *udata)
3759 {
3760         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3761         struct bnxt_re_dev *rdev = pd->rdev;
3762         struct bnxt_re_mr *mr;
3763         struct ib_umem *umem;
3764         unsigned long page_size;
3765         int umem_pgs, rc;
3766
3767         if (length > BNXT_RE_MAX_MR_SIZE) {
3768                 ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n",
3769                           length, BNXT_RE_MAX_MR_SIZE);
3770                 return ERR_PTR(-ENOMEM);
3771         }
3772
3773         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3774         if (!mr)
3775                 return ERR_PTR(-ENOMEM);
3776
3777         mr->rdev = rdev;
3778         mr->qplib_mr.pd = &pd->qplib_pd;
3779         mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
3780         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
3781
3782         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3783         if (rc) {
3784                 ibdev_err(&rdev->ibdev, "Failed to allocate MR");
3785                 goto free_mr;
3786         }
3787         /* The fixed portion of the rkey is the same as the lkey */
3788         mr->ib_mr.rkey = mr->qplib_mr.rkey;
3789
3790         umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags);
3791         if (IS_ERR(umem)) {
3792                 ibdev_err(&rdev->ibdev, "Failed to get umem");
3793                 rc = -EFAULT;
3794                 goto free_mrw;
3795         }
3796         mr->ib_umem = umem;
3797
3798         mr->qplib_mr.va = virt_addr;
3799         page_size = ib_umem_find_best_pgsz(
3800                 umem, BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M, virt_addr);
3801         if (!page_size) {
3802                 ibdev_err(&rdev->ibdev, "umem page size unsupported!");
3803                 rc = -EFAULT;
3804                 goto free_umem;
3805         }
3806         mr->qplib_mr.total_size = length;
3807
3808         umem_pgs = ib_umem_num_dma_blocks(umem, page_size);
3809         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem,
3810                                umem_pgs, page_size);
3811         if (rc) {
3812                 ibdev_err(&rdev->ibdev, "Failed to register user MR");
3813                 goto free_umem;
3814         }
3815
3816         mr->ib_mr.lkey = mr->qplib_mr.lkey;
3817         mr->ib_mr.rkey = mr->qplib_mr.lkey;
3818         atomic_inc(&rdev->mr_count);
3819
3820         return &mr->ib_mr;
3821 free_umem:
3822         ib_umem_release(umem);
3823 free_mrw:
3824         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3825 free_mr:
3826         kfree(mr);
3827         return ERR_PTR(rc);
3828 }
3829
3830 int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
3831 {
3832         struct ib_device *ibdev = ctx->device;
3833         struct bnxt_re_ucontext *uctx =
3834                 container_of(ctx, struct bnxt_re_ucontext, ib_uctx);
3835         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
3836         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
3837         struct bnxt_re_uctx_resp resp = {};
3838         u32 chip_met_rev_num = 0;
3839         int rc;
3840
3841         ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver);
3842
3843         if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
3844                 ibdev_dbg(ibdev, " is different from the device %d ",
3845                           BNXT_RE_ABI_VERSION);
3846                 return -EPERM;
3847         }
3848
3849         uctx->rdev = rdev;
3850
3851         uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
3852         if (!uctx->shpg) {
3853                 rc = -ENOMEM;
3854                 goto fail;
3855         }
3856         spin_lock_init(&uctx->sh_lock);
3857
3858         resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX;
3859         chip_met_rev_num = rdev->chip_ctx->chip_num;
3860         chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) <<
3861                              BNXT_RE_CHIP_ID0_CHIP_REV_SFT;
3862         chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) <<
3863                              BNXT_RE_CHIP_ID0_CHIP_MET_SFT;
3864         resp.chip_id0 = chip_met_rev_num;
3865         /*Temp, Use xa_alloc instead */
3866         resp.dev_id = rdev->en_dev->pdev->devfn;
3867         resp.max_qp = rdev->qplib_ctx.qpc_count;
3868         resp.pg_size = PAGE_SIZE;
3869         resp.cqe_sz = sizeof(struct cq_base);
3870         resp.max_cqd = dev_attr->max_cq_wqes;
3871
3872         resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE;
3873         resp.mode = rdev->chip_ctx->modes.wqe_mode;
3874
3875         rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
3876         if (rc) {
3877                 ibdev_err(ibdev, "Failed to copy user context");
3878                 rc = -EFAULT;
3879                 goto cfail;
3880         }
3881
3882         return 0;
3883 cfail:
3884         free_page((unsigned long)uctx->shpg);
3885         uctx->shpg = NULL;
3886 fail:
3887         return rc;
3888 }
3889
3890 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
3891 {
3892         struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3893                                                    struct bnxt_re_ucontext,
3894                                                    ib_uctx);
3895
3896         struct bnxt_re_dev *rdev = uctx->rdev;
3897
3898         if (uctx->shpg)
3899                 free_page((unsigned long)uctx->shpg);
3900
3901         if (uctx->dpi.dbr) {
3902                 /* Free DPI only if this is the first PD allocated by the
3903                  * application and mark the context dpi as NULL
3904                  */
3905                 bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
3906                                        &rdev->qplib_res.dpi_tbl, &uctx->dpi);
3907                 uctx->dpi.dbr = NULL;
3908         }
3909 }
3910
3911 /* Helper function to mmap the virtual memory from user app */
3912 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
3913 {
3914         struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3915                                                    struct bnxt_re_ucontext,
3916                                                    ib_uctx);
3917         struct bnxt_re_dev *rdev = uctx->rdev;
3918         u64 pfn;
3919
3920         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3921                 return -EINVAL;
3922
3923         if (vma->vm_pgoff) {
3924                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3925                 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
3926                                        PAGE_SIZE, vma->vm_page_prot)) {
3927                         ibdev_err(&rdev->ibdev, "Failed to map DPI");
3928                         return -EAGAIN;
3929                 }
3930         } else {
3931                 pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT;
3932                 if (remap_pfn_range(vma, vma->vm_start,
3933                                     pfn, PAGE_SIZE, vma->vm_page_prot)) {
3934                         ibdev_err(&rdev->ibdev, "Failed to map shared page");
3935                         return -EAGAIN;
3936                 }
3937         }
3938
3939         return 0;
3940 }