drivers/infiniband/hw/bnxt_re/ib_verbs.c

   1 /*
   2  * Broadcom NetXtreme-E RoCE driver.
   3  *
   4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * BSD license below:
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in
  21  *    the documentation and/or other materials provided with the
  22  *    distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Description: IB Verbs interpreter
  37  */
  38
  39 #include <linux/interrupt.h>
  40 #include <linux/types.h>
  41 #include <linux/pci.h>
  42 #include <linux/netdevice.h>
  43 #include <linux/if_ether.h>
  44
  45 #include <rdma/ib_verbs.h>
  46 #include <rdma/ib_user_verbs.h>
  47 #include <rdma/ib_umem.h>
  48 #include <rdma/ib_addr.h>
  49 #include <rdma/ib_mad.h>
  50 #include <rdma/ib_cache.h>
  51 #include <rdma/uverbs_ioctl.h>
  52
  53 #include "bnxt_ulp.h"
  54
  55 #include "roce_hsi.h"
  56 #include "qplib_res.h"
  57 #include "qplib_sp.h"
  58 #include "qplib_fp.h"
  59 #include "qplib_rcfw.h"
  60
  61 #include "bnxt_re.h"
  62 #include "ib_verbs.h"
  63 #include <rdma/bnxt_re-abi.h>
  64
  65 static int __from_ib_access_flags(int iflags)
  66 {
  67         int qflags = 0;
  68
  69         if (iflags & IB_ACCESS_LOCAL_WRITE)
  70                 qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
  71         if (iflags & IB_ACCESS_REMOTE_READ)
  72                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
  73         if (iflags & IB_ACCESS_REMOTE_WRITE)
  74                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
  75         if (iflags & IB_ACCESS_REMOTE_ATOMIC)
  76                 qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
  77         if (iflags & IB_ACCESS_MW_BIND)
  78                 qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
  79         if (iflags & IB_ZERO_BASED)
  80                 qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
  81         if (iflags & IB_ACCESS_ON_DEMAND)
  82                 qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
  83         return qflags;
  84 };
  85
  86 static enum ib_access_flags __to_ib_access_flags(int qflags)
  87 {
  88         enum ib_access_flags iflags = 0;
  89
  90         if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
  91                 iflags |= IB_ACCESS_LOCAL_WRITE;
  92         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
  93                 iflags |= IB_ACCESS_REMOTE_WRITE;
  94         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
  95                 iflags |= IB_ACCESS_REMOTE_READ;
  96         if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
  97                 iflags |= IB_ACCESS_REMOTE_ATOMIC;
  98         if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
  99                 iflags |= IB_ACCESS_MW_BIND;
 100         if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
 101                 iflags |= IB_ZERO_BASED;
 102         if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
 103                 iflags |= IB_ACCESS_ON_DEMAND;
 104         return iflags;
 105 };
 106
 107 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
 108                              struct bnxt_qplib_sge *sg_list, int num)
 109 {
 110         int i, total = 0;
 111
 112         for (i = 0; i < num; i++) {
 113                 sg_list[i].addr = ib_sg_list[i].addr;
 114                 sg_list[i].lkey = ib_sg_list[i].lkey;
 115                 sg_list[i].size = ib_sg_list[i].length;
 116                 total += sg_list[i].size;
 117         }
 118         return total;
 119 }
 120
 121 /* Device */
 122 int bnxt_re_query_device(struct ib_device *ibdev,
 123                          struct ib_device_attr *ib_attr,
 124                          struct ib_udata *udata)
 125 {
 126         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 127         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
 128
 129         memset(ib_attr, 0, sizeof(*ib_attr));
 130         memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
 131                min(sizeof(dev_attr->fw_ver),
 132                    sizeof(ib_attr->fw_ver)));
 133         bnxt_qplib_get_guid(rdev->netdev->dev_addr,
 134                             (u8 *)&ib_attr->sys_image_guid);
 135         ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
 136         ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M;
 137
 138         ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
 139         ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
 140         ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
 141         ib_attr->max_qp = dev_attr->max_qp;
 142         ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
 143         ib_attr->device_cap_flags =
 144                                     IB_DEVICE_CURR_QP_STATE_MOD
 145                                     | IB_DEVICE_RC_RNR_NAK_GEN
 146                                     | IB_DEVICE_SHUTDOWN_PORT
 147                                     | IB_DEVICE_SYS_IMAGE_GUID
 148                                     | IB_DEVICE_LOCAL_DMA_LKEY
 149                                     | IB_DEVICE_RESIZE_MAX_WR
 150                                     | IB_DEVICE_PORT_ACTIVE_EVENT
 151                                     | IB_DEVICE_N_NOTIFY_CQ
 152                                     | IB_DEVICE_MEM_WINDOW
 153                                     | IB_DEVICE_MEM_WINDOW_TYPE_2B
 154                                     | IB_DEVICE_MEM_MGT_EXTENSIONS;
 155         ib_attr->max_send_sge = dev_attr->max_qp_sges;
 156         ib_attr->max_recv_sge = dev_attr->max_qp_sges;
 157         ib_attr->max_sge_rd = dev_attr->max_qp_sges;
 158         ib_attr->max_cq = dev_attr->max_cq;
 159         ib_attr->max_cqe = dev_attr->max_cq_wqes;
 160         ib_attr->max_mr = dev_attr->max_mr;
 161         ib_attr->max_pd = dev_attr->max_pd;
 162         ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
 163         ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
 164         ib_attr->atomic_cap = IB_ATOMIC_NONE;
 165         ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
 166         if (dev_attr->is_atomic) {
 167                 ib_attr->atomic_cap = IB_ATOMIC_GLOB;
 168                 ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
 169         }
 170
 171         ib_attr->max_ee_rd_atom = 0;
 172         ib_attr->max_res_rd_atom = 0;
 173         ib_attr->max_ee_init_rd_atom = 0;
 174         ib_attr->max_ee = 0;
 175         ib_attr->max_rdd = 0;
 176         ib_attr->max_mw = dev_attr->max_mw;
 177         ib_attr->max_raw_ipv6_qp = 0;
 178         ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
 179         ib_attr->max_mcast_grp = 0;
 180         ib_attr->max_mcast_qp_attach = 0;
 181         ib_attr->max_total_mcast_qp_attach = 0;
 182         ib_attr->max_ah = dev_attr->max_ah;
 183
 184         ib_attr->max_srq = dev_attr->max_srq;
 185         ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
 186         ib_attr->max_srq_sge = dev_attr->max_srq_sges;
 187
 188         ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
 189
 190         ib_attr->max_pkeys = 1;
 191         ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
 192         return 0;
 193 }
 194
 195 /* Port */
 196 int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,
 197                        struct ib_port_attr *port_attr)
 198 {
 199         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 200         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
 201
 202         memset(port_attr, 0, sizeof(*port_attr));
 203
 204         if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
 205                 port_attr->state = IB_PORT_ACTIVE;
 206                 port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 207         } else {
 208                 port_attr->state = IB_PORT_DOWN;
 209                 port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 210         }
 211         port_attr->max_mtu = IB_MTU_4096;
 212         port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
 213         port_attr->gid_tbl_len = dev_attr->max_sgid;
 214         port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 215                                     IB_PORT_DEVICE_MGMT_SUP |
 216                                     IB_PORT_VENDOR_CLASS_SUP;
 217         port_attr->ip_gids = true;
 218
 219         port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
 220         port_attr->bad_pkey_cntr = 0;
 221         port_attr->qkey_viol_cntr = 0;
 222         port_attr->pkey_tbl_len = dev_attr->max_pkey;
 223         port_attr->lid = 0;
 224         port_attr->sm_lid = 0;
 225         port_attr->lmc = 0;
 226         port_attr->max_vl_num = 4;
 227         port_attr->sm_sl = 0;
 228         port_attr->subnet_timeout = 0;
 229         port_attr->init_type_reply = 0;
 230         port_attr->active_speed = rdev->active_speed;
 231         port_attr->active_width = rdev->active_width;
 232
 233         return 0;
 234 }
 235
 236 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
 237                                struct ib_port_immutable *immutable)
 238 {
 239         struct ib_port_attr port_attr;
 240
 241         if (bnxt_re_query_port(ibdev, port_num, &port_attr))
 242                 return -EINVAL;
 243
 244         immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
 245         immutable->gid_tbl_len = port_attr.gid_tbl_len;
 246         immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
 247         immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
 248         immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 249         return 0;
 250 }
 251
 252 void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)
 253 {
 254         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 255
 256         snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d",
 257                  rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1],
 258                  rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]);
 259 }
 260
 261 int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,
 262                        u16 index, u16 *pkey)
 263 {
 264         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 265
 266         /* Ignore port_num */
 267
 268         memset(pkey, 0, sizeof(*pkey));
 269         return bnxt_qplib_get_pkey(&rdev->qplib_res,
 270                                    &rdev->qplib_res.pkey_tbl, index, pkey);
 271 }
 272
 273 int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,
 274                       int index, union ib_gid *gid)
 275 {
 276         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 277         int rc = 0;
 278
 279         /* Ignore port_num */
 280         memset(gid, 0, sizeof(*gid));
 281         rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
 282                                  &rdev->qplib_res.sgid_tbl, index,
 283                                  (struct bnxt_qplib_gid *)gid);
 284         return rc;
 285 }
 286
 287 int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)
 288 {
 289         int rc = 0;
 290         struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
 291         struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 292         struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 293         struct bnxt_qplib_gid *gid_to_del;
 294         u16 vlan_id = 0xFFFF;
 295
 296         /* Delete the entry from the hardware */
 297         ctx = *context;
 298         if (!ctx)
 299                 return -EINVAL;
 300
 301         if (sgid_tbl && sgid_tbl->active) {
 302                 if (ctx->idx >= sgid_tbl->max)
 303                         return -EINVAL;
 304                 gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
 305                 vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
 306                 /* DEL_GID is called in WQ context(netdevice_event_work_handler)
 307                  * or via the ib_unregister_device path. In the former case QP1
 308                  * may not be destroyed yet, in which case just return as FW
 309                  * needs that entry to be present and will fail it's deletion.
 310                  * We could get invoked again after QP1 is destroyed OR get an
 311                  * ADD_GID call with a different GID value for the same index
 312                  * where we issue MODIFY_GID cmd to update the GID entry -- TBD
 313                  */
 314                 if (ctx->idx == 0 &&
 315                     rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
 316                     ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) {
 317                         ibdev_dbg(&rdev->ibdev,
 318                                   "Trying to delete GID0 while QP1 is alive\n");
 319                         return -EFAULT;
 320                 }
 321                 ctx->refcnt--;
 322                 if (!ctx->refcnt) {
 323                         rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
 324                                                  vlan_id,  true);
 325                         if (rc) {
 326                                 ibdev_err(&rdev->ibdev,
 327                                           "Failed to remove GID: %#x", rc);
 328                         } else {
 329                                 ctx_tbl = sgid_tbl->ctx;
 330                                 ctx_tbl[ctx->idx] = NULL;
 331                                 kfree(ctx);
 332                         }
 333                 }
 334         } else {
 335                 return -EINVAL;
 336         }
 337         return rc;
 338 }
 339
 340 int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)
 341 {
 342         int rc;
 343         u32 tbl_idx = 0;
 344         u16 vlan_id = 0xFFFF;
 345         struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
 346         struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 347         struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 348
 349         rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL);
 350         if (rc)
 351                 return rc;
 352
 353         rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid,
 354                                  rdev->qplib_res.netdev->dev_addr,
 355                                  vlan_id, true, &tbl_idx);
 356         if (rc == -EALREADY) {
 357                 ctx_tbl = sgid_tbl->ctx;
 358                 ctx_tbl[tbl_idx]->refcnt++;
 359                 *context = ctx_tbl[tbl_idx];
 360                 return 0;
 361         }
 362
 363         if (rc < 0) {
 364                 ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc);
 365                 return rc;
 366         }
 367
 368         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 369         if (!ctx)
 370                 return -ENOMEM;
 371         ctx_tbl = sgid_tbl->ctx;
 372         ctx->idx = tbl_idx;
 373         ctx->refcnt = 1;
 374         ctx_tbl[tbl_idx] = ctx;
 375         *context = ctx;
 376
 377         return rc;
 378 }
 379
 380 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
 381                                             u32 port_num)
 382 {
 383         return IB_LINK_LAYER_ETHERNET;
 384 }
 385
 386 #define BNXT_RE_FENCE_PBL_SIZE  DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE)
 387
 388 static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)
 389 {
 390         struct bnxt_re_fence_data *fence = &pd->fence;
 391         struct ib_mr *ib_mr = &fence->mr->ib_mr;
 392         struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
 393
 394         memset(wqe, 0, sizeof(*wqe));
 395         wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
 396         wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
 397         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
 398         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
 399         wqe->bind.zero_based = false;
 400         wqe->bind.parent_l_key = ib_mr->lkey;
 401         wqe->bind.va = (u64)(unsigned long)fence->va;
 402         wqe->bind.length = fence->size;
 403         wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
 404         wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
 405
 406         /* Save the initial rkey in fence structure for now;
 407          * wqe->bind.r_key will be set at (re)bind time.
 408          */
 409         fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
 410 }
 411
 412 static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
 413 {
 414         struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
 415                                              qplib_qp);
 416         struct ib_pd *ib_pd = qp->ib_qp.pd;
 417         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 418         struct bnxt_re_fence_data *fence = &pd->fence;
 419         struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
 420         struct bnxt_qplib_swqe wqe;
 421         int rc;
 422
 423         memcpy(&wqe, fence_wqe, sizeof(wqe));
 424         wqe.bind.r_key = fence->bind_rkey;
 425         fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
 426
 427         ibdev_dbg(&qp->rdev->ibdev,
 428                   "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
 429                 wqe.bind.r_key, qp->qplib_qp.id, pd);
 430         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
 431         if (rc) {
 432                 ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n");
 433                 return rc;
 434         }
 435         bnxt_qplib_post_send_db(&qp->qplib_qp);
 436
 437         return rc;
 438 }
 439
 440 static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)
 441 {
 442         struct bnxt_re_fence_data *fence = &pd->fence;
 443         struct bnxt_re_dev *rdev = pd->rdev;
 444         struct device *dev = &rdev->en_dev->pdev->dev;
 445         struct bnxt_re_mr *mr = fence->mr;
 446
 447         if (fence->mw) {
 448                 bnxt_re_dealloc_mw(fence->mw);
 449                 fence->mw = NULL;
 450         }
 451         if (mr) {
 452                 if (mr->ib_mr.rkey)
 453                         bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
 454                                              true);
 455                 if (mr->ib_mr.lkey)
 456                         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
 457                 kfree(mr);
 458                 fence->mr = NULL;
 459         }
 460         if (fence->dma_addr) {
 461                 dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES,
 462                                  DMA_BIDIRECTIONAL);
 463                 fence->dma_addr = 0;
 464         }
 465 }
 466
 467 static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
 468 {
 469         int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
 470         struct bnxt_re_fence_data *fence = &pd->fence;
 471         struct bnxt_re_dev *rdev = pd->rdev;
 472         struct device *dev = &rdev->en_dev->pdev->dev;
 473         struct bnxt_re_mr *mr = NULL;
 474         dma_addr_t dma_addr = 0;
 475         struct ib_mw *mw;
 476         int rc;
 477
 478         dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
 479                                   DMA_BIDIRECTIONAL);
 480         rc = dma_mapping_error(dev, dma_addr);
 481         if (rc) {
 482                 ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n");
 483                 rc = -EIO;
 484                 fence->dma_addr = 0;
 485                 goto fail;
 486         }
 487         fence->dma_addr = dma_addr;
 488
 489         /* Allocate a MR */
 490         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 491         if (!mr) {
 492                 rc = -ENOMEM;
 493                 goto fail;
 494         }
 495         fence->mr = mr;
 496         mr->rdev = rdev;
 497         mr->qplib_mr.pd = &pd->qplib_pd;
 498         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
 499         mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
 500         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
 501         if (rc) {
 502                 ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n");
 503                 goto fail;
 504         }
 505
 506         /* Register MR */
 507         mr->ib_mr.lkey = mr->qplib_mr.lkey;
 508         mr->qplib_mr.va = (u64)(unsigned long)fence->va;
 509         mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
 510         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
 511                                BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE);
 512         if (rc) {
 513                 ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
 514                 goto fail;
 515         }
 516         mr->ib_mr.rkey = mr->qplib_mr.rkey;
 517
 518         /* Create a fence MW only for kernel consumers */
 519         mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
 520         if (IS_ERR(mw)) {
 521                 ibdev_err(&rdev->ibdev,
 522                           "Failed to create fence-MW for PD: %p\n", pd);
 523                 rc = PTR_ERR(mw);
 524                 goto fail;
 525         }
 526         fence->mw = mw;
 527
 528         bnxt_re_create_fence_wqe(pd);
 529         return 0;
 530
 531 fail:
 532         bnxt_re_destroy_fence_mr(pd);
 533         return rc;
 534 }
 535
 536 /* Protection Domains */
 537 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
 538 {
 539         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 540         struct bnxt_re_dev *rdev = pd->rdev;
 541
 542         bnxt_re_destroy_fence_mr(pd);
 543
 544         if (pd->qplib_pd.id)
 545                 bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
 546                                       &pd->qplib_pd);
 547         return 0;
 548 }
 549
 550 int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 551 {
 552         struct ib_device *ibdev = ibpd->device;
 553         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 554         struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(
 555                 udata, struct bnxt_re_ucontext, ib_uctx);
 556         struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);
 557         int rc;
 558
 559         pd->rdev = rdev;
 560         if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) {
 561                 ibdev_err(&rdev->ibdev, "Failed to allocate HW PD");
 562                 rc = -ENOMEM;
 563                 goto fail;
 564         }
 565
 566         if (udata) {
 567                 struct bnxt_re_pd_resp resp;
 568
 569                 if (!ucntx->dpi.dbr) {
 570                         /* Allocate DPI in alloc_pd to avoid failing of
 571                          * ibv_devinfo and family of application when DPIs
 572                          * are depleted.
 573                          */
 574                         if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl,
 575                                                  &ucntx->dpi, ucntx)) {
 576                                 rc = -ENOMEM;
 577                                 goto dbfail;
 578                         }
 579                 }
 580
 581                 resp.pdid = pd->qplib_pd.id;
 582                 /* Still allow mapping this DBR to the new user PD. */
 583                 resp.dpi = ucntx->dpi.dpi;
 584                 resp.dbr = (u64)ucntx->dpi.umdbr;
 585
 586                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
 587                 if (rc) {
 588                         ibdev_err(&rdev->ibdev,
 589                                   "Failed to copy user response\n");
 590                         goto dbfail;
 591                 }
 592         }
 593
 594         if (!udata)
 595                 if (bnxt_re_create_fence_mr(pd))
 596                         ibdev_warn(&rdev->ibdev,
 597                                    "Failed to create Fence-MR\n");
 598         return 0;
 599 dbfail:
 600         bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
 601                               &pd->qplib_pd);
 602 fail:
 603         return rc;
 604 }
 605
 606 /* Address Handles */
 607 int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
 608 {
 609         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 610         struct bnxt_re_dev *rdev = ah->rdev;
 611
 612         bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah,
 613                               !(flags & RDMA_DESTROY_AH_SLEEPABLE));
 614         return 0;
 615 }
 616
 617 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
 618 {
 619         u8 nw_type;
 620
 621         switch (ntype) {
 622         case RDMA_NETWORK_IPV4:
 623                 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
 624                 break;
 625         case RDMA_NETWORK_IPV6:
 626                 nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
 627                 break;
 628         default:
 629                 nw_type = CMDQ_CREATE_AH_TYPE_V1;
 630                 break;
 631         }
 632         return nw_type;
 633 }
 634
 635 int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
 636                       struct ib_udata *udata)
 637 {
 638         struct ib_pd *ib_pd = ib_ah->pd;
 639         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 640         struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 641         const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
 642         struct bnxt_re_dev *rdev = pd->rdev;
 643         const struct ib_gid_attr *sgid_attr;
 644         struct bnxt_re_gid_ctx *ctx;
 645         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 646         u8 nw_type;
 647         int rc;
 648
 649         if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
 650                 ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set");
 651                 return -EINVAL;
 652         }
 653
 654         ah->rdev = rdev;
 655         ah->qplib_ah.pd = &pd->qplib_pd;
 656
 657         /* Supply the configuration for the HW */
 658         memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
 659                sizeof(union ib_gid));
 660         sgid_attr = grh->sgid_attr;
 661         /* Get the HW context of the GID. The reference
 662          * of GID table entry is already taken by the caller.
 663          */
 664         ctx = rdma_read_gid_hw_context(sgid_attr);
 665         ah->qplib_ah.sgid_index = ctx->idx;
 666         ah->qplib_ah.host_sgid_index = grh->sgid_index;
 667         ah->qplib_ah.traffic_class = grh->traffic_class;
 668         ah->qplib_ah.flow_label = grh->flow_label;
 669         ah->qplib_ah.hop_limit = grh->hop_limit;
 670         ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
 671
 672         /* Get network header type for this GID */
 673         nw_type = rdma_gid_attr_network_type(sgid_attr);
 674         ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type);
 675
 676         memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
 677         rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah,
 678                                   !(init_attr->flags &
 679                                     RDMA_CREATE_AH_SLEEPABLE));
 680         if (rc) {
 681                 ibdev_err(&rdev->ibdev, "Failed to allocate HW AH");
 682                 return rc;
 683         }
 684
 685         /* Write AVID to shared page. */
 686         if (udata) {
 687                 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
 688                         udata, struct bnxt_re_ucontext, ib_uctx);
 689                 unsigned long flag;
 690                 u32 *wrptr;
 691
 692                 spin_lock_irqsave(&uctx->sh_lock, flag);
 693                 wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
 694                 *wrptr = ah->qplib_ah.id;
 695                 wmb(); /* make sure cache is updated. */
 696                 spin_unlock_irqrestore(&uctx->sh_lock, flag);
 697         }
 698
 699         return 0;
 700 }
 701
 702 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 703 {
 704         return 0;
 705 }
 706
 707 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 708 {
 709         struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 710
 711         ah_attr->type = ib_ah->type;
 712         rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
 713         memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
 714         rdma_ah_set_grh(ah_attr, NULL, 0,
 715                         ah->qplib_ah.host_sgid_index,
 716                         0, ah->qplib_ah.traffic_class);
 717         rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data);
 718         rdma_ah_set_port_num(ah_attr, 1);
 719         rdma_ah_set_static_rate(ah_attr, 0);
 720         return 0;
 721 }
 722
 723 unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
 724         __acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
 725 {
 726         unsigned long flags;
 727
 728         spin_lock_irqsave(&qp->scq->cq_lock, flags);
 729         if (qp->rcq != qp->scq)
 730                 spin_lock(&qp->rcq->cq_lock);
 731         else
 732                 __acquire(&qp->rcq->cq_lock);
 733
 734         return flags;
 735 }
 736
 737 void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
 738                         unsigned long flags)
 739         __releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
 740 {
 741         if (qp->rcq != qp->scq)
 742                 spin_unlock(&qp->rcq->cq_lock);
 743         else
 744                 __release(&qp->rcq->cq_lock);
 745         spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
 746 }
 747
 748 static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
 749 {
 750         struct bnxt_re_qp *gsi_sqp;
 751         struct bnxt_re_ah *gsi_sah;
 752         struct bnxt_re_dev *rdev;
 753         int rc = 0;
 754
 755         rdev = qp->rdev;
 756         gsi_sqp = rdev->gsi_ctx.gsi_sqp;
 757         gsi_sah = rdev->gsi_ctx.gsi_sah;
 758
 759         ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n");
 760         bnxt_qplib_destroy_ah(&rdev->qplib_res,
 761                               &gsi_sah->qplib_ah,
 762                               true);
 763         bnxt_qplib_clean_qp(&qp->qplib_qp);
 764
 765         ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n");
 766         rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
 767         if (rc) {
 768                 ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed");
 769                 goto fail;
 770         }
 771         bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
 772
 773         /* remove from active qp list */
 774         mutex_lock(&rdev->qp_lock);
 775         list_del(&gsi_sqp->list);
 776         mutex_unlock(&rdev->qp_lock);
 777         atomic_dec(&rdev->qp_count);
 778
 779         kfree(rdev->gsi_ctx.sqp_tbl);
 780         kfree(gsi_sah);
 781         kfree(gsi_sqp);
 782         rdev->gsi_ctx.gsi_sqp = NULL;
 783         rdev->gsi_ctx.gsi_sah = NULL;
 784         rdev->gsi_ctx.sqp_tbl = NULL;
 785
 786         return 0;
 787 fail:
 788         return rc;
 789 }
 790
 791 /* Queue Pairs */
 792 int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 793 {
 794         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 795         struct bnxt_re_dev *rdev = qp->rdev;
 796         unsigned int flags;
 797         int rc;
 798
 799         bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
 800
 801         rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
 802         if (rc) {
 803                 ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
 804                 return rc;
 805         }
 806
 807         if (rdma_is_kernel_res(&qp->ib_qp.res)) {
 808                 flags = bnxt_re_lock_cqs(qp);
 809                 bnxt_qplib_clean_qp(&qp->qplib_qp);
 810                 bnxt_re_unlock_cqs(qp, flags);
 811         }
 812
 813         bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
 814
 815         if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp) {
 816                 rc = bnxt_re_destroy_gsi_sqp(qp);
 817                 if (rc)
 818                         return rc;
 819         }
 820
 821         mutex_lock(&rdev->qp_lock);
 822         list_del(&qp->list);
 823         mutex_unlock(&rdev->qp_lock);
 824         atomic_dec(&rdev->qp_count);
 825
 826         ib_umem_release(qp->rumem);
 827         ib_umem_release(qp->sumem);
 828
 829         return 0;
 830 }
 831
 832 static u8 __from_ib_qp_type(enum ib_qp_type type)
 833 {
 834         switch (type) {
 835         case IB_QPT_GSI:
 836                 return CMDQ_CREATE_QP1_TYPE_GSI;
 837         case IB_QPT_RC:
 838                 return CMDQ_CREATE_QP_TYPE_RC;
 839         case IB_QPT_UD:
 840                 return CMDQ_CREATE_QP_TYPE_UD;
 841         default:
 842                 return IB_QPT_MAX;
 843         }
 844 }
 845
 846 static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
 847                                    int rsge, int max)
 848 {
 849         if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
 850                 rsge = max;
 851         return bnxt_re_get_rwqe_size(rsge);
 852 }
 853
 854 static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
 855 {
 856         u16 wqe_size, calc_ils;
 857
 858         wqe_size = bnxt_re_get_swqe_size(nsge);
 859         if (ilsize) {
 860                 calc_ils = sizeof(struct sq_send_hdr) + ilsize;
 861                 wqe_size = max_t(u16, calc_ils, wqe_size);
 862                 wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
 863         }
 864         return wqe_size;
 865 }
 866
 867 static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
 868                                    struct ib_qp_init_attr *init_attr)
 869 {
 870         struct bnxt_qplib_dev_attr *dev_attr;
 871         struct bnxt_qplib_qp *qplqp;
 872         struct bnxt_re_dev *rdev;
 873         struct bnxt_qplib_q *sq;
 874         int align, ilsize;
 875
 876         rdev = qp->rdev;
 877         qplqp = &qp->qplib_qp;
 878         sq = &qplqp->sq;
 879         dev_attr = &rdev->dev_attr;
 880
 881         align = sizeof(struct sq_send_hdr);
 882         ilsize = ALIGN(init_attr->cap.max_inline_data, align);
 883
 884         sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge);
 885         if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges))
 886                 return -EINVAL;
 887         /* For gen p4 and gen p5 backward compatibility mode
 888          * wqe size is fixed to 128 bytes
 889          */
 890         if (sq->wqe_size < bnxt_re_get_swqe_size(dev_attr->max_qp_sges) &&
 891                         qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
 892                 sq->wqe_size = bnxt_re_get_swqe_size(dev_attr->max_qp_sges);
 893
 894         if (init_attr->cap.max_inline_data) {
 895                 qplqp->max_inline_data = sq->wqe_size -
 896                         sizeof(struct sq_send_hdr);
 897                 init_attr->cap.max_inline_data = qplqp->max_inline_data;
 898                 if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
 899                         sq->max_sge = qplqp->max_inline_data /
 900                                 sizeof(struct sq_sge);
 901         }
 902
 903         return 0;
 904 }
 905
 906 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
 907                                 struct bnxt_re_qp *qp, struct ib_udata *udata)
 908 {
 909         struct bnxt_qplib_qp *qplib_qp;
 910         struct bnxt_re_ucontext *cntx;
 911         struct bnxt_re_qp_req ureq;
 912         int bytes = 0, psn_sz;
 913         struct ib_umem *umem;
 914         int psn_nume;
 915
 916         qplib_qp = &qp->qplib_qp;
 917         cntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
 918                                          ib_uctx);
 919         if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
 920                 return -EFAULT;
 921
 922         bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
 923         /* Consider mapping PSN search memory only for RC QPs. */
 924         if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
 925                 psn_sz = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
 926                                                    sizeof(struct sq_psn_search_ext) :
 927                                                    sizeof(struct sq_psn_search);
 928                 psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
 929                             qplib_qp->sq.max_wqe :
 930                             ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
 931                               sizeof(struct bnxt_qplib_sge));
 932                 bytes += (psn_nume * psn_sz);
 933         }
 934
 935         bytes = PAGE_ALIGN(bytes);
 936         umem = ib_umem_get(&rdev->ibdev, ureq.qpsva, bytes,
 937                            IB_ACCESS_LOCAL_WRITE);
 938         if (IS_ERR(umem))
 939                 return PTR_ERR(umem);
 940
 941         qp->sumem = umem;
 942         qplib_qp->sq.sg_info.umem = umem;
 943         qplib_qp->sq.sg_info.pgsize = PAGE_SIZE;
 944         qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT;
 945         qplib_qp->qp_handle = ureq.qp_handle;
 946
 947         if (!qp->qplib_qp.srq) {
 948                 bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
 949                 bytes = PAGE_ALIGN(bytes);
 950                 umem = ib_umem_get(&rdev->ibdev, ureq.qprva, bytes,
 951                                    IB_ACCESS_LOCAL_WRITE);
 952                 if (IS_ERR(umem))
 953                         goto rqfail;
 954                 qp->rumem = umem;
 955                 qplib_qp->rq.sg_info.umem = umem;
 956                 qplib_qp->rq.sg_info.pgsize = PAGE_SIZE;
 957                 qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT;
 958         }
 959
 960         qplib_qp->dpi = &cntx->dpi;
 961         return 0;
 962 rqfail:
 963         ib_umem_release(qp->sumem);
 964         qp->sumem = NULL;
 965         memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
 966
 967         return PTR_ERR(umem);
 968 }
 969
 970 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
 971                                 (struct bnxt_re_pd *pd,
 972                                  struct bnxt_qplib_res *qp1_res,
 973                                  struct bnxt_qplib_qp *qp1_qp)
 974 {
 975         struct bnxt_re_dev *rdev = pd->rdev;
 976         struct bnxt_re_ah *ah;
 977         union ib_gid sgid;
 978         int rc;
 979
 980         ah = kzalloc(sizeof(*ah), GFP_KERNEL);
 981         if (!ah)
 982                 return NULL;
 983
 984         ah->rdev = rdev;
 985         ah->qplib_ah.pd = &pd->qplib_pd;
 986
 987         rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
 988         if (rc)
 989                 goto fail;
 990
 991         /* supply the dgid data same as sgid */
 992         memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
 993                sizeof(union ib_gid));
 994         ah->qplib_ah.sgid_index = 0;
 995
 996         ah->qplib_ah.traffic_class = 0;
 997         ah->qplib_ah.flow_label = 0;
 998         ah->qplib_ah.hop_limit = 1;
 999         ah->qplib_ah.sl = 0;
1000         /* Have DMAC same as SMAC */
1001         ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
1002
1003         rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false);
1004         if (rc) {
1005                 ibdev_err(&rdev->ibdev,
1006                           "Failed to allocate HW AH for Shadow QP");
1007                 goto fail;
1008         }
1009
1010         return ah;
1011
1012 fail:
1013         kfree(ah);
1014         return NULL;
1015 }
1016
1017 static struct bnxt_re_qp *bnxt_re_create_shadow_qp
1018                                 (struct bnxt_re_pd *pd,
1019                                  struct bnxt_qplib_res *qp1_res,
1020                                  struct bnxt_qplib_qp *qp1_qp)
1021 {
1022         struct bnxt_re_dev *rdev = pd->rdev;
1023         struct bnxt_re_qp *qp;
1024         int rc;
1025
1026         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1027         if (!qp)
1028                 return NULL;
1029
1030         qp->rdev = rdev;
1031
1032         /* Initialize the shadow QP structure from the QP1 values */
1033         ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
1034
1035         qp->qplib_qp.pd = &pd->qplib_pd;
1036         qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
1037         qp->qplib_qp.type = IB_QPT_UD;
1038
1039         qp->qplib_qp.max_inline_data = 0;
1040         qp->qplib_qp.sig_type = true;
1041
1042         /* Shadow QP SQ depth should be same as QP1 RQ depth */
1043         qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);
1044         qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
1045         qp->qplib_qp.sq.max_sge = 2;
1046         /* Q full delta can be 1 since it is internal QP */
1047         qp->qplib_qp.sq.q_full_delta = 1;
1048         qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE;
1049         qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT;
1050
1051         qp->qplib_qp.scq = qp1_qp->scq;
1052         qp->qplib_qp.rcq = qp1_qp->rcq;
1053
1054         qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);
1055         qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
1056         qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
1057         /* Q full delta can be 1 since it is internal QP */
1058         qp->qplib_qp.rq.q_full_delta = 1;
1059         qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE;
1060         qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT;
1061
1062         qp->qplib_qp.mtu = qp1_qp->mtu;
1063
1064         qp->qplib_qp.sq_hdr_buf_size = 0;
1065         qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
1066         qp->qplib_qp.dpi = &rdev->dpi_privileged;
1067
1068         rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
1069         if (rc)
1070                 goto fail;
1071
1072         spin_lock_init(&qp->sq_lock);
1073         INIT_LIST_HEAD(&qp->list);
1074         mutex_lock(&rdev->qp_lock);
1075         list_add_tail(&qp->list, &rdev->qp_list);
1076         atomic_inc(&rdev->qp_count);
1077         mutex_unlock(&rdev->qp_lock);
1078         return qp;
1079 fail:
1080         kfree(qp);
1081         return NULL;
1082 }
1083
1084 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
1085                                 struct ib_qp_init_attr *init_attr)
1086 {
1087         struct bnxt_qplib_dev_attr *dev_attr;
1088         struct bnxt_qplib_qp *qplqp;
1089         struct bnxt_re_dev *rdev;
1090         struct bnxt_qplib_q *rq;
1091         int entries;
1092
1093         rdev = qp->rdev;
1094         qplqp = &qp->qplib_qp;
1095         rq = &qplqp->rq;
1096         dev_attr = &rdev->dev_attr;
1097
1098         if (init_attr->srq) {
1099                 struct bnxt_re_srq *srq;
1100
1101                 srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq);
1102                 qplqp->srq = &srq->qplib_srq;
1103                 rq->max_wqe = 0;
1104         } else {
1105                 rq->max_sge = init_attr->cap.max_recv_sge;
1106                 if (rq->max_sge > dev_attr->max_qp_sges)
1107                         rq->max_sge = dev_attr->max_qp_sges;
1108                 init_attr->cap.max_recv_sge = rq->max_sge;
1109                 rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
1110                                                        dev_attr->max_qp_sges);
1111                 /* Allocate 1 more than what's provided so posting max doesn't
1112                  * mean empty.
1113                  */
1114                 entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1);
1115                 rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1116                 rq->q_full_delta = 0;
1117                 rq->sg_info.pgsize = PAGE_SIZE;
1118                 rq->sg_info.pgshft = PAGE_SHIFT;
1119         }
1120
1121         return 0;
1122 }
1123
1124 static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
1125 {
1126         struct bnxt_qplib_dev_attr *dev_attr;
1127         struct bnxt_qplib_qp *qplqp;
1128         struct bnxt_re_dev *rdev;
1129
1130         rdev = qp->rdev;
1131         qplqp = &qp->qplib_qp;
1132         dev_attr = &rdev->dev_attr;
1133
1134         if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1135                 qplqp->rq.max_sge = dev_attr->max_qp_sges;
1136                 if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
1137                         qplqp->rq.max_sge = dev_attr->max_qp_sges;
1138                 qplqp->rq.max_sge = 6;
1139         }
1140 }
1141
1142 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
1143                                 struct ib_qp_init_attr *init_attr,
1144                                 struct ib_udata *udata)
1145 {
1146         struct bnxt_qplib_dev_attr *dev_attr;
1147         struct bnxt_qplib_qp *qplqp;
1148         struct bnxt_re_dev *rdev;
1149         struct bnxt_qplib_q *sq;
1150         int entries;
1151         int diff;
1152         int rc;
1153
1154         rdev = qp->rdev;
1155         qplqp = &qp->qplib_qp;
1156         sq = &qplqp->sq;
1157         dev_attr = &rdev->dev_attr;
1158
1159         sq->max_sge = init_attr->cap.max_send_sge;
1160         if (sq->max_sge > dev_attr->max_qp_sges) {
1161                 sq->max_sge = dev_attr->max_qp_sges;
1162                 init_attr->cap.max_send_sge = sq->max_sge;
1163         }
1164
1165         rc = bnxt_re_setup_swqe_size(qp, init_attr);
1166         if (rc)
1167                 return rc;
1168
1169         entries = init_attr->cap.max_send_wr;
1170         /* Allocate 128 + 1 more than what's provided */
1171         diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
1172                 0 : BNXT_QPLIB_RESERVED_QP_WRS;
1173         entries = roundup_pow_of_two(entries + diff + 1);
1174         sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
1175         sq->q_full_delta = diff + 1;
1176         /*
1177          * Reserving one slot for Phantom WQE. Application can
1178          * post one extra entry in this case. But allowing this to avoid
1179          * unexpected Queue full condition
1180          */
1181         qplqp->sq.q_full_delta -= 1;
1182         qplqp->sq.sg_info.pgsize = PAGE_SIZE;
1183         qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
1184
1185         return 0;
1186 }
1187
1188 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
1189                                        struct ib_qp_init_attr *init_attr)
1190 {
1191         struct bnxt_qplib_dev_attr *dev_attr;
1192         struct bnxt_qplib_qp *qplqp;
1193         struct bnxt_re_dev *rdev;
1194         int entries;
1195
1196         rdev = qp->rdev;
1197         qplqp = &qp->qplib_qp;
1198         dev_attr = &rdev->dev_attr;
1199
1200         if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
1201                 entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1);
1202                 qplqp->sq.max_wqe = min_t(u32, entries,
1203                                           dev_attr->max_qp_wqes + 1);
1204                 qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
1205                         init_attr->cap.max_send_wr;
1206                 qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
1207                 if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
1208                         qplqp->sq.max_sge = dev_attr->max_qp_sges;
1209         }
1210 }
1211
1212 static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
1213                                 struct ib_qp_init_attr *init_attr)
1214 {
1215         struct bnxt_qplib_chip_ctx *chip_ctx;
1216         int qptype;
1217
1218         chip_ctx = rdev->chip_ctx;
1219
1220         qptype = __from_ib_qp_type(init_attr->qp_type);
1221         if (qptype == IB_QPT_MAX) {
1222                 ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype);
1223                 qptype = -EOPNOTSUPP;
1224                 goto out;
1225         }
1226
1227         if (bnxt_qplib_is_chip_gen_p5(chip_ctx) &&
1228             init_attr->qp_type == IB_QPT_GSI)
1229                 qptype = CMDQ_CREATE_QP_TYPE_GSI;
1230 out:
1231         return qptype;
1232 }
1233
1234 static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1235                                 struct ib_qp_init_attr *init_attr,
1236                                 struct ib_udata *udata)
1237 {
1238         struct bnxt_qplib_dev_attr *dev_attr;
1239         struct bnxt_qplib_qp *qplqp;
1240         struct bnxt_re_dev *rdev;
1241         struct bnxt_re_cq *cq;
1242         int rc = 0, qptype;
1243
1244         rdev = qp->rdev;
1245         qplqp = &qp->qplib_qp;
1246         dev_attr = &rdev->dev_attr;
1247
1248         /* Setup misc params */
1249         ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr);
1250         qplqp->pd = &pd->qplib_pd;
1251         qplqp->qp_handle = (u64)qplqp;
1252         qplqp->max_inline_data = init_attr->cap.max_inline_data;
1253         qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ?
1254                             true : false);
1255         qptype = bnxt_re_init_qp_type(rdev, init_attr);
1256         if (qptype < 0) {
1257                 rc = qptype;
1258                 goto out;
1259         }
1260         qplqp->type = (u8)qptype;
1261         qplqp->wqe_mode = rdev->chip_ctx->modes.wqe_mode;
1262
1263         if (init_attr->qp_type == IB_QPT_RC) {
1264                 qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
1265                 qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1266         }
1267         qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1268         qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
1269         if (init_attr->create_flags) {
1270                 ibdev_dbg(&rdev->ibdev,
1271                           "QP create flags 0x%x not supported",
1272                           init_attr->create_flags);
1273                 return -EOPNOTSUPP;
1274         }
1275
1276         /* Setup CQs */
1277         if (init_attr->send_cq) {
1278                 cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq);
1279                 qplqp->scq = &cq->qplib_cq;
1280                 qp->scq = cq;
1281         }
1282
1283         if (init_attr->recv_cq) {
1284                 cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq);
1285                 qplqp->rcq = &cq->qplib_cq;
1286                 qp->rcq = cq;
1287         }
1288
1289         /* Setup RQ/SRQ */
1290         rc = bnxt_re_init_rq_attr(qp, init_attr);
1291         if (rc)
1292                 goto out;
1293         if (init_attr->qp_type == IB_QPT_GSI)
1294                 bnxt_re_adjust_gsi_rq_attr(qp);
1295
1296         /* Setup SQ */
1297         rc = bnxt_re_init_sq_attr(qp, init_attr, udata);
1298         if (rc)
1299                 goto out;
1300         if (init_attr->qp_type == IB_QPT_GSI)
1301                 bnxt_re_adjust_gsi_sq_attr(qp, init_attr);
1302
1303         if (udata) /* This will update DPI and qp_handle */
1304                 rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
1305 out:
1306         return rc;
1307 }
1308
1309 static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
1310                                      struct bnxt_re_pd *pd)
1311 {
1312         struct bnxt_re_sqp_entries *sqp_tbl = NULL;
1313         struct bnxt_re_dev *rdev;
1314         struct bnxt_re_qp *sqp;
1315         struct bnxt_re_ah *sah;
1316         int rc = 0;
1317
1318         rdev = qp->rdev;
1319         /* Create a shadow QP to handle the QP1 traffic */
1320         sqp_tbl = kzalloc(sizeof(*sqp_tbl) * BNXT_RE_MAX_GSI_SQP_ENTRIES,
1321                           GFP_KERNEL);
1322         if (!sqp_tbl)
1323                 return -ENOMEM;
1324         rdev->gsi_ctx.sqp_tbl = sqp_tbl;
1325
1326         sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
1327         if (!sqp) {
1328                 rc = -ENODEV;
1329                 ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1");
1330                 goto out;
1331         }
1332         rdev->gsi_ctx.gsi_sqp = sqp;
1333
1334         sqp->rcq = qp->rcq;
1335         sqp->scq = qp->scq;
1336         sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
1337                                           &qp->qplib_qp);
1338         if (!sah) {
1339                 bnxt_qplib_destroy_qp(&rdev->qplib_res,
1340                                       &sqp->qplib_qp);
1341                 rc = -ENODEV;
1342                 ibdev_err(&rdev->ibdev,
1343                           "Failed to create AH entry for ShadowQP");
1344                 goto out;
1345         }
1346         rdev->gsi_ctx.gsi_sah = sah;
1347
1348         return 0;
1349 out:
1350         kfree(sqp_tbl);
1351         return rc;
1352 }
1353
1354 static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1355                                  struct ib_qp_init_attr *init_attr)
1356 {
1357         struct bnxt_re_dev *rdev;
1358         struct bnxt_qplib_qp *qplqp;
1359         int rc = 0;
1360
1361         rdev = qp->rdev;
1362         qplqp = &qp->qplib_qp;
1363
1364         qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1365         qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
1366
1367         rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp);
1368         if (rc) {
1369                 ibdev_err(&rdev->ibdev, "create HW QP1 failed!");
1370                 goto out;
1371         }
1372
1373         rc = bnxt_re_create_shadow_gsi(qp, pd);
1374 out:
1375         return rc;
1376 }
1377
1378 static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
1379                                    struct ib_qp_init_attr *init_attr,
1380                                    struct bnxt_qplib_dev_attr *dev_attr)
1381 {
1382         bool rc = true;
1383
1384         if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes ||
1385             init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes ||
1386             init_attr->cap.max_send_sge > dev_attr->max_qp_sges ||
1387             init_attr->cap.max_recv_sge > dev_attr->max_qp_sges ||
1388             init_attr->cap.max_inline_data > dev_attr->max_inline_data) {
1389                 ibdev_err(&rdev->ibdev,
1390                           "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",
1391                           init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
1392                           init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
1393                           init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
1394                           init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
1395                           init_attr->cap.max_inline_data,
1396                           dev_attr->max_inline_data);
1397                 rc = false;
1398         }
1399         return rc;
1400 }
1401
1402 int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
1403                       struct ib_udata *udata)
1404 {
1405         struct ib_pd *ib_pd = ib_qp->pd;
1406         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1407         struct bnxt_re_dev *rdev = pd->rdev;
1408         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1409         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1410         int rc;
1411
1412         rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
1413         if (!rc) {
1414                 rc = -EINVAL;
1415                 goto fail;
1416         }
1417
1418         qp->rdev = rdev;
1419         rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata);
1420         if (rc)
1421                 goto fail;
1422
1423         if (qp_init_attr->qp_type == IB_QPT_GSI &&
1424             !(bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))) {
1425                 rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr);
1426                 if (rc == -ENODEV)
1427                         goto qp_destroy;
1428                 if (rc)
1429                         goto fail;
1430         } else {
1431                 rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1432                 if (rc) {
1433                         ibdev_err(&rdev->ibdev, "Failed to create HW QP");
1434                         goto free_umem;
1435                 }
1436                 if (udata) {
1437                         struct bnxt_re_qp_resp resp;
1438
1439                         resp.qpid = qp->qplib_qp.id;
1440                         resp.rsvd = 0;
1441                         rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1442                         if (rc) {
1443                                 ibdev_err(&rdev->ibdev, "Failed to copy QP udata");
1444                                 goto qp_destroy;
1445                         }
1446                 }
1447         }
1448
1449         qp->ib_qp.qp_num = qp->qplib_qp.id;
1450         if (qp_init_attr->qp_type == IB_QPT_GSI)
1451                 rdev->gsi_ctx.gsi_qp = qp;
1452         spin_lock_init(&qp->sq_lock);
1453         spin_lock_init(&qp->rq_lock);
1454         INIT_LIST_HEAD(&qp->list);
1455         mutex_lock(&rdev->qp_lock);
1456         list_add_tail(&qp->list, &rdev->qp_list);
1457         mutex_unlock(&rdev->qp_lock);
1458         atomic_inc(&rdev->qp_count);
1459
1460         return 0;
1461 qp_destroy:
1462         bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1463 free_umem:
1464         ib_umem_release(qp->rumem);
1465         ib_umem_release(qp->sumem);
1466 fail:
1467         return rc;
1468 }
1469
1470 static u8 __from_ib_qp_state(enum ib_qp_state state)
1471 {
1472         switch (state) {
1473         case IB_QPS_RESET:
1474                 return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1475         case IB_QPS_INIT:
1476                 return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1477         case IB_QPS_RTR:
1478                 return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1479         case IB_QPS_RTS:
1480                 return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1481         case IB_QPS_SQD:
1482                 return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1483         case IB_QPS_SQE:
1484                 return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1485         case IB_QPS_ERR:
1486         default:
1487                 return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1488         }
1489 }
1490
1491 static enum ib_qp_state __to_ib_qp_state(u8 state)
1492 {
1493         switch (state) {
1494         case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1495                 return IB_QPS_RESET;
1496         case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1497                 return IB_QPS_INIT;
1498         case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1499                 return IB_QPS_RTR;
1500         case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1501                 return IB_QPS_RTS;
1502         case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1503                 return IB_QPS_SQD;
1504         case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1505                 return IB_QPS_SQE;
1506         case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1507         default:
1508                 return IB_QPS_ERR;
1509         }
1510 }
1511
1512 static u32 __from_ib_mtu(enum ib_mtu mtu)
1513 {
1514         switch (mtu) {
1515         case IB_MTU_256:
1516                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1517         case IB_MTU_512:
1518                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1519         case IB_MTU_1024:
1520                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1521         case IB_MTU_2048:
1522                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1523         case IB_MTU_4096:
1524                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1525         default:
1526                 return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1527         }
1528 }
1529
1530 static enum ib_mtu __to_ib_mtu(u32 mtu)
1531 {
1532         switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1533         case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1534                 return IB_MTU_256;
1535         case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1536                 return IB_MTU_512;
1537         case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1538                 return IB_MTU_1024;
1539         case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1540                 return IB_MTU_2048;
1541         case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1542                 return IB_MTU_4096;
1543         default:
1544                 return IB_MTU_2048;
1545         }
1546 }
1547
1548 /* Shared Receive Queues */
1549 int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
1550 {
1551         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1552                                                ib_srq);
1553         struct bnxt_re_dev *rdev = srq->rdev;
1554         struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1555         struct bnxt_qplib_nq *nq = NULL;
1556
1557         if (qplib_srq->cq)
1558                 nq = qplib_srq->cq->nq;
1559         bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
1560         ib_umem_release(srq->umem);
1561         atomic_dec(&rdev->srq_count);
1562         if (nq)
1563                 nq->budget--;
1564         return 0;
1565 }
1566
1567 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
1568                                  struct bnxt_re_pd *pd,
1569                                  struct bnxt_re_srq *srq,
1570                                  struct ib_udata *udata)
1571 {
1572         struct bnxt_re_srq_req ureq;
1573         struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1574         struct ib_umem *umem;
1575         int bytes = 0;
1576         struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
1577                 udata, struct bnxt_re_ucontext, ib_uctx);
1578
1579         if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1580                 return -EFAULT;
1581
1582         bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
1583         bytes = PAGE_ALIGN(bytes);
1584         umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes,
1585                            IB_ACCESS_LOCAL_WRITE);
1586         if (IS_ERR(umem))
1587                 return PTR_ERR(umem);
1588
1589         srq->umem = umem;
1590         qplib_srq->sg_info.umem = umem;
1591         qplib_srq->sg_info.pgsize = PAGE_SIZE;
1592         qplib_srq->sg_info.pgshft = PAGE_SHIFT;
1593         qplib_srq->srq_handle = ureq.srq_handle;
1594         qplib_srq->dpi = &cntx->dpi;
1595
1596         return 0;
1597 }
1598
1599 int bnxt_re_create_srq(struct ib_srq *ib_srq,
1600                        struct ib_srq_init_attr *srq_init_attr,
1601                        struct ib_udata *udata)
1602 {
1603         struct bnxt_qplib_dev_attr *dev_attr;
1604         struct bnxt_qplib_nq *nq = NULL;
1605         struct bnxt_re_dev *rdev;
1606         struct bnxt_re_srq *srq;
1607         struct bnxt_re_pd *pd;
1608         struct ib_pd *ib_pd;
1609         int rc, entries;
1610
1611         ib_pd = ib_srq->pd;
1612         pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1613         rdev = pd->rdev;
1614         dev_attr = &rdev->dev_attr;
1615         srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);
1616
1617         if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
1618                 ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded");
1619                 rc = -EINVAL;
1620                 goto exit;
1621         }
1622
1623         if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
1624                 rc = -EOPNOTSUPP;
1625                 goto exit;
1626         }
1627
1628         srq->rdev = rdev;
1629         srq->qplib_srq.pd = &pd->qplib_pd;
1630         srq->qplib_srq.dpi = &rdev->dpi_privileged;
1631         /* Allocate 1 more than what's provided so posting max doesn't
1632          * mean empty
1633          */
1634         entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
1635         if (entries > dev_attr->max_srq_wqes + 1)
1636                 entries = dev_attr->max_srq_wqes + 1;
1637         srq->qplib_srq.max_wqe = entries;
1638
1639         srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
1640          /* 128 byte wqe size for SRQ . So use max sges */
1641         srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);
1642         srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
1643         srq->srq_limit = srq_init_attr->attr.srq_limit;
1644         srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
1645         nq = &rdev->nq[0];
1646
1647         if (udata) {
1648                 rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
1649                 if (rc)
1650                         goto fail;
1651         }
1652
1653         rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
1654         if (rc) {
1655                 ibdev_err(&rdev->ibdev, "Create HW SRQ failed!");
1656                 goto fail;
1657         }
1658
1659         if (udata) {
1660                 struct bnxt_re_srq_resp resp;
1661
1662                 resp.srqid = srq->qplib_srq.id;
1663                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1664                 if (rc) {
1665                         ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!");
1666                         bnxt_qplib_destroy_srq(&rdev->qplib_res,
1667                                                &srq->qplib_srq);
1668                         goto fail;
1669                 }
1670         }
1671         if (nq)
1672                 nq->budget++;
1673         atomic_inc(&rdev->srq_count);
1674         spin_lock_init(&srq->lock);
1675
1676         return 0;
1677
1678 fail:
1679         ib_umem_release(srq->umem);
1680 exit:
1681         return rc;
1682 }
1683
1684 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
1685                        enum ib_srq_attr_mask srq_attr_mask,
1686                        struct ib_udata *udata)
1687 {
1688         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1689                                                ib_srq);
1690         struct bnxt_re_dev *rdev = srq->rdev;
1691         int rc;
1692
1693         switch (srq_attr_mask) {
1694         case IB_SRQ_MAX_WR:
1695                 /* SRQ resize is not supported */
1696                 break;
1697         case IB_SRQ_LIMIT:
1698                 /* Change the SRQ threshold */
1699                 if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
1700                         return -EINVAL;
1701
1702                 srq->qplib_srq.threshold = srq_attr->srq_limit;
1703                 rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
1704                 if (rc) {
1705                         ibdev_err(&rdev->ibdev, "Modify HW SRQ failed!");
1706                         return rc;
1707                 }
1708                 /* On success, update the shadow */
1709                 srq->srq_limit = srq_attr->srq_limit;
1710                 /* No need to Build and send response back to udata */
1711                 break;
1712         default:
1713                 ibdev_err(&rdev->ibdev,
1714                           "Unsupported srq_attr_mask 0x%x", srq_attr_mask);
1715                 return -EINVAL;
1716         }
1717         return 0;
1718 }
1719
1720 int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
1721 {
1722         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1723                                                ib_srq);
1724         struct bnxt_re_srq tsrq;
1725         struct bnxt_re_dev *rdev = srq->rdev;
1726         int rc;
1727
1728         /* Get live SRQ attr */
1729         tsrq.qplib_srq.id = srq->qplib_srq.id;
1730         rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq);
1731         if (rc) {
1732                 ibdev_err(&rdev->ibdev, "Query HW SRQ failed!");
1733                 return rc;
1734         }
1735         srq_attr->max_wr = srq->qplib_srq.max_wqe;
1736         srq_attr->max_sge = srq->qplib_srq.max_sge;
1737         srq_attr->srq_limit = tsrq.qplib_srq.threshold;
1738
1739         return 0;
1740 }
1741
1742 int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
1743                           const struct ib_recv_wr **bad_wr)
1744 {
1745         struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1746                                                ib_srq);
1747         struct bnxt_qplib_swqe wqe;
1748         unsigned long flags;
1749         int rc = 0;
1750
1751         spin_lock_irqsave(&srq->lock, flags);
1752         while (wr) {
1753                 /* Transcribe each ib_recv_wr to qplib_swqe */
1754                 wqe.num_sge = wr->num_sge;
1755                 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
1756                 wqe.wr_id = wr->wr_id;
1757                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
1758
1759                 rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
1760                 if (rc) {
1761                         *bad_wr = wr;
1762                         break;
1763                 }
1764                 wr = wr->next;
1765         }
1766         spin_unlock_irqrestore(&srq->lock, flags);
1767
1768         return rc;
1769 }
1770 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
1771                                     struct bnxt_re_qp *qp1_qp,
1772                                     int qp_attr_mask)
1773 {
1774         struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
1775         int rc = 0;
1776
1777         if (qp_attr_mask & IB_QP_STATE) {
1778                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1779                 qp->qplib_qp.state = qp1_qp->qplib_qp.state;
1780         }
1781         if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1782                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1783                 qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
1784         }
1785
1786         if (qp_attr_mask & IB_QP_QKEY) {
1787                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1788                 /* Using a Random  QKEY */
1789                 qp->qplib_qp.qkey = 0x81818181;
1790         }
1791         if (qp_attr_mask & IB_QP_SQ_PSN) {
1792                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1793                 qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
1794         }
1795
1796         rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1797         if (rc)
1798                 ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1");
1799         return rc;
1800 }
1801
1802 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1803                       int qp_attr_mask, struct ib_udata *udata)
1804 {
1805         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1806         struct bnxt_re_dev *rdev = qp->rdev;
1807         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1808         enum ib_qp_state curr_qp_state, new_qp_state;
1809         int rc, entries;
1810         unsigned int flags;
1811         u8 nw_type;
1812
1813         if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1814                 return -EOPNOTSUPP;
1815
1816         qp->qplib_qp.modify_flags = 0;
1817         if (qp_attr_mask & IB_QP_STATE) {
1818                 curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
1819                 new_qp_state = qp_attr->qp_state;
1820                 if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
1821                                         ib_qp->qp_type, qp_attr_mask)) {
1822                         ibdev_err(&rdev->ibdev,
1823                                   "Invalid attribute mask: %#x specified ",
1824                                   qp_attr_mask);
1825                         ibdev_err(&rdev->ibdev,
1826                                   "for qpn: %#x type: %#x",
1827                                   ib_qp->qp_num, ib_qp->qp_type);
1828                         ibdev_err(&rdev->ibdev,
1829                                   "curr_qp_state=0x%x, new_qp_state=0x%x\n",
1830                                   curr_qp_state, new_qp_state);
1831                         return -EINVAL;
1832                 }
1833                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1834                 qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
1835
1836                 if (!qp->sumem &&
1837                     qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1838                         ibdev_dbg(&rdev->ibdev,
1839                                   "Move QP = %p to flush list\n", qp);
1840                         flags = bnxt_re_lock_cqs(qp);
1841                         bnxt_qplib_add_flush_qp(&qp->qplib_qp);
1842                         bnxt_re_unlock_cqs(qp, flags);
1843                 }
1844                 if (!qp->sumem &&
1845                     qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
1846                         ibdev_dbg(&rdev->ibdev,
1847                                   "Move QP = %p out of flush list\n", qp);
1848                         flags = bnxt_re_lock_cqs(qp);
1849                         bnxt_qplib_clean_qp(&qp->qplib_qp);
1850                         bnxt_re_unlock_cqs(qp, flags);
1851                 }
1852         }
1853         if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
1854                 qp->qplib_qp.modify_flags |=
1855                                 CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
1856                 qp->qplib_qp.en_sqd_async_notify = true;
1857         }
1858         if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
1859                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
1860                 qp->qplib_qp.access =
1861                         __from_ib_access_flags(qp_attr->qp_access_flags);
1862                 /* LOCAL_WRITE access must be set to allow RC receive */
1863                 qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1864                 /* Temp: Set all params on QP as of now */
1865                 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
1866                 qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
1867         }
1868         if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1869                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1870                 qp->qplib_qp.pkey_index = qp_attr->pkey_index;
1871         }
1872         if (qp_attr_mask & IB_QP_QKEY) {
1873                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1874                 qp->qplib_qp.qkey = qp_attr->qkey;
1875         }
1876         if (qp_attr_mask & IB_QP_AV) {
1877                 const struct ib_global_route *grh =
1878                         rdma_ah_read_grh(&qp_attr->ah_attr);
1879                 const struct ib_gid_attr *sgid_attr;
1880                 struct bnxt_re_gid_ctx *ctx;
1881
1882                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1883                                      CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1884                                      CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1885                                      CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1886                                      CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1887                                      CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1888                                      CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1889                 memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
1890                        sizeof(qp->qplib_qp.ah.dgid.data));
1891                 qp->qplib_qp.ah.flow_label = grh->flow_label;
1892                 sgid_attr = grh->sgid_attr;
1893                 /* Get the HW context of the GID. The reference
1894                  * of GID table entry is already taken by the caller.
1895                  */
1896                 ctx = rdma_read_gid_hw_context(sgid_attr);
1897                 qp->qplib_qp.ah.sgid_index = ctx->idx;
1898                 qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
1899                 qp->qplib_qp.ah.hop_limit = grh->hop_limit;
1900                 qp->qplib_qp.ah.traffic_class = grh->traffic_class;
1901                 qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
1902                 ether_addr_copy(qp->qplib_qp.ah.dmac,
1903                                 qp_attr->ah_attr.roce.dmac);
1904
1905                 rc = rdma_read_gid_l2_fields(sgid_attr, NULL,
1906                                              &qp->qplib_qp.smac[0]);
1907                 if (rc)
1908                         return rc;
1909
1910                 nw_type = rdma_gid_attr_network_type(sgid_attr);
1911                 switch (nw_type) {
1912                 case RDMA_NETWORK_IPV4:
1913                         qp->qplib_qp.nw_type =
1914                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
1915                         break;
1916                 case RDMA_NETWORK_IPV6:
1917                         qp->qplib_qp.nw_type =
1918                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
1919                         break;
1920                 default:
1921                         qp->qplib_qp.nw_type =
1922                                 CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
1923                         break;
1924                 }
1925         }
1926
1927         if (qp_attr_mask & IB_QP_PATH_MTU) {
1928                 qp->qplib_qp.modify_flags |=
1929                                 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1930                 qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu);
1931                 qp->qplib_qp.mtu = ib_mtu_enum_to_int(qp_attr->path_mtu);
1932         } else if (qp_attr->qp_state == IB_QPS_RTR) {
1933                 qp->qplib_qp.modify_flags |=
1934                         CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1935                 qp->qplib_qp.path_mtu =
1936                         __from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu));
1937                 qp->qplib_qp.mtu =
1938                         ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1939         }
1940
1941         if (qp_attr_mask & IB_QP_TIMEOUT) {
1942                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
1943                 qp->qplib_qp.timeout = qp_attr->timeout;
1944         }
1945         if (qp_attr_mask & IB_QP_RETRY_CNT) {
1946                 qp->qplib_qp.modify_flags |=
1947                                 CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
1948                 qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
1949         }
1950         if (qp_attr_mask & IB_QP_RNR_RETRY) {
1951                 qp->qplib_qp.modify_flags |=
1952                                 CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
1953                 qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
1954         }
1955         if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
1956                 qp->qplib_qp.modify_flags |=
1957                                 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
1958                 qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
1959         }
1960         if (qp_attr_mask & IB_QP_RQ_PSN) {
1961                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
1962                 qp->qplib_qp.rq.psn = qp_attr->rq_psn;
1963         }
1964         if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1965                 qp->qplib_qp.modify_flags |=
1966                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
1967                 /* Cap the max_rd_atomic to device max */
1968                 qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
1969                                                    dev_attr->max_qp_rd_atom);
1970         }
1971         if (qp_attr_mask & IB_QP_SQ_PSN) {
1972                 qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1973                 qp->qplib_qp.sq.psn = qp_attr->sq_psn;
1974         }
1975         if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1976                 if (qp_attr->max_dest_rd_atomic >
1977                     dev_attr->max_qp_init_rd_atom) {
1978                         ibdev_err(&rdev->ibdev,
1979                                   "max_dest_rd_atomic requested%d is > dev_max%d",
1980                                   qp_attr->max_dest_rd_atomic,
1981                                   dev_attr->max_qp_init_rd_atom);
1982                         return -EINVAL;
1983                 }
1984
1985                 qp->qplib_qp.modify_flags |=
1986                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
1987                 qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
1988         }
1989         if (qp_attr_mask & IB_QP_CAP) {
1990                 qp->qplib_qp.modify_flags |=
1991                                 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
1992                                 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
1993                                 CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
1994                                 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
1995                                 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
1996                 if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
1997                     (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
1998                     (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
1999                     (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
2000                     (qp_attr->cap.max_inline_data >=
2001                                                 dev_attr->max_inline_data)) {
2002                         ibdev_err(&rdev->ibdev,
2003                                   "Create QP failed - max exceeded");
2004                         return -EINVAL;
2005                 }
2006                 entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
2007                 qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
2008                                                 dev_attr->max_qp_wqes + 1);
2009                 qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
2010                                                 qp_attr->cap.max_send_wr;
2011                 /*
2012                  * Reserving one slot for Phantom WQE. Some application can
2013                  * post one extra entry in this case. Allowing this to avoid
2014                  * unexpected Queue full condition
2015                  */
2016                 qp->qplib_qp.sq.q_full_delta -= 1;
2017                 qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
2018                 if (qp->qplib_qp.rq.max_wqe) {
2019                         entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
2020                         qp->qplib_qp.rq.max_wqe =
2021                                 min_t(u32, entries, dev_attr->max_qp_wqes + 1);
2022                         qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
2023                                                        qp_attr->cap.max_recv_wr;
2024                         qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
2025                 } else {
2026                         /* SRQ was used prior, just ignore the RQ caps */
2027                 }
2028         }
2029         if (qp_attr_mask & IB_QP_DEST_QPN) {
2030                 qp->qplib_qp.modify_flags |=
2031                                 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
2032                 qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
2033         }
2034         rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2035         if (rc) {
2036                 ibdev_err(&rdev->ibdev, "Failed to modify HW QP");
2037                 return rc;
2038         }
2039         if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
2040                 rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
2041         return rc;
2042 }
2043
2044 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2045                      int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2046 {
2047         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2048         struct bnxt_re_dev *rdev = qp->rdev;
2049         struct bnxt_qplib_qp *qplib_qp;
2050         int rc;
2051
2052         qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL);
2053         if (!qplib_qp)
2054                 return -ENOMEM;
2055
2056         qplib_qp->id = qp->qplib_qp.id;
2057         qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
2058
2059         rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
2060         if (rc) {
2061                 ibdev_err(&rdev->ibdev, "Failed to query HW QP");
2062                 goto out;
2063         }
2064         qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
2065         qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
2066         qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
2067         qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access);
2068         qp_attr->pkey_index = qplib_qp->pkey_index;
2069         qp_attr->qkey = qplib_qp->qkey;
2070         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2071         rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->ah.flow_label,
2072                         qplib_qp->ah.host_sgid_index,
2073                         qplib_qp->ah.hop_limit,
2074                         qplib_qp->ah.traffic_class);
2075         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data);
2076         rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl);
2077         ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac);
2078         qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
2079         qp_attr->timeout = qplib_qp->timeout;
2080         qp_attr->retry_cnt = qplib_qp->retry_cnt;
2081         qp_attr->rnr_retry = qplib_qp->rnr_retry;
2082         qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
2083         qp_attr->rq_psn = qplib_qp->rq.psn;
2084         qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
2085         qp_attr->sq_psn = qplib_qp->sq.psn;
2086         qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
2087         qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
2088                                                          IB_SIGNAL_REQ_WR;
2089         qp_attr->dest_qp_num = qplib_qp->dest_qpn;
2090
2091         qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
2092         qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
2093         qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
2094         qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
2095         qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
2096         qp_init_attr->cap = qp_attr->cap;
2097
2098 out:
2099         kfree(qplib_qp);
2100         return rc;
2101 }
2102
2103 /* Routine for sending QP1 packets for RoCE V1 an V2
2104  */
2105 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
2106                                      const struct ib_send_wr *wr,
2107                                      struct bnxt_qplib_swqe *wqe,
2108                                      int payload_size)
2109 {
2110         struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
2111                                              ib_ah);
2112         struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
2113         const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr;
2114         struct bnxt_qplib_sge sge;
2115         u8 nw_type;
2116         u16 ether_type;
2117         union ib_gid dgid;
2118         bool is_eth = false;
2119         bool is_vlan = false;
2120         bool is_grh = false;
2121         bool is_udp = false;
2122         u8 ip_version = 0;
2123         u16 vlan_id = 0xFFFF;
2124         void *buf;
2125         int i, rc = 0;
2126
2127         memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
2128
2129         rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL);
2130         if (rc)
2131                 return rc;
2132
2133         /* Get network header type for this GID */
2134         nw_type = rdma_gid_attr_network_type(sgid_attr);
2135         switch (nw_type) {
2136         case RDMA_NETWORK_IPV4:
2137                 nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
2138                 break;
2139         case RDMA_NETWORK_IPV6:
2140                 nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
2141                 break;
2142         default:
2143                 nw_type = BNXT_RE_ROCE_V1_PACKET;
2144                 break;
2145         }
2146         memcpy(&dgid.raw, &qplib_ah->dgid, 16);
2147         is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2148         if (is_udp) {
2149                 if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) {
2150                         ip_version = 4;
2151                         ether_type = ETH_P_IP;
2152                 } else {
2153                         ip_version = 6;
2154                         ether_type = ETH_P_IPV6;
2155                 }
2156                 is_grh = false;
2157         } else {
2158                 ether_type = ETH_P_IBOE;
2159                 is_grh = true;
2160         }
2161
2162         is_eth = true;
2163         is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false;
2164
2165         ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
2166                           ip_version, is_udp, 0, &qp->qp1_hdr);
2167
2168         /* ETH */
2169         ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
2170         ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
2171
2172         /* For vlan, check the sgid for vlan existence */
2173
2174         if (!is_vlan) {
2175                 qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
2176         } else {
2177                 qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
2178                 qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
2179         }
2180
2181         if (is_grh || (ip_version == 6)) {
2182                 memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw,
2183                        sizeof(sgid_attr->gid));
2184                 memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
2185                        sizeof(sgid_attr->gid));
2186                 qp->qp1_hdr.grh.hop_limit     = qplib_ah->hop_limit;
2187         }
2188
2189         if (ip_version == 4) {
2190                 qp->qp1_hdr.ip4.tos = 0;
2191                 qp->qp1_hdr.ip4.id = 0;
2192                 qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
2193                 qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
2194
2195                 memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4);
2196                 memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
2197                 qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
2198         }
2199
2200         if (is_udp) {
2201                 qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
2202                 qp->qp1_hdr.udp.sport = htons(0x8CD1);
2203                 qp->qp1_hdr.udp.csum = 0;
2204         }
2205
2206         /* BTH */
2207         if (wr->opcode == IB_WR_SEND_WITH_IMM) {
2208                 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2209                 qp->qp1_hdr.immediate_present = 1;
2210         } else {
2211                 qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2212         }
2213         if (wr->send_flags & IB_SEND_SOLICITED)
2214                 qp->qp1_hdr.bth.solicited_event = 1;
2215         /* pad_count */
2216         qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
2217
2218         /* P_key for QP1 is for all members */
2219         qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
2220         qp->qp1_hdr.bth.destination_qpn = IB_QP1;
2221         qp->qp1_hdr.bth.ack_req = 0;
2222         qp->send_psn++;
2223         qp->send_psn &= BTH_PSN_MASK;
2224         qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
2225         /* DETH */
2226         /* Use the priviledged Q_Key for QP1 */
2227         qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
2228         qp->qp1_hdr.deth.source_qpn = IB_QP1;
2229
2230         /* Pack the QP1 to the transmit buffer */
2231         buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
2232         if (buf) {
2233                 ib_ud_header_pack(&qp->qp1_hdr, buf);
2234                 for (i = wqe->num_sge; i; i--) {
2235                         wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
2236                         wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
2237                         wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
2238                 }
2239
2240                 /*
2241                  * Max Header buf size for IPV6 RoCE V2 is 86,
2242                  * which is same as the QP1 SQ header buffer.
2243                  * Header buf size for IPV4 RoCE V2 can be 66.
2244                  * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
2245                  * Subtract 20 bytes from QP1 SQ header buf size
2246                  */
2247                 if (is_udp && ip_version == 4)
2248                         sge.size -= 20;
2249                 /*
2250                  * Max Header buf size for RoCE V1 is 78.
2251                  * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
2252                  * Subtract 8 bytes from QP1 SQ header buf size
2253                  */
2254                 if (!is_udp)
2255                         sge.size -= 8;
2256
2257                 /* Subtract 4 bytes for non vlan packets */
2258                 if (!is_vlan)
2259                         sge.size -= 4;
2260
2261                 wqe->sg_list[0].addr = sge.addr;
2262                 wqe->sg_list[0].lkey = sge.lkey;
2263                 wqe->sg_list[0].size = sge.size;
2264                 wqe->num_sge++;
2265
2266         } else {
2267                 ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!");
2268                 rc = -ENOMEM;
2269         }
2270         return rc;
2271 }
2272
2273 /* For the MAD layer, it only provides the recv SGE the size of
2274  * ib_grh + MAD datagram.  No Ethernet headers, Ethertype, BTH, DETH,
2275  * nor RoCE iCRC.  The Cu+ solution must provide buffer for the entire
2276  * receive packet (334 bytes) with no VLAN and then copy the GRH
2277  * and the MAD datagram out to the provided SGE.
2278  */
2279 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
2280                                             const struct ib_recv_wr *wr,
2281                                             struct bnxt_qplib_swqe *wqe,
2282                                             int payload_size)
2283 {
2284         struct bnxt_re_sqp_entries *sqp_entry;
2285         struct bnxt_qplib_sge ref, sge;
2286         struct bnxt_re_dev *rdev;
2287         u32 rq_prod_index;
2288
2289         rdev = qp->rdev;
2290
2291         rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
2292
2293         if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
2294                 return -ENOMEM;
2295
2296         /* Create 1 SGE to receive the entire
2297          * ethernet packet
2298          */
2299         /* Save the reference from ULP */
2300         ref.addr = wqe->sg_list[0].addr;
2301         ref.lkey = wqe->sg_list[0].lkey;
2302         ref.size = wqe->sg_list[0].size;
2303
2304         sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
2305
2306         /* SGE 1 */
2307         wqe->sg_list[0].addr = sge.addr;
2308         wqe->sg_list[0].lkey = sge.lkey;
2309         wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
2310         sge.size -= wqe->sg_list[0].size;
2311
2312         sqp_entry->sge.addr = ref.addr;
2313         sqp_entry->sge.lkey = ref.lkey;
2314         sqp_entry->sge.size = ref.size;
2315         /* Store the wrid for reporting completion */
2316         sqp_entry->wrid = wqe->wr_id;
2317         /* change the wqe->wrid to table index */
2318         wqe->wr_id = rq_prod_index;
2319         return 0;
2320 }
2321
2322 static int is_ud_qp(struct bnxt_re_qp *qp)
2323 {
2324         return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
2325                 qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
2326 }
2327
2328 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
2329                                   const struct ib_send_wr *wr,
2330                                   struct bnxt_qplib_swqe *wqe)
2331 {
2332         struct bnxt_re_ah *ah = NULL;
2333
2334         if (is_ud_qp(qp)) {
2335                 ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
2336                 wqe->send.q_key = ud_wr(wr)->remote_qkey;
2337                 wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
2338                 wqe->send.avid = ah->qplib_ah.id;
2339         }
2340         switch (wr->opcode) {
2341         case IB_WR_SEND:
2342                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
2343                 break;
2344         case IB_WR_SEND_WITH_IMM:
2345                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
2346                 wqe->send.imm_data = wr->ex.imm_data;
2347                 break;
2348         case IB_WR_SEND_WITH_INV:
2349                 wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
2350                 wqe->send.inv_key = wr->ex.invalidate_rkey;
2351                 break;
2352         default:
2353                 return -EINVAL;
2354         }
2355         if (wr->send_flags & IB_SEND_SIGNALED)
2356                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2357         if (wr->send_flags & IB_SEND_FENCE)
2358                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2359         if (wr->send_flags & IB_SEND_SOLICITED)
2360                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2361         if (wr->send_flags & IB_SEND_INLINE)
2362                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2363
2364         return 0;
2365 }
2366
2367 static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
2368                                   struct bnxt_qplib_swqe *wqe)
2369 {
2370         switch (wr->opcode) {
2371         case IB_WR_RDMA_WRITE:
2372                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
2373                 break;
2374         case IB_WR_RDMA_WRITE_WITH_IMM:
2375                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
2376                 wqe->rdma.imm_data = wr->ex.imm_data;
2377                 break;
2378         case IB_WR_RDMA_READ:
2379                 wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
2380                 wqe->rdma.inv_key = wr->ex.invalidate_rkey;
2381                 break;
2382         default:
2383                 return -EINVAL;
2384         }
2385         wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
2386         wqe->rdma.r_key = rdma_wr(wr)->rkey;
2387         if (wr->send_flags & IB_SEND_SIGNALED)
2388                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2389         if (wr->send_flags & IB_SEND_FENCE)
2390                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2391         if (wr->send_flags & IB_SEND_SOLICITED)
2392                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2393         if (wr->send_flags & IB_SEND_INLINE)
2394                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2395
2396         return 0;
2397 }
2398
2399 static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
2400                                     struct bnxt_qplib_swqe *wqe)
2401 {
2402         switch (wr->opcode) {
2403         case IB_WR_ATOMIC_CMP_AND_SWP:
2404                 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
2405                 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2406                 wqe->atomic.swap_data = atomic_wr(wr)->swap;
2407                 break;
2408         case IB_WR_ATOMIC_FETCH_AND_ADD:
2409                 wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
2410                 wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2411                 break;
2412         default:
2413                 return -EINVAL;
2414         }
2415         wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
2416         wqe->atomic.r_key = atomic_wr(wr)->rkey;
2417         if (wr->send_flags & IB_SEND_SIGNALED)
2418                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2419         if (wr->send_flags & IB_SEND_FENCE)
2420                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2421         if (wr->send_flags & IB_SEND_SOLICITED)
2422                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2423         return 0;
2424 }
2425
2426 static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
2427                                  struct bnxt_qplib_swqe *wqe)
2428 {
2429         wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
2430         wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
2431
2432         /* Need unconditional fence for local invalidate
2433          * opcode to work as expected.
2434          */
2435         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2436
2437         if (wr->send_flags & IB_SEND_SIGNALED)
2438                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2439         if (wr->send_flags & IB_SEND_SOLICITED)
2440                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2441
2442         return 0;
2443 }
2444
2445 static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
2446                                  struct bnxt_qplib_swqe *wqe)
2447 {
2448         struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
2449         struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
2450         int access = wr->access;
2451
2452         wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
2453         wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
2454         wqe->frmr.page_list = mr->pages;
2455         wqe->frmr.page_list_len = mr->npages;
2456         wqe->frmr.levels = qplib_frpl->hwq.level;
2457         wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
2458
2459         /* Need unconditional fence for reg_mr
2460          * opcode to function as expected.
2461          */
2462
2463         wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2464
2465         if (wr->wr.send_flags & IB_SEND_SIGNALED)
2466                 wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2467
2468         if (access & IB_ACCESS_LOCAL_WRITE)
2469                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2470         if (access & IB_ACCESS_REMOTE_READ)
2471                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
2472         if (access & IB_ACCESS_REMOTE_WRITE)
2473                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
2474         if (access & IB_ACCESS_REMOTE_ATOMIC)
2475                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
2476         if (access & IB_ACCESS_MW_BIND)
2477                 wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
2478
2479         wqe->frmr.l_key = wr->key;
2480         wqe->frmr.length = wr->mr->length;
2481         wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1;
2482         wqe->frmr.va = wr->mr->iova;
2483         return 0;
2484 }
2485
2486 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
2487                                     const struct ib_send_wr *wr,
2488                                     struct bnxt_qplib_swqe *wqe)
2489 {
2490         /*  Copy the inline data to the data  field */
2491         u8 *in_data;
2492         u32 i, sge_len;
2493         void *sge_addr;
2494
2495         in_data = wqe->inline_data;
2496         for (i = 0; i < wr->num_sge; i++) {
2497                 sge_addr = (void *)(unsigned long)
2498                                 wr->sg_list[i].addr;
2499                 sge_len = wr->sg_list[i].length;
2500
2501                 if ((sge_len + wqe->inline_len) >
2502                     BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
2503                         ibdev_err(&rdev->ibdev,
2504                                   "Inline data size requested > supported value");
2505                         return -EINVAL;
2506                 }
2507                 sge_len = wr->sg_list[i].length;
2508
2509                 memcpy(in_data, sge_addr, sge_len);
2510                 in_data += wr->sg_list[i].length;
2511                 wqe->inline_len += wr->sg_list[i].length;
2512         }
2513         return wqe->inline_len;
2514 }
2515
2516 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
2517                                    const struct ib_send_wr *wr,
2518                                    struct bnxt_qplib_swqe *wqe)
2519 {
2520         int payload_sz = 0;
2521
2522         if (wr->send_flags & IB_SEND_INLINE)
2523                 payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
2524         else
2525                 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
2526                                                wqe->num_sge);
2527
2528         return payload_sz;
2529 }
2530
2531 static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
2532 {
2533         if ((qp->ib_qp.qp_type == IB_QPT_UD ||
2534              qp->ib_qp.qp_type == IB_QPT_GSI ||
2535              qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
2536              qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
2537                 int qp_attr_mask;
2538                 struct ib_qp_attr qp_attr;
2539
2540                 qp_attr_mask = IB_QP_STATE;
2541                 qp_attr.qp_state = IB_QPS_RTS;
2542                 bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
2543                 qp->qplib_qp.wqe_cnt = 0;
2544         }
2545 }
2546
2547 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
2548                                        struct bnxt_re_qp *qp,
2549                                        const struct ib_send_wr *wr)
2550 {
2551         int rc = 0, payload_sz = 0;
2552         unsigned long flags;
2553
2554         spin_lock_irqsave(&qp->sq_lock, flags);
2555         while (wr) {
2556                 struct bnxt_qplib_swqe wqe = {};
2557
2558                 /* Common */
2559                 wqe.num_sge = wr->num_sge;
2560                 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2561                         ibdev_err(&rdev->ibdev,
2562                                   "Limit exceeded for Send SGEs");
2563                         rc = -EINVAL;
2564                         goto bad;
2565                 }
2566
2567                 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2568                 if (payload_sz < 0) {
2569                         rc = -EINVAL;
2570                         goto bad;
2571                 }
2572                 wqe.wr_id = wr->wr_id;
2573
2574                 wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
2575
2576                 rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2577                 if (!rc)
2578                         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2579 bad:
2580                 if (rc) {
2581                         ibdev_err(&rdev->ibdev,
2582                                   "Post send failed opcode = %#x rc = %d",
2583                                   wr->opcode, rc);
2584                         break;
2585                 }
2586                 wr = wr->next;
2587         }
2588         bnxt_qplib_post_send_db(&qp->qplib_qp);
2589         bnxt_ud_qp_hw_stall_workaround(qp);
2590         spin_unlock_irqrestore(&qp->sq_lock, flags);
2591         return rc;
2592 }
2593
2594 int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
2595                       const struct ib_send_wr **bad_wr)
2596 {
2597         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2598         struct bnxt_qplib_swqe wqe;
2599         int rc = 0, payload_sz = 0;
2600         unsigned long flags;
2601
2602         spin_lock_irqsave(&qp->sq_lock, flags);
2603         while (wr) {
2604                 /* House keeping */
2605                 memset(&wqe, 0, sizeof(wqe));
2606
2607                 /* Common */
2608                 wqe.num_sge = wr->num_sge;
2609                 if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2610                         ibdev_err(&qp->rdev->ibdev,
2611                                   "Limit exceeded for Send SGEs");
2612                         rc = -EINVAL;
2613                         goto bad;
2614                 }
2615
2616                 payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2617                 if (payload_sz < 0) {
2618                         rc = -EINVAL;
2619                         goto bad;
2620                 }
2621                 wqe.wr_id = wr->wr_id;
2622
2623                 switch (wr->opcode) {
2624                 case IB_WR_SEND:
2625                 case IB_WR_SEND_WITH_IMM:
2626                         if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) {
2627                                 rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
2628                                                                payload_sz);
2629                                 if (rc)
2630                                         goto bad;
2631                                 wqe.rawqp1.lflags |=
2632                                         SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
2633                         }
2634                         switch (wr->send_flags) {
2635                         case IB_SEND_IP_CSUM:
2636                                 wqe.rawqp1.lflags |=
2637                                         SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
2638                                 break;
2639                         default:
2640                                 break;
2641                         }
2642                         fallthrough;
2643                 case IB_WR_SEND_WITH_INV:
2644                         rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2645                         break;
2646                 case IB_WR_RDMA_WRITE:
2647                 case IB_WR_RDMA_WRITE_WITH_IMM:
2648                 case IB_WR_RDMA_READ:
2649                         rc = bnxt_re_build_rdma_wqe(wr, &wqe);
2650                         break;
2651                 case IB_WR_ATOMIC_CMP_AND_SWP:
2652                 case IB_WR_ATOMIC_FETCH_AND_ADD:
2653                         rc = bnxt_re_build_atomic_wqe(wr, &wqe);
2654                         break;
2655                 case IB_WR_RDMA_READ_WITH_INV:
2656                         ibdev_err(&qp->rdev->ibdev,
2657                                   "RDMA Read with Invalidate is not supported");
2658                         rc = -EINVAL;
2659                         goto bad;
2660                 case IB_WR_LOCAL_INV:
2661                         rc = bnxt_re_build_inv_wqe(wr, &wqe);
2662                         break;
2663                 case IB_WR_REG_MR:
2664                         rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2665                         break;
2666                 default:
2667                         /* Unsupported WRs */
2668                         ibdev_err(&qp->rdev->ibdev,
2669                                   "WR (%#x) is not supported", wr->opcode);
2670                         rc = -EINVAL;
2671                         goto bad;
2672                 }
2673                 if (!rc)
2674                         rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2675 bad:
2676                 if (rc) {
2677                         ibdev_err(&qp->rdev->ibdev,
2678                                   "post_send failed op:%#x qps = %#x rc = %d\n",
2679                                   wr->opcode, qp->qplib_qp.state, rc);
2680                         *bad_wr = wr;
2681                         break;
2682                 }
2683                 wr = wr->next;
2684         }
2685         bnxt_qplib_post_send_db(&qp->qplib_qp);
2686         bnxt_ud_qp_hw_stall_workaround(qp);
2687         spin_unlock_irqrestore(&qp->sq_lock, flags);
2688
2689         return rc;
2690 }
2691
2692 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2693                                        struct bnxt_re_qp *qp,
2694                                        const struct ib_recv_wr *wr)
2695 {
2696         struct bnxt_qplib_swqe wqe;
2697         int rc = 0;
2698
2699         memset(&wqe, 0, sizeof(wqe));
2700         while (wr) {
2701                 /* House keeping */
2702                 memset(&wqe, 0, sizeof(wqe));
2703
2704                 /* Common */
2705                 wqe.num_sge = wr->num_sge;
2706                 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2707                         ibdev_err(&rdev->ibdev,
2708                                   "Limit exceeded for Receive SGEs");
2709                         rc = -EINVAL;
2710                         break;
2711                 }
2712                 bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
2713                 wqe.wr_id = wr->wr_id;
2714                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2715
2716                 rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2717                 if (rc)
2718                         break;
2719
2720                 wr = wr->next;
2721         }
2722         if (!rc)
2723                 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2724         return rc;
2725 }
2726
2727 int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
2728                       const struct ib_recv_wr **bad_wr)
2729 {
2730         struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2731         struct bnxt_qplib_swqe wqe;
2732         int rc = 0, payload_sz = 0;
2733         unsigned long flags;
2734         u32 count = 0;
2735
2736         spin_lock_irqsave(&qp->rq_lock, flags);
2737         while (wr) {
2738                 /* House keeping */
2739                 memset(&wqe, 0, sizeof(wqe));
2740
2741                 /* Common */
2742                 wqe.num_sge = wr->num_sge;
2743                 if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2744                         ibdev_err(&qp->rdev->ibdev,
2745                                   "Limit exceeded for Receive SGEs");
2746                         rc = -EINVAL;
2747                         *bad_wr = wr;
2748                         break;
2749                 }
2750
2751                 payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2752                                                wr->num_sge);
2753                 wqe.wr_id = wr->wr_id;
2754                 wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2755
2756                 if (ib_qp->qp_type == IB_QPT_GSI &&
2757                     qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI)
2758                         rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
2759                                                               payload_sz);
2760                 if (!rc)
2761                         rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2762                 if (rc) {
2763                         *bad_wr = wr;
2764                         break;
2765                 }
2766
2767                 /* Ring DB if the RQEs posted reaches a threshold value */
2768                 if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
2769                         bnxt_qplib_post_recv_db(&qp->qplib_qp);
2770                         count = 0;
2771                 }
2772
2773                 wr = wr->next;
2774         }
2775
2776         if (count)
2777                 bnxt_qplib_post_recv_db(&qp->qplib_qp);
2778
2779         spin_unlock_irqrestore(&qp->rq_lock, flags);
2780
2781         return rc;
2782 }
2783
2784 /* Completion Queues */
2785 int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
2786 {
2787         struct bnxt_re_cq *cq;
2788         struct bnxt_qplib_nq *nq;
2789         struct bnxt_re_dev *rdev;
2790
2791         cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2792         rdev = cq->rdev;
2793         nq = cq->qplib_cq.nq;
2794
2795         bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2796         ib_umem_release(cq->umem);
2797
2798         atomic_dec(&rdev->cq_count);
2799         nq->budget--;
2800         kfree(cq->cql);
2801         return 0;
2802 }
2803
2804 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
2805                       struct ib_udata *udata)
2806 {
2807         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
2808         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
2809         struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
2810         int rc, entries;
2811         int cqe = attr->cqe;
2812         struct bnxt_qplib_nq *nq = NULL;
2813         unsigned int nq_alloc_cnt;
2814
2815         if (attr->flags)
2816                 return -EOPNOTSUPP;
2817
2818         /* Validate CQ fields */
2819         if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
2820                 ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded");
2821                 return -EINVAL;
2822         }
2823
2824         cq->rdev = rdev;
2825         cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
2826
2827         entries = roundup_pow_of_two(cqe + 1);
2828         if (entries > dev_attr->max_cq_wqes + 1)
2829                 entries = dev_attr->max_cq_wqes + 1;
2830
2831         cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
2832         cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
2833         if (udata) {
2834                 struct bnxt_re_cq_req req;
2835                 struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
2836                         udata, struct bnxt_re_ucontext, ib_uctx);
2837                 if (ib_copy_from_udata(&req, udata, sizeof(req))) {
2838                         rc = -EFAULT;
2839                         goto fail;
2840                 }
2841
2842                 cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
2843                                        entries * sizeof(struct cq_base),
2844                                        IB_ACCESS_LOCAL_WRITE);
2845                 if (IS_ERR(cq->umem)) {
2846                         rc = PTR_ERR(cq->umem);
2847                         goto fail;
2848                 }
2849                 cq->qplib_cq.sg_info.umem = cq->umem;
2850                 cq->qplib_cq.dpi = &uctx->dpi;
2851         } else {
2852                 cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
2853                 cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
2854                                   GFP_KERNEL);
2855                 if (!cq->cql) {
2856                         rc = -ENOMEM;
2857                         goto fail;
2858                 }
2859
2860                 cq->qplib_cq.dpi = &rdev->dpi_privileged;
2861         }
2862         /*
2863          * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
2864          * used for getting the NQ index.
2865          */
2866         nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt);
2867         nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)];
2868         cq->qplib_cq.max_wqe = entries;
2869         cq->qplib_cq.cnq_hw_ring_id = nq->ring_id;
2870         cq->qplib_cq.nq = nq;
2871
2872         rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
2873         if (rc) {
2874                 ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
2875                 goto fail;
2876         }
2877
2878         cq->ib_cq.cqe = entries;
2879         cq->cq_period = cq->qplib_cq.period;
2880         nq->budget++;
2881
2882         atomic_inc(&rdev->cq_count);
2883         spin_lock_init(&cq->cq_lock);
2884
2885         if (udata) {
2886                 struct bnxt_re_cq_resp resp;
2887
2888                 resp.cqid = cq->qplib_cq.id;
2889                 resp.tail = cq->qplib_cq.hwq.cons;
2890                 resp.phase = cq->qplib_cq.period;
2891                 resp.rsvd = 0;
2892                 rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
2893                 if (rc) {
2894                         ibdev_err(&rdev->ibdev, "Failed to copy CQ udata");
2895                         bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2896                         goto c2fail;
2897                 }
2898         }
2899
2900         return 0;
2901
2902 c2fail:
2903         ib_umem_release(cq->umem);
2904 fail:
2905         kfree(cq->cql);
2906         return rc;
2907 }
2908
2909 static u8 __req_to_ib_wc_status(u8 qstatus)
2910 {
2911         switch (qstatus) {
2912         case CQ_REQ_STATUS_OK:
2913                 return IB_WC_SUCCESS;
2914         case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
2915                 return IB_WC_BAD_RESP_ERR;
2916         case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
2917                 return IB_WC_LOC_LEN_ERR;
2918         case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
2919                 return IB_WC_LOC_QP_OP_ERR;
2920         case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
2921                 return IB_WC_LOC_PROT_ERR;
2922         case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
2923                 return IB_WC_GENERAL_ERR;
2924         case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
2925                 return IB_WC_REM_INV_REQ_ERR;
2926         case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
2927                 return IB_WC_REM_ACCESS_ERR;
2928         case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
2929                 return IB_WC_REM_OP_ERR;
2930         case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
2931                 return IB_WC_RNR_RETRY_EXC_ERR;
2932         case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
2933                 return IB_WC_RETRY_EXC_ERR;
2934         case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
2935                 return IB_WC_WR_FLUSH_ERR;
2936         default:
2937                 return IB_WC_GENERAL_ERR;
2938         }
2939         return 0;
2940 }
2941
2942 static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
2943 {
2944         switch (qstatus) {
2945         case CQ_RES_RAWETH_QP1_STATUS_OK:
2946                 return IB_WC_SUCCESS;
2947         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
2948                 return IB_WC_LOC_ACCESS_ERR;
2949         case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
2950                 return IB_WC_LOC_LEN_ERR;
2951         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
2952                 return IB_WC_LOC_PROT_ERR;
2953         case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
2954                 return IB_WC_LOC_QP_OP_ERR;
2955         case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
2956                 return IB_WC_GENERAL_ERR;
2957         case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
2958                 return IB_WC_WR_FLUSH_ERR;
2959         case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
2960                 return IB_WC_WR_FLUSH_ERR;
2961         default:
2962                 return IB_WC_GENERAL_ERR;
2963         }
2964 }
2965
2966 static u8 __rc_to_ib_wc_status(u8 qstatus)
2967 {
2968         switch (qstatus) {
2969         case CQ_RES_RC_STATUS_OK:
2970                 return IB_WC_SUCCESS;
2971         case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
2972                 return IB_WC_LOC_ACCESS_ERR;
2973         case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
2974                 return IB_WC_LOC_LEN_ERR;
2975         case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
2976                 return IB_WC_LOC_PROT_ERR;
2977         case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
2978                 return IB_WC_LOC_QP_OP_ERR;
2979         case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
2980                 return IB_WC_GENERAL_ERR;
2981         case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
2982                 return IB_WC_REM_INV_REQ_ERR;
2983         case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
2984                 return IB_WC_WR_FLUSH_ERR;
2985         case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
2986                 return IB_WC_WR_FLUSH_ERR;
2987         default:
2988                 return IB_WC_GENERAL_ERR;
2989         }
2990 }
2991
2992 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
2993 {
2994         switch (cqe->type) {
2995         case BNXT_QPLIB_SWQE_TYPE_SEND:
2996                 wc->opcode = IB_WC_SEND;
2997                 break;
2998         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
2999                 wc->opcode = IB_WC_SEND;
3000                 wc->wc_flags |= IB_WC_WITH_IMM;
3001                 break;
3002         case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
3003                 wc->opcode = IB_WC_SEND;
3004                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3005                 break;
3006         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
3007                 wc->opcode = IB_WC_RDMA_WRITE;
3008                 break;
3009         case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
3010                 wc->opcode = IB_WC_RDMA_WRITE;
3011                 wc->wc_flags |= IB_WC_WITH_IMM;
3012                 break;
3013         case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
3014                 wc->opcode = IB_WC_RDMA_READ;
3015                 break;
3016         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
3017                 wc->opcode = IB_WC_COMP_SWAP;
3018                 break;
3019         case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
3020                 wc->opcode = IB_WC_FETCH_ADD;
3021                 break;
3022         case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
3023                 wc->opcode = IB_WC_LOCAL_INV;
3024                 break;
3025         case BNXT_QPLIB_SWQE_TYPE_REG_MR:
3026                 wc->opcode = IB_WC_REG_MR;
3027                 break;
3028         default:
3029                 wc->opcode = IB_WC_SEND;
3030                 break;
3031         }
3032
3033         wc->status = __req_to_ib_wc_status(cqe->status);
3034 }
3035
3036 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
3037                                      u16 raweth_qp1_flags2)
3038 {
3039         bool is_ipv6 = false, is_ipv4 = false;
3040
3041         /* raweth_qp1_flags Bit 9-6 indicates itype */
3042         if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3043             != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3044                 return -1;
3045
3046         if (raweth_qp1_flags2 &
3047             CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
3048             raweth_qp1_flags2 &
3049             CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
3050                 /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
3051                 (raweth_qp1_flags2 &
3052                  CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
3053                         (is_ipv6 = true) : (is_ipv4 = true);
3054                 return ((is_ipv6) ?
3055                          BNXT_RE_ROCEV2_IPV6_PACKET :
3056                          BNXT_RE_ROCEV2_IPV4_PACKET);
3057         } else {
3058                 return BNXT_RE_ROCE_V1_PACKET;
3059         }
3060 }
3061
3062 static int bnxt_re_to_ib_nw_type(int nw_type)
3063 {
3064         u8 nw_hdr_type = 0xFF;
3065
3066         switch (nw_type) {
3067         case BNXT_RE_ROCE_V1_PACKET:
3068                 nw_hdr_type = RDMA_NETWORK_ROCE_V1;
3069                 break;
3070         case BNXT_RE_ROCEV2_IPV4_PACKET:
3071                 nw_hdr_type = RDMA_NETWORK_IPV4;
3072                 break;
3073         case BNXT_RE_ROCEV2_IPV6_PACKET:
3074                 nw_hdr_type = RDMA_NETWORK_IPV6;
3075                 break;
3076         }
3077         return nw_hdr_type;
3078 }
3079
3080 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
3081                                        void *rq_hdr_buf)
3082 {
3083         u8 *tmp_buf = NULL;
3084         struct ethhdr *eth_hdr;
3085         u16 eth_type;
3086         bool rc = false;
3087
3088         tmp_buf = (u8 *)rq_hdr_buf;
3089         /*
3090          * If dest mac is not same as I/F mac, this could be a
3091          * loopback address or multicast address, check whether
3092          * it is a loopback packet
3093          */
3094         if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
3095                 tmp_buf += 4;
3096                 /* Check the  ether type */
3097                 eth_hdr = (struct ethhdr *)tmp_buf;
3098                 eth_type = ntohs(eth_hdr->h_proto);
3099                 switch (eth_type) {
3100                 case ETH_P_IBOE:
3101                         rc = true;
3102                         break;
3103                 case ETH_P_IP:
3104                 case ETH_P_IPV6: {
3105                         u32 len;
3106                         struct udphdr *udp_hdr;
3107
3108                         len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
3109                                                       sizeof(struct ipv6hdr));
3110                         tmp_buf += sizeof(struct ethhdr) + len;
3111                         udp_hdr = (struct udphdr *)tmp_buf;
3112                         if (ntohs(udp_hdr->dest) ==
3113                                     ROCE_V2_UDP_DPORT)
3114                                 rc = true;
3115                         break;
3116                         }
3117                 default:
3118                         break;
3119                 }
3120         }
3121
3122         return rc;
3123 }
3124
3125 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp,
3126                                          struct bnxt_qplib_cqe *cqe)
3127 {
3128         struct bnxt_re_dev *rdev = gsi_qp->rdev;
3129         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3130         struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp;
3131         struct bnxt_re_ah *gsi_sah;
3132         struct ib_send_wr *swr;
3133         struct ib_ud_wr udwr;
3134         struct ib_recv_wr rwr;
3135         int pkt_type = 0;
3136         u32 tbl_idx;
3137         void *rq_hdr_buf;
3138         dma_addr_t rq_hdr_buf_map;
3139         dma_addr_t shrq_hdr_buf_map;
3140         u32 offset = 0;
3141         u32 skip_bytes = 0;
3142         struct ib_sge s_sge[2];
3143         struct ib_sge r_sge[2];
3144         int rc;
3145
3146         memset(&udwr, 0, sizeof(udwr));
3147         memset(&rwr, 0, sizeof(rwr));
3148         memset(&s_sge, 0, sizeof(s_sge));
3149         memset(&r_sge, 0, sizeof(r_sge));
3150
3151         swr = &udwr.wr;
3152         tbl_idx = cqe->wr_id;
3153
3154         rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf +
3155                         (tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size);
3156         rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3157                                                           tbl_idx);
3158
3159         /* Shadow QP header buffer */
3160         shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3161                                                             tbl_idx);
3162         sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3163
3164         /* Store this cqe */
3165         memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
3166         sqp_entry->qp1_qp = gsi_qp;
3167
3168         /* Find packet type from the cqe */
3169
3170         pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
3171                                              cqe->raweth_qp1_flags2);
3172         if (pkt_type < 0) {
3173                 ibdev_err(&rdev->ibdev, "Invalid packet\n");
3174                 return -EINVAL;
3175         }
3176
3177         /* Adjust the offset for the user buffer and post in the rq */
3178
3179         if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
3180                 offset = 20;
3181
3182         /*
3183          * QP1 loopback packet has 4 bytes of internal header before
3184          * ether header. Skip these four bytes.
3185          */
3186         if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
3187                 skip_bytes = 4;
3188
3189         /* First send SGE . Skip the ether header*/
3190         s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
3191                         + skip_bytes;
3192         s_sge[0].lkey = 0xFFFFFFFF;
3193         s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
3194                                 BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
3195
3196         /* Second Send SGE */
3197         s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
3198                         BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
3199         if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
3200                 s_sge[1].addr += 8;
3201         s_sge[1].lkey = 0xFFFFFFFF;
3202         s_sge[1].length = 256;
3203
3204         /* First recv SGE */
3205
3206         r_sge[0].addr = shrq_hdr_buf_map;
3207         r_sge[0].lkey = 0xFFFFFFFF;
3208         r_sge[0].length = 40;
3209
3210         r_sge[1].addr = sqp_entry->sge.addr + offset;
3211         r_sge[1].lkey = sqp_entry->sge.lkey;
3212         r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
3213
3214         /* Create receive work request */
3215         rwr.num_sge = 2;
3216         rwr.sg_list = r_sge;
3217         rwr.wr_id = tbl_idx;
3218         rwr.next = NULL;
3219
3220         rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr);
3221         if (rc) {
3222                 ibdev_err(&rdev->ibdev,
3223                           "Failed to post Rx buffers to shadow QP");
3224                 return -ENOMEM;
3225         }
3226
3227         swr->num_sge = 2;
3228         swr->sg_list = s_sge;
3229         swr->wr_id = tbl_idx;
3230         swr->opcode = IB_WR_SEND;
3231         swr->next = NULL;
3232         gsi_sah = rdev->gsi_ctx.gsi_sah;
3233         udwr.ah = &gsi_sah->ib_ah;
3234         udwr.remote_qpn = gsi_sqp->qplib_qp.id;
3235         udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
3236
3237         /* post data received  in the send queue */
3238         rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);
3239
3240         return 0;
3241 }
3242
3243 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
3244                                           struct bnxt_qplib_cqe *cqe)
3245 {
3246         wc->opcode = IB_WC_RECV;
3247         wc->status = __rawqp1_to_ib_wc_status(cqe->status);
3248         wc->wc_flags |= IB_WC_GRH;
3249 }
3250
3251 static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,
3252                                         u16 vlan_id)
3253 {
3254         /*
3255          * Check if the vlan is configured in the host.  If not configured, it
3256          * can be a transparent VLAN. So dont report the vlan id.
3257          */
3258         if (!__vlan_find_dev_deep_rcu(rdev->netdev,
3259                                       htons(ETH_P_8021Q), vlan_id))
3260                 return false;
3261         return true;
3262 }
3263
3264 static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe,
3265                                 u16 *vid, u8 *sl)
3266 {
3267         bool ret = false;
3268         u32 metadata;
3269         u16 tpid;
3270
3271         metadata = orig_cqe->raweth_qp1_metadata;
3272         if (orig_cqe->raweth_qp1_flags2 &
3273                 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
3274                 tpid = ((metadata &
3275                          CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
3276                          CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
3277                 if (tpid == ETH_P_8021Q) {
3278                         *vid = metadata &
3279                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
3280                         *sl = (metadata &
3281                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
3282                                CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
3283                         ret = true;
3284                 }
3285         }
3286
3287         return ret;
3288 }
3289
3290 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
3291                                       struct bnxt_qplib_cqe *cqe)
3292 {
3293         wc->opcode = IB_WC_RECV;
3294         wc->status = __rc_to_ib_wc_status(cqe->status);
3295
3296         if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3297                 wc->wc_flags |= IB_WC_WITH_IMM;
3298         if (cqe->flags & CQ_RES_RC_FLAGS_INV)
3299                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3300         if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
3301             (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
3302                 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3303 }
3304
3305 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
3306                                              struct ib_wc *wc,
3307                                              struct bnxt_qplib_cqe *cqe)
3308 {
3309         struct bnxt_re_dev *rdev = gsi_sqp->rdev;
3310         struct bnxt_re_qp *gsi_qp = NULL;
3311         struct bnxt_qplib_cqe *orig_cqe = NULL;
3312         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3313         int nw_type;
3314         u32 tbl_idx;
3315         u16 vlan_id;
3316         u8 sl;
3317
3318         tbl_idx = cqe->wr_id;
3319
3320         sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3321         gsi_qp = sqp_entry->qp1_qp;
3322         orig_cqe = &sqp_entry->cqe;
3323
3324         wc->wr_id = sqp_entry->wrid;
3325         wc->byte_len = orig_cqe->length;
3326         wc->qp = &gsi_qp->ib_qp;
3327
3328         wc->ex.imm_data = orig_cqe->immdata;
3329         wc->src_qp = orig_cqe->src_qp;
3330         memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
3331         if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) {
3332                 if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3333                         wc->vlan_id = vlan_id;
3334                         wc->sl = sl;
3335                         wc->wc_flags |= IB_WC_WITH_VLAN;
3336                 }
3337         }
3338         wc->port_num = 1;
3339         wc->vendor_err = orig_cqe->status;
3340
3341         wc->opcode = IB_WC_RECV;
3342         wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
3343         wc->wc_flags |= IB_WC_GRH;
3344
3345         nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
3346                                             orig_cqe->raweth_qp1_flags2);
3347         if (nw_type >= 0) {
3348                 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3349                 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3350         }
3351 }
3352
3353 static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,
3354                                       struct ib_wc *wc,
3355                                       struct bnxt_qplib_cqe *cqe)
3356 {
3357         u8 nw_type;
3358
3359         wc->opcode = IB_WC_RECV;
3360         wc->status = __rc_to_ib_wc_status(cqe->status);
3361
3362         if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
3363                 wc->wc_flags |= IB_WC_WITH_IMM;
3364         /* report only on GSI QP for Thor */
3365         if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) {
3366                 wc->wc_flags |= IB_WC_GRH;
3367                 memcpy(wc->smac, cqe->smac, ETH_ALEN);
3368                 wc->wc_flags |= IB_WC_WITH_SMAC;
3369                 if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
3370                         wc->vlan_id = (cqe->cfa_meta & 0xFFF);
3371                         if (wc->vlan_id < 0x1000)
3372                                 wc->wc_flags |= IB_WC_WITH_VLAN;
3373                 }
3374                 nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >>
3375                            CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT;
3376                 wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3377                 wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3378         }
3379
3380 }
3381
3382 static int send_phantom_wqe(struct bnxt_re_qp *qp)
3383 {
3384         struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
3385         unsigned long flags;
3386         int rc = 0;
3387
3388         spin_lock_irqsave(&qp->sq_lock, flags);
3389
3390         rc = bnxt_re_bind_fence_mw(lib_qp);
3391         if (!rc) {
3392                 lib_qp->sq.phantom_wqe_cnt++;
3393                 ibdev_dbg(&qp->rdev->ibdev,
3394                           "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
3395                           lib_qp->id, lib_qp->sq.hwq.prod,
3396                           HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
3397                           lib_qp->sq.phantom_wqe_cnt);
3398         }
3399
3400         spin_unlock_irqrestore(&qp->sq_lock, flags);
3401         return rc;
3402 }
3403
3404 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
3405 {
3406         struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3407         struct bnxt_re_qp *qp, *sh_qp;
3408         struct bnxt_qplib_cqe *cqe;
3409         int i, ncqe, budget;
3410         struct bnxt_qplib_q *sq;
3411         struct bnxt_qplib_qp *lib_qp;
3412         u32 tbl_idx;
3413         struct bnxt_re_sqp_entries *sqp_entry = NULL;
3414         unsigned long flags;
3415
3416         spin_lock_irqsave(&cq->cq_lock, flags);
3417         budget = min_t(u32, num_entries, cq->max_cql);
3418         num_entries = budget;
3419         if (!cq->cql) {
3420                 ibdev_err(&cq->rdev->ibdev, "POLL CQ : no CQL to use");
3421                 goto exit;
3422         }
3423         cqe = &cq->cql[0];
3424         while (budget) {
3425                 lib_qp = NULL;
3426                 ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
3427                 if (lib_qp) {
3428                         sq = &lib_qp->sq;
3429                         if (sq->send_phantom) {
3430                                 qp = container_of(lib_qp,
3431                                                   struct bnxt_re_qp, qplib_qp);
3432                                 if (send_phantom_wqe(qp) == -ENOMEM)
3433                                         ibdev_err(&cq->rdev->ibdev,
3434                                                   "Phantom failed! Scheduled to send again\n");
3435                                 else
3436                                         sq->send_phantom = false;
3437                         }
3438                 }
3439                 if (ncqe < budget)
3440                         ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
3441                                                               cqe + ncqe,
3442                                                               budget - ncqe);
3443
3444                 if (!ncqe)
3445                         break;
3446
3447                 for (i = 0; i < ncqe; i++, cqe++) {
3448                         /* Transcribe each qplib_wqe back to ib_wc */
3449                         memset(wc, 0, sizeof(*wc));
3450
3451                         wc->wr_id = cqe->wr_id;
3452                         wc->byte_len = cqe->length;
3453                         qp = container_of
3454                                 ((struct bnxt_qplib_qp *)
3455                                  (unsigned long)(cqe->qp_handle),
3456                                  struct bnxt_re_qp, qplib_qp);
3457                         wc->qp = &qp->ib_qp;
3458                         wc->ex.imm_data = cqe->immdata;
3459                         wc->src_qp = cqe->src_qp;
3460                         memcpy(wc->smac, cqe->smac, ETH_ALEN);
3461                         wc->port_num = 1;
3462                         wc->vendor_err = cqe->status;
3463
3464                         switch (cqe->opcode) {
3465                         case CQ_BASE_CQE_TYPE_REQ:
3466                                 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3467                                 if (sh_qp &&
3468                                     qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3469                                         /* Handle this completion with
3470                                          * the stored completion
3471                                          */
3472                                         memset(wc, 0, sizeof(*wc));
3473                                         continue;
3474                                 }
3475                                 bnxt_re_process_req_wc(wc, cqe);
3476                                 break;
3477                         case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
3478                                 if (!cqe->status) {
3479                                         int rc = 0;
3480
3481                                         rc = bnxt_re_process_raw_qp_pkt_rx
3482                                                                 (qp, cqe);
3483                                         if (!rc) {
3484                                                 memset(wc, 0, sizeof(*wc));
3485                                                 continue;
3486                                         }
3487                                         cqe->status = -1;
3488                                 }
3489                                 /* Errors need not be looped back.
3490                                  * But change the wr_id to the one
3491                                  * stored in the table
3492                                  */
3493                                 tbl_idx = cqe->wr_id;
3494                                 sqp_entry = &cq->rdev->gsi_ctx.sqp_tbl[tbl_idx];
3495                                 wc->wr_id = sqp_entry->wrid;
3496                                 bnxt_re_process_res_rawqp1_wc(wc, cqe);
3497                                 break;
3498                         case CQ_BASE_CQE_TYPE_RES_RC:
3499                                 bnxt_re_process_res_rc_wc(wc, cqe);
3500                                 break;
3501                         case CQ_BASE_CQE_TYPE_RES_UD:
3502                                 sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3503                                 if (sh_qp &&
3504                                     qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3505                                         /* Handle this completion with
3506                                          * the stored completion
3507                                          */
3508                                         if (cqe->status) {
3509                                                 continue;
3510                                         } else {
3511                                                 bnxt_re_process_res_shadow_qp_wc
3512                                                                 (qp, wc, cqe);
3513                                                 break;
3514                                         }
3515                                 }
3516                                 bnxt_re_process_res_ud_wc(qp, wc, cqe);
3517                                 break;
3518                         default:
3519                                 ibdev_err(&cq->rdev->ibdev,
3520                                           "POLL CQ : type 0x%x not handled",
3521                                           cqe->opcode);
3522                                 continue;
3523                         }
3524                         wc++;
3525                         budget--;
3526                 }
3527         }
3528 exit:
3529         spin_unlock_irqrestore(&cq->cq_lock, flags);
3530         return num_entries - budget;
3531 }
3532
3533 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
3534                           enum ib_cq_notify_flags ib_cqn_flags)
3535 {
3536         struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3537         int type = 0, rc = 0;
3538         unsigned long flags;
3539
3540         spin_lock_irqsave(&cq->cq_lock, flags);
3541         /* Trigger on the very next completion */
3542         if (ib_cqn_flags & IB_CQ_NEXT_COMP)
3543                 type = DBC_DBC_TYPE_CQ_ARMALL;
3544         /* Trigger on the next solicited completion */
3545         else if (ib_cqn_flags & IB_CQ_SOLICITED)
3546                 type = DBC_DBC_TYPE_CQ_ARMSE;
3547
3548         /* Poll to see if there are missed events */
3549         if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
3550             !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) {
3551                 rc = 1;
3552                 goto exit;
3553         }
3554         bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
3555
3556 exit:
3557         spin_unlock_irqrestore(&cq->cq_lock, flags);
3558         return rc;
3559 }
3560
3561 /* Memory Regions */
3562 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
3563 {
3564         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3565         struct bnxt_re_dev *rdev = pd->rdev;
3566         struct bnxt_re_mr *mr;
3567         int rc;
3568
3569         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3570         if (!mr)
3571                 return ERR_PTR(-ENOMEM);
3572
3573         mr->rdev = rdev;
3574         mr->qplib_mr.pd = &pd->qplib_pd;
3575         mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
3576         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
3577
3578         /* Allocate and register 0 as the address */
3579         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3580         if (rc)
3581                 goto fail;
3582
3583         mr->qplib_mr.hwq.level = PBL_LVL_MAX;
3584         mr->qplib_mr.total_size = -1; /* Infinte length */
3585         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0,
3586                                PAGE_SIZE);
3587         if (rc)
3588                 goto fail_mr;
3589
3590         mr->ib_mr.lkey = mr->qplib_mr.lkey;
3591         if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
3592                                IB_ACCESS_REMOTE_ATOMIC))
3593                 mr->ib_mr.rkey = mr->ib_mr.lkey;
3594         atomic_inc(&rdev->mr_count);
3595
3596         return &mr->ib_mr;
3597
3598 fail_mr:
3599         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3600 fail:
3601         kfree(mr);
3602         return ERR_PTR(rc);
3603 }
3604
3605 int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3606 {
3607         struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3608         struct bnxt_re_dev *rdev = mr->rdev;
3609         int rc;
3610
3611         rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3612         if (rc) {
3613                 ibdev_err(&rdev->ibdev, "Dereg MR failed: %#x\n", rc);
3614                 return rc;
3615         }
3616
3617         if (mr->pages) {
3618                 rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
3619                                                         &mr->qplib_frpl);
3620                 kfree(mr->pages);
3621                 mr->npages = 0;
3622                 mr->pages = NULL;
3623         }
3624         ib_umem_release(mr->ib_umem);
3625
3626         kfree(mr);
3627         atomic_dec(&rdev->mr_count);
3628         return rc;
3629 }
3630
3631 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
3632 {
3633         struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3634
3635         if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
3636                 return -ENOMEM;
3637
3638         mr->pages[mr->npages++] = addr;
3639         return 0;
3640 }
3641
3642 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
3643                       unsigned int *sg_offset)
3644 {
3645         struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
3646
3647         mr->npages = 0;
3648         return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page);
3649 }
3650
3651 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
3652                                u32 max_num_sg)
3653 {
3654         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3655         struct bnxt_re_dev *rdev = pd->rdev;
3656         struct bnxt_re_mr *mr = NULL;
3657         int rc;
3658
3659         if (type != IB_MR_TYPE_MEM_REG) {
3660                 ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported", type);
3661                 return ERR_PTR(-EINVAL);
3662         }
3663         if (max_num_sg > MAX_PBL_LVL_1_PGS)
3664                 return ERR_PTR(-EINVAL);
3665
3666         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3667         if (!mr)
3668                 return ERR_PTR(-ENOMEM);
3669
3670         mr->rdev = rdev;
3671         mr->qplib_mr.pd = &pd->qplib_pd;
3672         mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR;
3673         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
3674
3675         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3676         if (rc)
3677                 goto bail;
3678
3679         mr->ib_mr.lkey = mr->qplib_mr.lkey;
3680         mr->ib_mr.rkey = mr->ib_mr.lkey;
3681
3682         mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
3683         if (!mr->pages) {
3684                 rc = -ENOMEM;
3685                 goto fail;
3686         }
3687         rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
3688                                                  &mr->qplib_frpl, max_num_sg);
3689         if (rc) {
3690                 ibdev_err(&rdev->ibdev,
3691                           "Failed to allocate HW FR page list");
3692                 goto fail_mr;
3693         }
3694
3695         atomic_inc(&rdev->mr_count);
3696         return &mr->ib_mr;
3697
3698 fail_mr:
3699         kfree(mr->pages);
3700 fail:
3701         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3702 bail:
3703         kfree(mr);
3704         return ERR_PTR(rc);
3705 }
3706
3707 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
3708                                struct ib_udata *udata)
3709 {
3710         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3711         struct bnxt_re_dev *rdev = pd->rdev;
3712         struct bnxt_re_mw *mw;
3713         int rc;
3714
3715         mw = kzalloc(sizeof(*mw), GFP_KERNEL);
3716         if (!mw)
3717                 return ERR_PTR(-ENOMEM);
3718         mw->rdev = rdev;
3719         mw->qplib_mw.pd = &pd->qplib_pd;
3720
3721         mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
3722                                CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
3723                                CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
3724         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw);
3725         if (rc) {
3726                 ibdev_err(&rdev->ibdev, "Allocate MW failed!");
3727                 goto fail;
3728         }
3729         mw->ib_mw.rkey = mw->qplib_mw.rkey;
3730
3731         atomic_inc(&rdev->mw_count);
3732         return &mw->ib_mw;
3733
3734 fail:
3735         kfree(mw);
3736         return ERR_PTR(rc);
3737 }
3738
3739 int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
3740 {
3741         struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw);
3742         struct bnxt_re_dev *rdev = mw->rdev;
3743         int rc;
3744
3745         rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw);
3746         if (rc) {
3747                 ibdev_err(&rdev->ibdev, "Free MW failed: %#x\n", rc);
3748                 return rc;
3749         }
3750
3751         kfree(mw);
3752         atomic_dec(&rdev->mw_count);
3753         return rc;
3754 }
3755
3756 /* uverbs */
3757 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
3758                                   u64 virt_addr, int mr_access_flags,
3759                                   struct ib_udata *udata)
3760 {
3761         struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3762         struct bnxt_re_dev *rdev = pd->rdev;
3763         struct bnxt_re_mr *mr;
3764         struct ib_umem *umem;
3765         unsigned long page_size;
3766         int umem_pgs, rc;
3767
3768         if (length > BNXT_RE_MAX_MR_SIZE) {
3769                 ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n",
3770                           length, BNXT_RE_MAX_MR_SIZE);
3771                 return ERR_PTR(-ENOMEM);
3772         }
3773
3774         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3775         if (!mr)
3776                 return ERR_PTR(-ENOMEM);
3777
3778         mr->rdev = rdev;
3779         mr->qplib_mr.pd = &pd->qplib_pd;
3780         mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
3781         mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
3782
3783         rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3784         if (rc) {
3785                 ibdev_err(&rdev->ibdev, "Failed to allocate MR");
3786                 goto free_mr;
3787         }
3788         /* The fixed portion of the rkey is the same as the lkey */
3789         mr->ib_mr.rkey = mr->qplib_mr.rkey;
3790
3791         umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags);
3792         if (IS_ERR(umem)) {
3793                 ibdev_err(&rdev->ibdev, "Failed to get umem");
3794                 rc = -EFAULT;
3795                 goto free_mrw;
3796         }
3797         mr->ib_umem = umem;
3798
3799         mr->qplib_mr.va = virt_addr;
3800         page_size = ib_umem_find_best_pgsz(
3801                 umem, BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M, virt_addr);
3802         if (!page_size) {
3803                 ibdev_err(&rdev->ibdev, "umem page size unsupported!");
3804                 rc = -EFAULT;
3805                 goto free_umem;
3806         }
3807         mr->qplib_mr.total_size = length;
3808
3809         umem_pgs = ib_umem_num_dma_blocks(umem, page_size);
3810         rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem,
3811                                umem_pgs, page_size);
3812         if (rc) {
3813                 ibdev_err(&rdev->ibdev, "Failed to register user MR");
3814                 goto free_umem;
3815         }
3816
3817         mr->ib_mr.lkey = mr->qplib_mr.lkey;
3818         mr->ib_mr.rkey = mr->qplib_mr.lkey;
3819         atomic_inc(&rdev->mr_count);
3820
3821         return &mr->ib_mr;
3822 free_umem:
3823         ib_umem_release(umem);
3824 free_mrw:
3825         bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3826 free_mr:
3827         kfree(mr);
3828         return ERR_PTR(rc);
3829 }
3830
3831 int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
3832 {
3833         struct ib_device *ibdev = ctx->device;
3834         struct bnxt_re_ucontext *uctx =
3835                 container_of(ctx, struct bnxt_re_ucontext, ib_uctx);
3836         struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
3837         struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
3838         struct bnxt_re_uctx_resp resp = {};
3839         u32 chip_met_rev_num = 0;
3840         int rc;
3841
3842         ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver);
3843
3844         if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
3845                 ibdev_dbg(ibdev, " is different from the device %d ",
3846                           BNXT_RE_ABI_VERSION);
3847                 return -EPERM;
3848         }
3849
3850         uctx->rdev = rdev;
3851
3852         uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
3853         if (!uctx->shpg) {
3854                 rc = -ENOMEM;
3855                 goto fail;
3856         }
3857         spin_lock_init(&uctx->sh_lock);
3858
3859         resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX;
3860         chip_met_rev_num = rdev->chip_ctx->chip_num;
3861         chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) <<
3862                              BNXT_RE_CHIP_ID0_CHIP_REV_SFT;
3863         chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) <<
3864                              BNXT_RE_CHIP_ID0_CHIP_MET_SFT;
3865         resp.chip_id0 = chip_met_rev_num;
3866         /*Temp, Use xa_alloc instead */
3867         resp.dev_id = rdev->en_dev->pdev->devfn;
3868         resp.max_qp = rdev->qplib_ctx.qpc_count;
3869         resp.pg_size = PAGE_SIZE;
3870         resp.cqe_sz = sizeof(struct cq_base);
3871         resp.max_cqd = dev_attr->max_cq_wqes;
3872
3873         resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE;
3874         resp.mode = rdev->chip_ctx->modes.wqe_mode;
3875
3876         rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
3877         if (rc) {
3878                 ibdev_err(ibdev, "Failed to copy user context");
3879                 rc = -EFAULT;
3880                 goto cfail;
3881         }
3882
3883         return 0;
3884 cfail:
3885         free_page((unsigned long)uctx->shpg);
3886         uctx->shpg = NULL;
3887 fail:
3888         return rc;
3889 }
3890
3891 void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
3892 {
3893         struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3894                                                    struct bnxt_re_ucontext,
3895                                                    ib_uctx);
3896
3897         struct bnxt_re_dev *rdev = uctx->rdev;
3898
3899         if (uctx->shpg)
3900                 free_page((unsigned long)uctx->shpg);
3901
3902         if (uctx->dpi.dbr) {
3903                 /* Free DPI only if this is the first PD allocated by the
3904                  * application and mark the context dpi as NULL
3905                  */
3906                 bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
3907                                        &rdev->qplib_res.dpi_tbl, &uctx->dpi);
3908                 uctx->dpi.dbr = NULL;
3909         }
3910 }
3911
3912 /* Helper function to mmap the virtual memory from user app */
3913 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
3914 {
3915         struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3916                                                    struct bnxt_re_ucontext,
3917                                                    ib_uctx);
3918         struct bnxt_re_dev *rdev = uctx->rdev;
3919         u64 pfn;
3920
3921         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3922                 return -EINVAL;
3923
3924         if (vma->vm_pgoff) {
3925                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3926                 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
3927                                        PAGE_SIZE, vma->vm_page_prot)) {
3928                         ibdev_err(&rdev->ibdev, "Failed to map DPI");
3929                         return -EAGAIN;
3930                 }
3931         } else {
3932                 pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT;
3933                 if (remap_pfn_range(vma, vma->vm_start,
3934                                     pfn, PAGE_SIZE, vma->vm_page_prot)) {
3935                         ibdev_err(&rdev->ibdev, "Failed to map shared page");
3936                         return -EAGAIN;
3937                 }
3938         }
3939
3940         return 0;
3941 }