drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45 #include <rdma/uverbs_ioctl.h>
  46
  47 #include <linux/qed/common_hsi.h>
  48 #include "qedr_hsi_rdma.h"
  49 #include <linux/qed/qed_if.h>
  50 #include "qedr.h"
  51 #include "verbs.h"
  52 #include <rdma/qedr-abi.h>
  53 #include "qedr_roce_cm.h"
  54 #include "qedr_iw_cm.h"
  55
  56 #define QEDR_SRQ_WQE_ELEM_SIZE  sizeof(union rdma_srq_elm)
  57 #define RDMA_MAX_SGE_PER_SRQ    (4)
  58 #define RDMA_MAX_SRQ_WQE_SIZE   (RDMA_MAX_SGE_PER_SRQ + 1)
  59
  60 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  61
  62 enum {
  63         QEDR_USER_MMAP_IO_WC = 0,
  64         QEDR_USER_MMAP_PHYS_PAGE,
  65 };
  66
  67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  68                                         size_t len)
  69 {
  70         size_t min_len = min_t(size_t, len, udata->outlen);
  71
  72         return ib_copy_to_udata(udata, src, min_len);
  73 }
  74
  75 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  76 {
  77         if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  78                 return -EINVAL;
  79
  80         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  81         return 0;
  82 }
  83
  84 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
  85                       int index, union ib_gid *sgid)
  86 {
  87         struct qedr_dev *dev = get_qedr_dev(ibdev);
  88
  89         memset(sgid->raw, 0, sizeof(sgid->raw));
  90         ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  91
  92         DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  93                  sgid->global.interface_id, sgid->global.subnet_prefix);
  94
  95         return 0;
  96 }
  97
  98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  99 {
 100         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
 101         struct qedr_device_attr *qattr = &dev->attr;
 102         struct qedr_srq *srq = get_qedr_srq(ibsrq);
 103
 104         srq_attr->srq_limit = srq->srq_limit;
 105         srq_attr->max_wr = qattr->max_srq_wr;
 106         srq_attr->max_sge = qattr->max_sge;
 107
 108         return 0;
 109 }
 110
 111 int qedr_query_device(struct ib_device *ibdev,
 112                       struct ib_device_attr *attr, struct ib_udata *udata)
 113 {
 114         struct qedr_dev *dev = get_qedr_dev(ibdev);
 115         struct qedr_device_attr *qattr = &dev->attr;
 116
 117         if (!dev->rdma_ctx) {
 118                 DP_ERR(dev,
 119                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 120                        dev->rdma_ctx);
 121                 return -EINVAL;
 122         }
 123
 124         memset(attr, 0, sizeof(*attr));
 125
 126         attr->fw_ver = qattr->fw_ver;
 127         attr->sys_image_guid = qattr->sys_image_guid;
 128         attr->max_mr_size = qattr->max_mr_size;
 129         attr->page_size_cap = qattr->page_size_caps;
 130         attr->vendor_id = qattr->vendor_id;
 131         attr->vendor_part_id = qattr->vendor_part_id;
 132         attr->hw_ver = qattr->hw_ver;
 133         attr->max_qp = qattr->max_qp;
 134         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 135         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 136             IB_DEVICE_RC_RNR_NAK_GEN |
 137             IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
 138
 139         attr->max_send_sge = qattr->max_sge;
 140         attr->max_recv_sge = qattr->max_sge;
 141         attr->max_sge_rd = qattr->max_sge;
 142         attr->max_cq = qattr->max_cq;
 143         attr->max_cqe = qattr->max_cqe;
 144         attr->max_mr = qattr->max_mr;
 145         attr->max_mw = qattr->max_mw;
 146         attr->max_pd = qattr->max_pd;
 147         attr->atomic_cap = dev->atomic_cap;
 148         attr->max_qp_init_rd_atom =
 149             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 150         attr->max_qp_rd_atom =
 151             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 152                 attr->max_qp_init_rd_atom);
 153
 154         attr->max_srq = qattr->max_srq;
 155         attr->max_srq_sge = qattr->max_srq_sge;
 156         attr->max_srq_wr = qattr->max_srq_wr;
 157
 158         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 159         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 160         attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
 161         attr->max_ah = qattr->max_ah;
 162
 163         return 0;
 164 }
 165
 166 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
 167                                             u8 *ib_width)
 168 {
 169         switch (speed) {
 170         case 1000:
 171                 *ib_speed = IB_SPEED_SDR;
 172                 *ib_width = IB_WIDTH_1X;
 173                 break;
 174         case 10000:
 175                 *ib_speed = IB_SPEED_QDR;
 176                 *ib_width = IB_WIDTH_1X;
 177                 break;
 178
 179         case 20000:
 180                 *ib_speed = IB_SPEED_DDR;
 181                 *ib_width = IB_WIDTH_4X;
 182                 break;
 183
 184         case 25000:
 185                 *ib_speed = IB_SPEED_EDR;
 186                 *ib_width = IB_WIDTH_1X;
 187                 break;
 188
 189         case 40000:
 190                 *ib_speed = IB_SPEED_QDR;
 191                 *ib_width = IB_WIDTH_4X;
 192                 break;
 193
 194         case 50000:
 195                 *ib_speed = IB_SPEED_HDR;
 196                 *ib_width = IB_WIDTH_1X;
 197                 break;
 198
 199         case 100000:
 200                 *ib_speed = IB_SPEED_EDR;
 201                 *ib_width = IB_WIDTH_4X;
 202                 break;
 203
 204         default:
 205                 /* Unsupported */
 206                 *ib_speed = IB_SPEED_SDR;
 207                 *ib_width = IB_WIDTH_1X;
 208         }
 209 }
 210
 211 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
 212 {
 213         struct qedr_dev *dev;
 214         struct qed_rdma_port *rdma_port;
 215
 216         dev = get_qedr_dev(ibdev);
 217
 218         if (!dev->rdma_ctx) {
 219                 DP_ERR(dev, "rdma_ctx is NULL\n");
 220                 return -EINVAL;
 221         }
 222
 223         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 224
 225         /* *attr being zeroed by the caller, avoid zeroing it here */
 226         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 227                 attr->state = IB_PORT_ACTIVE;
 228                 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 229         } else {
 230                 attr->state = IB_PORT_DOWN;
 231                 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 232         }
 233         attr->max_mtu = IB_MTU_4096;
 234         attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 235         attr->lid = 0;
 236         attr->lmc = 0;
 237         attr->sm_lid = 0;
 238         attr->sm_sl = 0;
 239         attr->ip_gids = true;
 240         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 241                 attr->gid_tbl_len = 1;
 242         } else {
 243                 attr->gid_tbl_len = QEDR_MAX_SGID;
 244                 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 245         }
 246         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 247         attr->qkey_viol_cntr = 0;
 248         get_link_speed_and_width(rdma_port->link_speed,
 249                                  &attr->active_speed, &attr->active_width);
 250         attr->max_msg_sz = rdma_port->max_msg_size;
 251         attr->max_vl_num = 4;
 252
 253         return 0;
 254 }
 255
 256 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 257 {
 258         struct ib_device *ibdev = uctx->device;
 259         int rc;
 260         struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
 261         struct qedr_alloc_ucontext_resp uresp = {};
 262         struct qedr_alloc_ucontext_req ureq = {};
 263         struct qedr_dev *dev = get_qedr_dev(ibdev);
 264         struct qed_rdma_add_user_out_params oparams;
 265         struct qedr_user_mmap_entry *entry;
 266
 267         if (!udata)
 268                 return -EFAULT;
 269
 270         if (udata->inlen) {
 271                 rc = ib_copy_from_udata(&ureq, udata,
 272                                         min(sizeof(ureq), udata->inlen));
 273                 if (rc) {
 274                         DP_ERR(dev, "Problem copying data from user space\n");
 275                         return -EFAULT;
 276                 }
 277                 ctx->edpm_mode = !!(ureq.context_flags &
 278                                     QEDR_ALLOC_UCTX_EDPM_MODE);
 279                 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
 280         }
 281
 282         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 283         if (rc) {
 284                 DP_ERR(dev,
 285                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 286                        rc);
 287                 return rc;
 288         }
 289
 290         ctx->dpi = oparams.dpi;
 291         ctx->dpi_addr = oparams.dpi_addr;
 292         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 293         ctx->dpi_size = oparams.dpi_size;
 294         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 295         if (!entry) {
 296                 rc = -ENOMEM;
 297                 goto err;
 298         }
 299
 300         entry->io_address = ctx->dpi_phys_addr;
 301         entry->length = ctx->dpi_size;
 302         entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
 303         entry->dpi = ctx->dpi;
 304         entry->dev = dev;
 305         rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
 306                                          ctx->dpi_size);
 307         if (rc) {
 308                 kfree(entry);
 309                 goto err;
 310         }
 311         ctx->db_mmap_entry = &entry->rdma_entry;
 312
 313         if (!dev->user_dpm_enabled)
 314                 uresp.dpm_flags = 0;
 315         else if (rdma_protocol_iwarp(&dev->ibdev, 1))
 316                 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
 317         else
 318                 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
 319                                   QEDR_DPM_TYPE_ROCE_LEGACY |
 320                                   QEDR_DPM_TYPE_ROCE_EDPM_MODE;
 321
 322         if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
 323                 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
 324                 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
 325                 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
 326                 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
 327         }
 328
 329         uresp.wids_enabled = 1;
 330         uresp.wid_count = oparams.wid_count;
 331         uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
 332         uresp.db_size = ctx->dpi_size;
 333         uresp.max_send_wr = dev->attr.max_sqe;
 334         uresp.max_recv_wr = dev->attr.max_rqe;
 335         uresp.max_srq_wr = dev->attr.max_srq_wr;
 336         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 337         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 338         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 339         uresp.max_cqes = QEDR_MAX_CQES;
 340
 341         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 342         if (rc)
 343                 goto err;
 344
 345         ctx->dev = dev;
 346
 347         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 348                  &ctx->ibucontext);
 349         return 0;
 350
 351 err:
 352         if (!ctx->db_mmap_entry)
 353                 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
 354         else
 355                 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
 356
 357         return rc;
 358 }
 359
 360 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 361 {
 362         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 363
 364         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 365                  uctx);
 366
 367         rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
 368 }
 369
 370 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
 371 {
 372         struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
 373         struct qedr_dev *dev = entry->dev;
 374
 375         if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
 376                 free_page((unsigned long)entry->address);
 377         else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
 378                 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
 379
 380         kfree(entry);
 381 }
 382
 383 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
 384 {
 385         struct ib_device *dev = ucontext->device;
 386         size_t length = vma->vm_end - vma->vm_start;
 387         struct rdma_user_mmap_entry *rdma_entry;
 388         struct qedr_user_mmap_entry *entry;
 389         int rc = 0;
 390         u64 pfn;
 391
 392         ibdev_dbg(dev,
 393                   "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
 394                   vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
 395
 396         rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
 397         if (!rdma_entry) {
 398                 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
 399                           vma->vm_pgoff);
 400                 return -EINVAL;
 401         }
 402         entry = get_qedr_mmap_entry(rdma_entry);
 403         ibdev_dbg(dev,
 404                   "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
 405                   entry->io_address, length, entry->mmap_flag);
 406
 407         switch (entry->mmap_flag) {
 408         case QEDR_USER_MMAP_IO_WC:
 409                 pfn = entry->io_address >> PAGE_SHIFT;
 410                 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
 411                                        pgprot_writecombine(vma->vm_page_prot),
 412                                        rdma_entry);
 413                 break;
 414         case QEDR_USER_MMAP_PHYS_PAGE:
 415                 rc = vm_insert_page(vma, vma->vm_start,
 416                                     virt_to_page(entry->address));
 417                 break;
 418         default:
 419                 rc = -EINVAL;
 420         }
 421
 422         if (rc)
 423                 ibdev_dbg(dev,
 424                           "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 425                           entry->io_address, length, entry->mmap_flag, rc);
 426
 427         rdma_user_mmap_entry_put(rdma_entry);
 428         return rc;
 429 }
 430
 431 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 432 {
 433         struct ib_device *ibdev = ibpd->device;
 434         struct qedr_dev *dev = get_qedr_dev(ibdev);
 435         struct qedr_pd *pd = get_qedr_pd(ibpd);
 436         u16 pd_id;
 437         int rc;
 438
 439         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 440                  udata ? "User Lib" : "Kernel");
 441
 442         if (!dev->rdma_ctx) {
 443                 DP_ERR(dev, "invalid RDMA context\n");
 444                 return -EINVAL;
 445         }
 446
 447         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 448         if (rc)
 449                 return rc;
 450
 451         pd->pd_id = pd_id;
 452
 453         if (udata) {
 454                 struct qedr_alloc_pd_uresp uresp = {
 455                         .pd_id = pd_id,
 456                 };
 457                 struct qedr_ucontext *context = rdma_udata_to_drv_context(
 458                         udata, struct qedr_ucontext, ibucontext);
 459
 460                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 461                 if (rc) {
 462                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 463                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 464                         return rc;
 465                 }
 466
 467                 pd->uctx = context;
 468                 pd->uctx->pd = pd;
 469         }
 470
 471         return 0;
 472 }
 473
 474 void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 475 {
 476         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 477         struct qedr_pd *pd = get_qedr_pd(ibpd);
 478
 479         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 480         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 481 }
 482
 483 static void qedr_free_pbl(struct qedr_dev *dev,
 484                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 485 {
 486         struct pci_dev *pdev = dev->pdev;
 487         int i;
 488
 489         for (i = 0; i < pbl_info->num_pbls; i++) {
 490                 if (!pbl[i].va)
 491                         continue;
 492                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 493                                   pbl[i].va, pbl[i].pa);
 494         }
 495
 496         kfree(pbl);
 497 }
 498
 499 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 500 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 501
 502 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 503 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 504 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 505
 506 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 507                                            struct qedr_pbl_info *pbl_info,
 508                                            gfp_t flags)
 509 {
 510         struct pci_dev *pdev = dev->pdev;
 511         struct qedr_pbl *pbl_table;
 512         dma_addr_t *pbl_main_tbl;
 513         dma_addr_t pa;
 514         void *va;
 515         int i;
 516
 517         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 518         if (!pbl_table)
 519                 return ERR_PTR(-ENOMEM);
 520
 521         for (i = 0; i < pbl_info->num_pbls; i++) {
 522                 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
 523                                         flags);
 524                 if (!va)
 525                         goto err;
 526
 527                 pbl_table[i].va = va;
 528                 pbl_table[i].pa = pa;
 529         }
 530
 531         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 532          * the first one with physical pointers to all of the rest
 533          */
 534         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 535         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 536                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 537
 538         return pbl_table;
 539
 540 err:
 541         for (i--; i >= 0; i--)
 542                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 543                                   pbl_table[i].va, pbl_table[i].pa);
 544
 545         qedr_free_pbl(dev, pbl_info, pbl_table);
 546
 547         return ERR_PTR(-ENOMEM);
 548 }
 549
 550 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 551                                 struct qedr_pbl_info *pbl_info,
 552                                 u32 num_pbes, int two_layer_capable)
 553 {
 554         u32 pbl_capacity;
 555         u32 pbl_size;
 556         u32 num_pbls;
 557
 558         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 559                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 560                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 561                                num_pbes);
 562                         return -EINVAL;
 563                 }
 564
 565                 /* calculate required pbl page size */
 566                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 567                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 568                                NUM_PBES_ON_PAGE(pbl_size);
 569
 570                 while (pbl_capacity < num_pbes) {
 571                         pbl_size *= 2;
 572                         pbl_capacity = pbl_size / sizeof(u64);
 573                         pbl_capacity = pbl_capacity * pbl_capacity;
 574                 }
 575
 576                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 577                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 578                 pbl_info->two_layered = true;
 579         } else {
 580                 /* One layered PBL */
 581                 num_pbls = 1;
 582                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 583                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 584                 pbl_info->two_layered = false;
 585         }
 586
 587         pbl_info->num_pbls = num_pbls;
 588         pbl_info->pbl_size = pbl_size;
 589         pbl_info->num_pbes = num_pbes;
 590
 591         DP_DEBUG(dev, QEDR_MSG_MR,
 592                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 593                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 594
 595         return 0;
 596 }
 597
 598 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 599                                struct qedr_pbl *pbl,
 600                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 601 {
 602         int pbe_cnt, total_num_pbes = 0;
 603         u32 fw_pg_cnt, fw_pg_per_umem_pg;
 604         struct qedr_pbl *pbl_tbl;
 605         struct sg_dma_page_iter sg_iter;
 606         struct regpair *pbe;
 607         u64 pg_addr;
 608
 609         if (!pbl_info->num_pbes)
 610                 return;
 611
 612         /* If we have a two layered pbl, the first pbl points to the rest
 613          * of the pbls and the first entry lays on the second pbl in the table
 614          */
 615         if (pbl_info->two_layered)
 616                 pbl_tbl = &pbl[1];
 617         else
 618                 pbl_tbl = pbl;
 619
 620         pbe = (struct regpair *)pbl_tbl->va;
 621         if (!pbe) {
 622                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 623                 return;
 624         }
 625
 626         pbe_cnt = 0;
 627
 628         fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
 629
 630         for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
 631                 pg_addr = sg_page_iter_dma_address(&sg_iter);
 632                 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
 633                         pbe->lo = cpu_to_le32(pg_addr);
 634                         pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 635
 636                         pg_addr += BIT(pg_shift);
 637                         pbe_cnt++;
 638                         total_num_pbes++;
 639                         pbe++;
 640
 641                         if (total_num_pbes == pbl_info->num_pbes)
 642                                 return;
 643
 644                         /* If the given pbl is full storing the pbes,
 645                          * move to next pbl.
 646                          */
 647                         if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
 648                                 pbl_tbl++;
 649                                 pbe = (struct regpair *)pbl_tbl->va;
 650                                 pbe_cnt = 0;
 651                         }
 652
 653                         fw_pg_cnt++;
 654                 }
 655         }
 656 }
 657
 658 static int qedr_db_recovery_add(struct qedr_dev *dev,
 659                                 void __iomem *db_addr,
 660                                 void *db_data,
 661                                 enum qed_db_rec_width db_width,
 662                                 enum qed_db_rec_space db_space)
 663 {
 664         if (!db_data) {
 665                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 666                 return 0;
 667         }
 668
 669         return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
 670                                                  db_width, db_space);
 671 }
 672
 673 static void qedr_db_recovery_del(struct qedr_dev *dev,
 674                                  void __iomem *db_addr,
 675                                  void *db_data)
 676 {
 677         if (!db_data) {
 678                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 679                 return;
 680         }
 681
 682         /* Ignore return code as there is not much we can do about it. Error
 683          * log will be printed inside.
 684          */
 685         dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
 686 }
 687
 688 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 689                               struct qedr_cq *cq, struct ib_udata *udata,
 690                               u32 db_offset)
 691 {
 692         struct qedr_create_cq_uresp uresp;
 693         int rc;
 694
 695         memset(&uresp, 0, sizeof(uresp));
 696
 697         uresp.db_offset = db_offset;
 698         uresp.icid = cq->icid;
 699         if (cq->q.db_mmap_entry)
 700                 uresp.db_rec_addr =
 701                         rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
 702
 703         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 704         if (rc)
 705                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 706
 707         return rc;
 708 }
 709
 710 static void consume_cqe(struct qedr_cq *cq)
 711 {
 712         if (cq->latest_cqe == cq->toggle_cqe)
 713                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 714
 715         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 716 }
 717
 718 static inline int qedr_align_cq_entries(int entries)
 719 {
 720         u64 size, aligned_size;
 721
 722         /* We allocate an extra entry that we don't report to the FW. */
 723         size = (entries + 1) * QEDR_CQE_SIZE;
 724         aligned_size = ALIGN(size, PAGE_SIZE);
 725
 726         return aligned_size / QEDR_CQE_SIZE;
 727 }
 728
 729 static int qedr_init_user_db_rec(struct ib_udata *udata,
 730                                  struct qedr_dev *dev, struct qedr_userq *q,
 731                                  bool requires_db_rec)
 732 {
 733         struct qedr_ucontext *uctx =
 734                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
 735                                           ibucontext);
 736         struct qedr_user_mmap_entry *entry;
 737         int rc;
 738
 739         /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
 740         if (requires_db_rec == 0 || !uctx->db_rec)
 741                 return 0;
 742
 743         /* Allocate a page for doorbell recovery, add to mmap */
 744         q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
 745         if (!q->db_rec_data) {
 746                 DP_ERR(dev, "get_zeroed_page failed\n");
 747                 return -ENOMEM;
 748         }
 749
 750         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 751         if (!entry)
 752                 goto err_free_db_data;
 753
 754         entry->address = q->db_rec_data;
 755         entry->length = PAGE_SIZE;
 756         entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
 757         rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
 758                                          &entry->rdma_entry,
 759                                          PAGE_SIZE);
 760         if (rc)
 761                 goto err_free_entry;
 762
 763         q->db_mmap_entry = &entry->rdma_entry;
 764
 765         return 0;
 766
 767 err_free_entry:
 768         kfree(entry);
 769
 770 err_free_db_data:
 771         free_page((unsigned long)q->db_rec_data);
 772         q->db_rec_data = NULL;
 773         return -ENOMEM;
 774 }
 775
 776 static inline int qedr_init_user_queue(struct ib_udata *udata,
 777                                        struct qedr_dev *dev,
 778                                        struct qedr_userq *q, u64 buf_addr,
 779                                        size_t buf_len, bool requires_db_rec,
 780                                        int access,
 781                                        int alloc_and_init)
 782 {
 783         u32 fw_pages;
 784         int rc;
 785
 786         q->buf_addr = buf_addr;
 787         q->buf_len = buf_len;
 788         q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
 789         if (IS_ERR(q->umem)) {
 790                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 791                        PTR_ERR(q->umem));
 792                 return PTR_ERR(q->umem);
 793         }
 794
 795         fw_pages = ib_umem_page_count(q->umem) <<
 796             (PAGE_SHIFT - FW_PAGE_SHIFT);
 797
 798         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 799         if (rc)
 800                 goto err0;
 801
 802         if (alloc_and_init) {
 803                 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 804                 if (IS_ERR(q->pbl_tbl)) {
 805                         rc = PTR_ERR(q->pbl_tbl);
 806                         goto err0;
 807                 }
 808                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 809                                    FW_PAGE_SHIFT);
 810         } else {
 811                 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 812                 if (!q->pbl_tbl) {
 813                         rc = -ENOMEM;
 814                         goto err0;
 815                 }
 816         }
 817
 818         /* mmap the user address used to store doorbell data for recovery */
 819         return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
 820
 821 err0:
 822         ib_umem_release(q->umem);
 823         q->umem = NULL;
 824
 825         return rc;
 826 }
 827
 828 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 829                                        struct qedr_ucontext *ctx,
 830                                        struct qedr_dev *dev, int vector,
 831                                        int chain_entries, int page_cnt,
 832                                        u64 pbl_ptr,
 833                                        struct qed_rdma_create_cq_in_params
 834                                        *params)
 835 {
 836         memset(params, 0, sizeof(*params));
 837         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 838         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 839         params->cnq_id = vector;
 840         params->cq_size = chain_entries - 1;
 841         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 842         params->pbl_num_pages = page_cnt;
 843         params->pbl_ptr = pbl_ptr;
 844         params->pbl_two_level = 0;
 845 }
 846
 847 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 848 {
 849         cq->db.data.agg_flags = flags;
 850         cq->db.data.value = cpu_to_le32(cons);
 851         writeq(cq->db.raw, cq->db_addr);
 852 }
 853
 854 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 855 {
 856         struct qedr_cq *cq = get_qedr_cq(ibcq);
 857         unsigned long sflags;
 858         struct qedr_dev *dev;
 859
 860         dev = get_qedr_dev(ibcq->device);
 861
 862         if (cq->destroyed) {
 863                 DP_ERR(dev,
 864                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 865                        cq, cq->icid);
 866                 return -EINVAL;
 867         }
 868
 869
 870         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 871                 return 0;
 872
 873         spin_lock_irqsave(&cq->cq_lock, sflags);
 874
 875         cq->arm_flags = 0;
 876
 877         if (flags & IB_CQ_SOLICITED)
 878                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 879
 880         if (flags & IB_CQ_NEXT_COMP)
 881                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 882
 883         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 884
 885         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 886
 887         return 0;
 888 }
 889
 890 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 891                    struct ib_udata *udata)
 892 {
 893         struct ib_device *ibdev = ibcq->device;
 894         struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
 895                 udata, struct qedr_ucontext, ibucontext);
 896         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 897         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 898         struct qedr_dev *dev = get_qedr_dev(ibdev);
 899         struct qed_rdma_create_cq_in_params params;
 900         struct qedr_create_cq_ureq ureq = {};
 901         int vector = attr->comp_vector;
 902         int entries = attr->cqe;
 903         struct qedr_cq *cq = get_qedr_cq(ibcq);
 904         int chain_entries;
 905         u32 db_offset;
 906         int page_cnt;
 907         u64 pbl_ptr;
 908         u16 icid;
 909         int rc;
 910
 911         DP_DEBUG(dev, QEDR_MSG_INIT,
 912                  "create_cq: called from %s. entries=%d, vector=%d\n",
 913                  udata ? "User Lib" : "Kernel", entries, vector);
 914
 915         if (entries > QEDR_MAX_CQES) {
 916                 DP_ERR(dev,
 917                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 918                        entries, QEDR_MAX_CQES);
 919                 return -EINVAL;
 920         }
 921
 922         chain_entries = qedr_align_cq_entries(entries);
 923         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 924
 925         /* calc db offset. user will add DPI base, kernel will add db addr */
 926         db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 927
 928         if (udata) {
 929                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
 930                                                          udata->inlen))) {
 931                         DP_ERR(dev,
 932                                "create cq: problem copying data from user space\n");
 933                         goto err0;
 934                 }
 935
 936                 if (!ureq.len) {
 937                         DP_ERR(dev,
 938                                "create cq: cannot create a cq with 0 entries\n");
 939                         goto err0;
 940                 }
 941
 942                 cq->cq_type = QEDR_CQ_TYPE_USER;
 943
 944                 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 945                                           ureq.len, true, IB_ACCESS_LOCAL_WRITE,
 946                                           1);
 947                 if (rc)
 948                         goto err0;
 949
 950                 pbl_ptr = cq->q.pbl_tbl->pa;
 951                 page_cnt = cq->q.pbl_info.num_pbes;
 952
 953                 cq->ibcq.cqe = chain_entries;
 954                 cq->q.db_addr = ctx->dpi_addr + db_offset;
 955         } else {
 956                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 957
 958                 rc = dev->ops->common->chain_alloc(dev->cdev,
 959                                                    QED_CHAIN_USE_TO_CONSUME,
 960                                                    QED_CHAIN_MODE_PBL,
 961                                                    QED_CHAIN_CNT_TYPE_U32,
 962                                                    chain_entries,
 963                                                    sizeof(union rdma_cqe),
 964                                                    &cq->pbl, NULL);
 965                 if (rc)
 966                         goto err0;
 967
 968                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 969                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 970                 cq->ibcq.cqe = cq->pbl.capacity;
 971         }
 972
 973         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 974                             pbl_ptr, &params);
 975
 976         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 977         if (rc)
 978                 goto err1;
 979
 980         cq->icid = icid;
 981         cq->sig = QEDR_CQ_MAGIC_NUMBER;
 982         spin_lock_init(&cq->cq_lock);
 983
 984         if (udata) {
 985                 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
 986                 if (rc)
 987                         goto err2;
 988
 989                 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
 990                                           &cq->q.db_rec_data->db_data,
 991                                           DB_REC_WIDTH_64B,
 992                                           DB_REC_USER);
 993                 if (rc)
 994                         goto err2;
 995
 996         } else {
 997                 /* Generate doorbell address. */
 998                 cq->db.data.icid = cq->icid;
 999                 cq->db_addr = dev->db_addr + db_offset;
1000                 cq->db.data.params = DB_AGG_CMD_SET <<
1001                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1002
1003                 /* point to the very last element, passing it we will toggle */
1004                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1005                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1006                 cq->latest_cqe = NULL;
1007                 consume_cqe(cq);
1008                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1009
1010                 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1011                                           DB_REC_WIDTH_64B, DB_REC_KERNEL);
1012                 if (rc)
1013                         goto err2;
1014         }
1015
1016         DP_DEBUG(dev, QEDR_MSG_CQ,
1017                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1018                  cq->icid, cq, params.cq_size);
1019
1020         return 0;
1021
1022 err2:
1023         destroy_iparams.icid = cq->icid;
1024         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1025                                   &destroy_oparams);
1026 err1:
1027         if (udata) {
1028                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1029                 ib_umem_release(cq->q.umem);
1030                 if (cq->q.db_mmap_entry)
1031                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1032         } else {
1033                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1034         }
1035 err0:
1036         return -EINVAL;
1037 }
1038
1039 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1040 {
1041         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1042         struct qedr_cq *cq = get_qedr_cq(ibcq);
1043
1044         DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1045
1046         return 0;
1047 }
1048
1049 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1050 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1051
1052 void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1053 {
1054         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1055         struct qed_rdma_destroy_cq_out_params oparams;
1056         struct qed_rdma_destroy_cq_in_params iparams;
1057         struct qedr_cq *cq = get_qedr_cq(ibcq);
1058         int iter;
1059
1060         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1061
1062         cq->destroyed = 1;
1063
1064         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1065         if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1066                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1067                 return;
1068         }
1069
1070         iparams.icid = cq->icid;
1071         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1072         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1073
1074         if (udata) {
1075                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1076                 ib_umem_release(cq->q.umem);
1077
1078                 if (cq->q.db_rec_data) {
1079                         qedr_db_recovery_del(dev, cq->q.db_addr,
1080                                              &cq->q.db_rec_data->db_data);
1081                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1082                 }
1083         } else {
1084                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1085         }
1086
1087         /* We don't want the IRQ handler to handle a non-existing CQ so we
1088          * wait until all CNQ interrupts, if any, are received. This will always
1089          * happen and will always happen very fast. If not, then a serious error
1090          * has occured. That is why we can use a long delay.
1091          * We spin for a short time so we don’t lose time on context switching
1092          * in case all the completions are handled in that span. Otherwise
1093          * we sleep for a while and check again. Since the CNQ may be
1094          * associated with (only) the current CPU we use msleep to allow the
1095          * current CPU to be freed.
1096          * The CNQ notification is increased in qedr_irq_handler().
1097          */
1098         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1099         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1100                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1101                 iter--;
1102         }
1103
1104         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1105         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1106                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1107                 iter--;
1108         }
1109
1110         /* Note that we don't need to have explicit code to wait for the
1111          * completion of the event handler because it is invoked from the EQ.
1112          * Since the destroy CQ ramrod has also been received on the EQ we can
1113          * be certain that there's no event handler in process.
1114          */
1115 }
1116
1117 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1118                                           struct ib_qp_attr *attr,
1119                                           int attr_mask,
1120                                           struct qed_rdma_modify_qp_in_params
1121                                           *qp_params)
1122 {
1123         const struct ib_gid_attr *gid_attr;
1124         enum rdma_network_type nw_type;
1125         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1126         u32 ipv4_addr;
1127         int ret;
1128         int i;
1129
1130         gid_attr = grh->sgid_attr;
1131         ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1132         if (ret)
1133                 return ret;
1134
1135         nw_type = rdma_gid_attr_network_type(gid_attr);
1136         switch (nw_type) {
1137         case RDMA_NETWORK_IPV6:
1138                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1139                        sizeof(qp_params->sgid));
1140                 memcpy(&qp_params->dgid.bytes[0],
1141                        &grh->dgid,
1142                        sizeof(qp_params->dgid));
1143                 qp_params->roce_mode = ROCE_V2_IPV6;
1144                 SET_FIELD(qp_params->modify_flags,
1145                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1146                 break;
1147         case RDMA_NETWORK_IB:
1148                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1149                        sizeof(qp_params->sgid));
1150                 memcpy(&qp_params->dgid.bytes[0],
1151                        &grh->dgid,
1152                        sizeof(qp_params->dgid));
1153                 qp_params->roce_mode = ROCE_V1;
1154                 break;
1155         case RDMA_NETWORK_IPV4:
1156                 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1157                 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1158                 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1159                 qp_params->sgid.ipv4_addr = ipv4_addr;
1160                 ipv4_addr =
1161                     qedr_get_ipv4_from_gid(grh->dgid.raw);
1162                 qp_params->dgid.ipv4_addr = ipv4_addr;
1163                 SET_FIELD(qp_params->modify_flags,
1164                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1165                 qp_params->roce_mode = ROCE_V2_IPV4;
1166                 break;
1167         }
1168
1169         for (i = 0; i < 4; i++) {
1170                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1171                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1172         }
1173
1174         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1175                 qp_params->vlan_id = 0;
1176
1177         return 0;
1178 }
1179
1180 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1181                                struct ib_qp_init_attr *attrs,
1182                                struct ib_udata *udata)
1183 {
1184         struct qedr_device_attr *qattr = &dev->attr;
1185
1186         /* QP0... attrs->qp_type == IB_QPT_GSI */
1187         if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1188                 DP_DEBUG(dev, QEDR_MSG_QP,
1189                          "create qp: unsupported qp type=0x%x requested\n",
1190                          attrs->qp_type);
1191                 return -EOPNOTSUPP;
1192         }
1193
1194         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1195                 DP_ERR(dev,
1196                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1197                        attrs->cap.max_send_wr, qattr->max_sqe);
1198                 return -EINVAL;
1199         }
1200
1201         if (attrs->cap.max_inline_data > qattr->max_inline) {
1202                 DP_ERR(dev,
1203                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1204                        attrs->cap.max_inline_data, qattr->max_inline);
1205                 return -EINVAL;
1206         }
1207
1208         if (attrs->cap.max_send_sge > qattr->max_sge) {
1209                 DP_ERR(dev,
1210                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1211                        attrs->cap.max_send_sge, qattr->max_sge);
1212                 return -EINVAL;
1213         }
1214
1215         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1216                 DP_ERR(dev,
1217                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1218                        attrs->cap.max_recv_sge, qattr->max_sge);
1219                 return -EINVAL;
1220         }
1221
1222         /* Unprivileged user space cannot create special QP */
1223         if (udata && attrs->qp_type == IB_QPT_GSI) {
1224                 DP_ERR(dev,
1225                        "create qp: userspace can't create special QPs of type=0x%x\n",
1226                        attrs->qp_type);
1227                 return -EINVAL;
1228         }
1229
1230         return 0;
1231 }
1232
1233 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1234                                struct qedr_srq *srq, struct ib_udata *udata)
1235 {
1236         struct qedr_create_srq_uresp uresp = {};
1237         int rc;
1238
1239         uresp.srq_id = srq->srq_id;
1240
1241         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1242         if (rc)
1243                 DP_ERR(dev, "create srq: problem copying data to user space\n");
1244
1245         return rc;
1246 }
1247
1248 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1249                               struct qedr_create_qp_uresp *uresp,
1250                               struct qedr_qp *qp)
1251 {
1252         /* iWARP requires two doorbells per RQ. */
1253         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1254                 uresp->rq_db_offset =
1255                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1256                 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1257         } else {
1258                 uresp->rq_db_offset =
1259                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1260         }
1261
1262         uresp->rq_icid = qp->icid;
1263         if (qp->urq.db_mmap_entry)
1264                 uresp->rq_db_rec_addr =
1265                         rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1266 }
1267
1268 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1269                                struct qedr_create_qp_uresp *uresp,
1270                                struct qedr_qp *qp)
1271 {
1272         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1273
1274         /* iWARP uses the same cid for rq and sq */
1275         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1276                 uresp->sq_icid = qp->icid;
1277         else
1278                 uresp->sq_icid = qp->icid + 1;
1279
1280         if (qp->usq.db_mmap_entry)
1281                 uresp->sq_db_rec_addr =
1282                         rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1283 }
1284
1285 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1286                               struct qedr_qp *qp, struct ib_udata *udata,
1287                               struct qedr_create_qp_uresp *uresp)
1288 {
1289         int rc;
1290
1291         memset(uresp, 0, sizeof(*uresp));
1292         qedr_copy_sq_uresp(dev, uresp, qp);
1293         qedr_copy_rq_uresp(dev, uresp, qp);
1294
1295         uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1296         uresp->qp_id = qp->qp_id;
1297
1298         rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1299         if (rc)
1300                 DP_ERR(dev,
1301                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1302                        qp->icid);
1303
1304         return rc;
1305 }
1306
1307 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1308                                       struct qedr_qp *qp,
1309                                       struct qedr_pd *pd,
1310                                       struct ib_qp_init_attr *attrs)
1311 {
1312         spin_lock_init(&qp->q_lock);
1313         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1314                 kref_init(&qp->refcnt);
1315                 init_completion(&qp->iwarp_cm_comp);
1316         }
1317         qp->pd = pd;
1318         qp->qp_type = attrs->qp_type;
1319         qp->max_inline_data = attrs->cap.max_inline_data;
1320         qp->sq.max_sges = attrs->cap.max_send_sge;
1321         qp->state = QED_ROCE_QP_STATE_RESET;
1322         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1323         qp->sq_cq = get_qedr_cq(attrs->send_cq);
1324         qp->dev = dev;
1325
1326         if (attrs->srq) {
1327                 qp->srq = get_qedr_srq(attrs->srq);
1328         } else {
1329                 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1330                 qp->rq.max_sges = attrs->cap.max_recv_sge;
1331                 DP_DEBUG(dev, QEDR_MSG_QP,
1332                          "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1333                          qp->rq.max_sges, qp->rq_cq->icid);
1334         }
1335
1336         DP_DEBUG(dev, QEDR_MSG_QP,
1337                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1338                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1339                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1340         DP_DEBUG(dev, QEDR_MSG_QP,
1341                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1342                  qp->sq.max_sges, qp->sq_cq->icid);
1343 }
1344
1345 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1346 {
1347         int rc;
1348
1349         qp->sq.db = dev->db_addr +
1350                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1351         qp->sq.db_data.data.icid = qp->icid + 1;
1352         rc = qedr_db_recovery_add(dev, qp->sq.db,
1353                                   &qp->sq.db_data,
1354                                   DB_REC_WIDTH_32B,
1355                                   DB_REC_KERNEL);
1356         if (rc)
1357                 return rc;
1358
1359         if (!qp->srq) {
1360                 qp->rq.db = dev->db_addr +
1361                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1362                 qp->rq.db_data.data.icid = qp->icid;
1363
1364                 rc = qedr_db_recovery_add(dev, qp->rq.db,
1365                                           &qp->rq.db_data,
1366                                           DB_REC_WIDTH_32B,
1367                                           DB_REC_KERNEL);
1368                 if (rc)
1369                         qedr_db_recovery_del(dev, qp->sq.db,
1370                                              &qp->sq.db_data);
1371         }
1372
1373         return rc;
1374 }
1375
1376 static int qedr_check_srq_params(struct qedr_dev *dev,
1377                                  struct ib_srq_init_attr *attrs,
1378                                  struct ib_udata *udata)
1379 {
1380         struct qedr_device_attr *qattr = &dev->attr;
1381
1382         if (attrs->attr.max_wr > qattr->max_srq_wr) {
1383                 DP_ERR(dev,
1384                        "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1385                        attrs->attr.max_wr, qattr->max_srq_wr);
1386                 return -EINVAL;
1387         }
1388
1389         if (attrs->attr.max_sge > qattr->max_sge) {
1390                 DP_ERR(dev,
1391                        "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1392                        attrs->attr.max_sge, qattr->max_sge);
1393                 return -EINVAL;
1394         }
1395
1396         return 0;
1397 }
1398
1399 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1400 {
1401         qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1402         ib_umem_release(srq->usrq.umem);
1403         ib_umem_release(srq->prod_umem);
1404 }
1405
1406 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1407 {
1408         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1409         struct qedr_dev *dev = srq->dev;
1410
1411         dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1412
1413         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1414                           hw_srq->virt_prod_pair_addr,
1415                           hw_srq->phy_prod_pair_addr);
1416 }
1417
1418 static int qedr_init_srq_user_params(struct ib_udata *udata,
1419                                      struct qedr_srq *srq,
1420                                      struct qedr_create_srq_ureq *ureq,
1421                                      int access)
1422 {
1423         struct scatterlist *sg;
1424         int rc;
1425
1426         rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1427                                   ureq->srq_len, false, access, 1);
1428         if (rc)
1429                 return rc;
1430
1431         srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1432                                      sizeof(struct rdma_srq_producers), access);
1433         if (IS_ERR(srq->prod_umem)) {
1434                 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1435                 ib_umem_release(srq->usrq.umem);
1436                 DP_ERR(srq->dev,
1437                        "create srq: failed ib_umem_get for producer, got %ld\n",
1438                        PTR_ERR(srq->prod_umem));
1439                 return PTR_ERR(srq->prod_umem);
1440         }
1441
1442         sg = srq->prod_umem->sg_head.sgl;
1443         srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1444
1445         return 0;
1446 }
1447
1448 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1449                                         struct qedr_dev *dev,
1450                                         struct ib_srq_init_attr *init_attr)
1451 {
1452         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1453         dma_addr_t phy_prod_pair_addr;
1454         u32 num_elems;
1455         void *va;
1456         int rc;
1457
1458         va = dma_alloc_coherent(&dev->pdev->dev,
1459                                 sizeof(struct rdma_srq_producers),
1460                                 &phy_prod_pair_addr, GFP_KERNEL);
1461         if (!va) {
1462                 DP_ERR(dev,
1463                        "create srq: failed to allocate dma memory for producer\n");
1464                 return -ENOMEM;
1465         }
1466
1467         hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1468         hw_srq->virt_prod_pair_addr = va;
1469
1470         num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1471         rc = dev->ops->common->chain_alloc(dev->cdev,
1472                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1473                                            QED_CHAIN_MODE_PBL,
1474                                            QED_CHAIN_CNT_TYPE_U32,
1475                                            num_elems,
1476                                            QEDR_SRQ_WQE_ELEM_SIZE,
1477                                            &hw_srq->pbl, NULL);
1478         if (rc)
1479                 goto err0;
1480
1481         hw_srq->num_elems = num_elems;
1482
1483         return 0;
1484
1485 err0:
1486         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1487                           va, phy_prod_pair_addr);
1488         return rc;
1489 }
1490
1491 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1492                     struct ib_udata *udata)
1493 {
1494         struct qed_rdma_destroy_srq_in_params destroy_in_params;
1495         struct qed_rdma_create_srq_in_params in_params = {};
1496         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1497         struct qed_rdma_create_srq_out_params out_params;
1498         struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1499         struct qedr_create_srq_ureq ureq = {};
1500         u64 pbl_base_addr, phy_prod_pair_addr;
1501         struct qedr_srq_hwq_info *hw_srq;
1502         u32 page_cnt, page_size;
1503         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1504         int rc = 0;
1505
1506         DP_DEBUG(dev, QEDR_MSG_QP,
1507                  "create SRQ called from %s (pd %p)\n",
1508                  (udata) ? "User lib" : "kernel", pd);
1509
1510         rc = qedr_check_srq_params(dev, init_attr, udata);
1511         if (rc)
1512                 return -EINVAL;
1513
1514         srq->dev = dev;
1515         hw_srq = &srq->hw_srq;
1516         spin_lock_init(&srq->lock);
1517
1518         hw_srq->max_wr = init_attr->attr.max_wr;
1519         hw_srq->max_sges = init_attr->attr.max_sge;
1520
1521         if (udata) {
1522                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1523                                                          udata->inlen))) {
1524                         DP_ERR(dev,
1525                                "create srq: problem copying data from user space\n");
1526                         goto err0;
1527                 }
1528
1529                 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1530                 if (rc)
1531                         goto err0;
1532
1533                 page_cnt = srq->usrq.pbl_info.num_pbes;
1534                 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1535                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1536                 page_size = PAGE_SIZE;
1537         } else {
1538                 struct qed_chain *pbl;
1539
1540                 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1541                 if (rc)
1542                         goto err0;
1543
1544                 pbl = &hw_srq->pbl;
1545                 page_cnt = qed_chain_get_page_cnt(pbl);
1546                 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1547                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1548                 page_size = QED_CHAIN_PAGE_SIZE;
1549         }
1550
1551         in_params.pd_id = pd->pd_id;
1552         in_params.pbl_base_addr = pbl_base_addr;
1553         in_params.prod_pair_addr = phy_prod_pair_addr;
1554         in_params.num_pages = page_cnt;
1555         in_params.page_size = page_size;
1556
1557         rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1558         if (rc)
1559                 goto err1;
1560
1561         srq->srq_id = out_params.srq_id;
1562
1563         if (udata) {
1564                 rc = qedr_copy_srq_uresp(dev, srq, udata);
1565                 if (rc)
1566                         goto err2;
1567         }
1568
1569         rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1570         if (rc)
1571                 goto err2;
1572
1573         DP_DEBUG(dev, QEDR_MSG_SRQ,
1574                  "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1575         return 0;
1576
1577 err2:
1578         destroy_in_params.srq_id = srq->srq_id;
1579
1580         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1581 err1:
1582         if (udata)
1583                 qedr_free_srq_user_params(srq);
1584         else
1585                 qedr_free_srq_kernel_params(srq);
1586 err0:
1587         return -EFAULT;
1588 }
1589
1590 void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1591 {
1592         struct qed_rdma_destroy_srq_in_params in_params = {};
1593         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1594         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1595
1596         xa_erase_irq(&dev->srqs, srq->srq_id);
1597         in_params.srq_id = srq->srq_id;
1598         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1599
1600         if (ibsrq->uobject)
1601                 qedr_free_srq_user_params(srq);
1602         else
1603                 qedr_free_srq_kernel_params(srq);
1604
1605         DP_DEBUG(dev, QEDR_MSG_SRQ,
1606                  "destroy srq: destroyed srq with srq_id=0x%0x\n",
1607                  srq->srq_id);
1608 }
1609
1610 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1611                     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1612 {
1613         struct qed_rdma_modify_srq_in_params in_params = {};
1614         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1615         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1616         int rc;
1617
1618         if (attr_mask & IB_SRQ_MAX_WR) {
1619                 DP_ERR(dev,
1620                        "modify srq: invalid attribute mask=0x%x specified for %p\n",
1621                        attr_mask, srq);
1622                 return -EINVAL;
1623         }
1624
1625         if (attr_mask & IB_SRQ_LIMIT) {
1626                 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1627                         DP_ERR(dev,
1628                                "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1629                                attr->srq_limit, srq->hw_srq.max_wr);
1630                         return -EINVAL;
1631                 }
1632
1633                 in_params.srq_id = srq->srq_id;
1634                 in_params.wqe_limit = attr->srq_limit;
1635                 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1636                 if (rc)
1637                         return rc;
1638         }
1639
1640         srq->srq_limit = attr->srq_limit;
1641
1642         DP_DEBUG(dev, QEDR_MSG_SRQ,
1643                  "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1644
1645         return 0;
1646 }
1647
1648 static inline void
1649 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1650                               struct qedr_pd *pd,
1651                               struct qedr_qp *qp,
1652                               struct ib_qp_init_attr *attrs,
1653                               bool fmr_and_reserved_lkey,
1654                               struct qed_rdma_create_qp_in_params *params)
1655 {
1656         /* QP handle to be written in an async event */
1657         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1658         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1659
1660         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1661         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1662         params->pd = pd->pd_id;
1663         params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1664         params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1665         params->stats_queue = 0;
1666         params->srq_id = 0;
1667         params->use_srq = false;
1668
1669         if (!qp->srq) {
1670                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1671
1672         } else {
1673                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1674                 params->srq_id = qp->srq->srq_id;
1675                 params->use_srq = true;
1676         }
1677 }
1678
1679 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1680 {
1681         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1682                  "qp=%p. "
1683                  "sq_addr=0x%llx, "
1684                  "sq_len=%zd, "
1685                  "rq_addr=0x%llx, "
1686                  "rq_len=%zd"
1687                  "\n",
1688                  qp,
1689                  qp->usq.buf_addr,
1690                  qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1691 }
1692
1693 static inline void
1694 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1695                             struct qedr_qp *qp,
1696                             struct qed_rdma_create_qp_out_params *out_params)
1697 {
1698         qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1699         qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1700
1701         qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1702                            &qp->usq.pbl_info, FW_PAGE_SHIFT);
1703         if (!qp->srq) {
1704                 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1705                 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1706         }
1707
1708         qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1709                            &qp->urq.pbl_info, FW_PAGE_SHIFT);
1710 }
1711
1712 static void qedr_cleanup_user(struct qedr_dev *dev,
1713                               struct qedr_ucontext *ctx,
1714                               struct qedr_qp *qp)
1715 {
1716         ib_umem_release(qp->usq.umem);
1717         qp->usq.umem = NULL;
1718
1719         ib_umem_release(qp->urq.umem);
1720         qp->urq.umem = NULL;
1721
1722         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1723                 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1724                 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1725         } else {
1726                 kfree(qp->usq.pbl_tbl);
1727                 kfree(qp->urq.pbl_tbl);
1728         }
1729
1730         if (qp->usq.db_rec_data) {
1731                 qedr_db_recovery_del(dev, qp->usq.db_addr,
1732                                      &qp->usq.db_rec_data->db_data);
1733                 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1734         }
1735
1736         if (qp->urq.db_rec_data) {
1737                 qedr_db_recovery_del(dev, qp->urq.db_addr,
1738                                      &qp->urq.db_rec_data->db_data);
1739                 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1740         }
1741
1742         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1743                 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1744                                      &qp->urq.db_rec_db2_data);
1745 }
1746
1747 static int qedr_create_user_qp(struct qedr_dev *dev,
1748                                struct qedr_qp *qp,
1749                                struct ib_pd *ibpd,
1750                                struct ib_udata *udata,
1751                                struct ib_qp_init_attr *attrs)
1752 {
1753         struct qed_rdma_create_qp_in_params in_params;
1754         struct qed_rdma_create_qp_out_params out_params;
1755         struct qedr_pd *pd = get_qedr_pd(ibpd);
1756         struct qedr_create_qp_uresp uresp;
1757         struct qedr_ucontext *ctx = pd ? pd->uctx : NULL;
1758         struct qedr_create_qp_ureq ureq;
1759         int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1760         int rc = -EINVAL;
1761
1762         qp->create_type = QEDR_QP_CREATE_USER;
1763         memset(&ureq, 0, sizeof(ureq));
1764         rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen));
1765         if (rc) {
1766                 DP_ERR(dev, "Problem copying data from user space\n");
1767                 return rc;
1768         }
1769
1770         /* SQ - read access only (0) */
1771         rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1772                                   ureq.sq_len, true, 0, alloc_and_init);
1773         if (rc)
1774                 return rc;
1775
1776         if (!qp->srq) {
1777                 /* RQ - read access only (0) */
1778                 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1779                                           ureq.rq_len, true, 0, alloc_and_init);
1780                 if (rc)
1781                         return rc;
1782         }
1783
1784         memset(&in_params, 0, sizeof(in_params));
1785         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1786         in_params.qp_handle_lo = ureq.qp_handle_lo;
1787         in_params.qp_handle_hi = ureq.qp_handle_hi;
1788         in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1789         in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1790         if (!qp->srq) {
1791                 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1792                 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1793         }
1794
1795         if (ctx)
1796                 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1797
1798         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1799                                               &in_params, &out_params);
1800
1801         if (!qp->qed_qp) {
1802                 rc = -ENOMEM;
1803                 goto err1;
1804         }
1805
1806         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1807                 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1808
1809         qp->qp_id = out_params.qp_id;
1810         qp->icid = out_params.icid;
1811
1812         rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1813         if (rc)
1814                 goto err;
1815
1816         /* db offset was calculated in copy_qp_uresp, now set in the user q */
1817         ctx = pd->uctx;
1818         qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1819         qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1820
1821         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1822                 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1823
1824                 /* calculate the db_rec_db2 data since it is constant so no
1825                  *  need to reflect from user
1826                  */
1827                 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1828                 qp->urq.db_rec_db2_data.data.value =
1829                         cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1830         }
1831
1832         rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1833                                   &qp->usq.db_rec_data->db_data,
1834                                   DB_REC_WIDTH_32B,
1835                                   DB_REC_USER);
1836         if (rc)
1837                 goto err;
1838
1839         rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1840                                   &qp->urq.db_rec_data->db_data,
1841                                   DB_REC_WIDTH_32B,
1842                                   DB_REC_USER);
1843         if (rc)
1844                 goto err;
1845
1846         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1847                 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1848                                           &qp->urq.db_rec_db2_data,
1849                                           DB_REC_WIDTH_32B,
1850                                           DB_REC_USER);
1851                 if (rc)
1852                         goto err;
1853         }
1854         qedr_qp_user_print(dev, qp);
1855
1856         return rc;
1857 err:
1858         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1859         if (rc)
1860                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1861
1862 err1:
1863         qedr_cleanup_user(dev, ctx, qp);
1864         return rc;
1865 }
1866
1867 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1868 {
1869         int rc;
1870
1871         qp->sq.db = dev->db_addr +
1872             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1873         qp->sq.db_data.data.icid = qp->icid;
1874
1875         rc = qedr_db_recovery_add(dev, qp->sq.db,
1876                                   &qp->sq.db_data,
1877                                   DB_REC_WIDTH_32B,
1878                                   DB_REC_KERNEL);
1879         if (rc)
1880                 return rc;
1881
1882         qp->rq.db = dev->db_addr +
1883                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1884         qp->rq.db_data.data.icid = qp->icid;
1885         qp->rq.iwarp_db2 = dev->db_addr +
1886                            DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1887         qp->rq.iwarp_db2_data.data.icid = qp->icid;
1888         qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1889
1890         rc = qedr_db_recovery_add(dev, qp->rq.db,
1891                                   &qp->rq.db_data,
1892                                   DB_REC_WIDTH_32B,
1893                                   DB_REC_KERNEL);
1894         if (rc)
1895                 return rc;
1896
1897         rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
1898                                   &qp->rq.iwarp_db2_data,
1899                                   DB_REC_WIDTH_32B,
1900                                   DB_REC_KERNEL);
1901         return rc;
1902 }
1903
1904 static int
1905 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1906                            struct qedr_qp *qp,
1907                            struct qed_rdma_create_qp_in_params *in_params,
1908                            u32 n_sq_elems, u32 n_rq_elems)
1909 {
1910         struct qed_rdma_create_qp_out_params out_params;
1911         int rc;
1912
1913         rc = dev->ops->common->chain_alloc(dev->cdev,
1914                                            QED_CHAIN_USE_TO_PRODUCE,
1915                                            QED_CHAIN_MODE_PBL,
1916                                            QED_CHAIN_CNT_TYPE_U32,
1917                                            n_sq_elems,
1918                                            QEDR_SQE_ELEMENT_SIZE,
1919                                            &qp->sq.pbl, NULL);
1920
1921         if (rc)
1922                 return rc;
1923
1924         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1925         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1926
1927         rc = dev->ops->common->chain_alloc(dev->cdev,
1928                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1929                                            QED_CHAIN_MODE_PBL,
1930                                            QED_CHAIN_CNT_TYPE_U32,
1931                                            n_rq_elems,
1932                                            QEDR_RQE_ELEMENT_SIZE,
1933                                            &qp->rq.pbl, NULL);
1934         if (rc)
1935                 return rc;
1936
1937         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1938         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1939
1940         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1941                                               in_params, &out_params);
1942
1943         if (!qp->qed_qp)
1944                 return -EINVAL;
1945
1946         qp->qp_id = out_params.qp_id;
1947         qp->icid = out_params.icid;
1948
1949         return qedr_set_roce_db_info(dev, qp);
1950 }
1951
1952 static int
1953 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1954                             struct qedr_qp *qp,
1955                             struct qed_rdma_create_qp_in_params *in_params,
1956                             u32 n_sq_elems, u32 n_rq_elems)
1957 {
1958         struct qed_rdma_create_qp_out_params out_params;
1959         struct qed_chain_ext_pbl ext_pbl;
1960         int rc;
1961
1962         in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1963                                                      QEDR_SQE_ELEMENT_SIZE,
1964                                                      QED_CHAIN_MODE_PBL);
1965         in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1966                                                      QEDR_RQE_ELEMENT_SIZE,
1967                                                      QED_CHAIN_MODE_PBL);
1968
1969         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1970                                               in_params, &out_params);
1971
1972         if (!qp->qed_qp)
1973                 return -EINVAL;
1974
1975         /* Now we allocate the chain */
1976         ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1977         ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1978
1979         rc = dev->ops->common->chain_alloc(dev->cdev,
1980                                            QED_CHAIN_USE_TO_PRODUCE,
1981                                            QED_CHAIN_MODE_PBL,
1982                                            QED_CHAIN_CNT_TYPE_U32,
1983                                            n_sq_elems,
1984                                            QEDR_SQE_ELEMENT_SIZE,
1985                                            &qp->sq.pbl, &ext_pbl);
1986
1987         if (rc)
1988                 goto err;
1989
1990         ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1991         ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1992
1993         rc = dev->ops->common->chain_alloc(dev->cdev,
1994                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1995                                            QED_CHAIN_MODE_PBL,
1996                                            QED_CHAIN_CNT_TYPE_U32,
1997                                            n_rq_elems,
1998                                            QEDR_RQE_ELEMENT_SIZE,
1999                                            &qp->rq.pbl, &ext_pbl);
2000
2001         if (rc)
2002                 goto err;
2003
2004         qp->qp_id = out_params.qp_id;
2005         qp->icid = out_params.icid;
2006
2007         return qedr_set_iwarp_db_info(dev, qp);
2008
2009 err:
2010         dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2011
2012         return rc;
2013 }
2014
2015 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2016 {
2017         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2018         kfree(qp->wqe_wr_id);
2019
2020         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2021         kfree(qp->rqe_wr_id);
2022
2023         /* GSI qp is not registered to db mechanism so no need to delete */
2024         if (qp->qp_type == IB_QPT_GSI)
2025                 return;
2026
2027         qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2028
2029         if (!qp->srq) {
2030                 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2031
2032                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2033                         qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2034                                              &qp->rq.iwarp_db2_data);
2035         }
2036 }
2037
2038 static int qedr_create_kernel_qp(struct qedr_dev *dev,
2039                                  struct qedr_qp *qp,
2040                                  struct ib_pd *ibpd,
2041                                  struct ib_qp_init_attr *attrs)
2042 {
2043         struct qed_rdma_create_qp_in_params in_params;
2044         struct qedr_pd *pd = get_qedr_pd(ibpd);
2045         int rc = -EINVAL;
2046         u32 n_rq_elems;
2047         u32 n_sq_elems;
2048         u32 n_sq_entries;
2049
2050         memset(&in_params, 0, sizeof(in_params));
2051         qp->create_type = QEDR_QP_CREATE_KERNEL;
2052
2053         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2054          * the ring. The ring should allow at least a single WR, even if the
2055          * user requested none, due to allocation issues.
2056          * We should add an extra WR since the prod and cons indices of
2057          * wqe_wr_id are managed in such a way that the WQ is considered full
2058          * when (prod+1)%max_wr==cons. We currently don't do that because we
2059          * double the number of entries due an iSER issue that pushes far more
2060          * WRs than indicated. If we decline its ib_post_send() then we get
2061          * error prints in the dmesg we'd like to avoid.
2062          */
2063         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2064                               dev->attr.max_sqe);
2065
2066         qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2067                                 GFP_KERNEL);
2068         if (!qp->wqe_wr_id) {
2069                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2070                 return -ENOMEM;
2071         }
2072
2073         /* QP handle to be written in CQE */
2074         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2075         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2076
2077         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2078          * the ring. There ring should allow at least a single WR, even if the
2079          * user requested none, due to allocation issues.
2080          */
2081         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2082
2083         /* Allocate driver internal RQ array */
2084         qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2085                                 GFP_KERNEL);
2086         if (!qp->rqe_wr_id) {
2087                 DP_ERR(dev,
2088                        "create qp: failed RQ shadow memory allocation\n");
2089                 kfree(qp->wqe_wr_id);
2090                 return -ENOMEM;
2091         }
2092
2093         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2094
2095         n_sq_entries = attrs->cap.max_send_wr;
2096         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2097         n_sq_entries = max_t(u32, n_sq_entries, 1);
2098         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2099
2100         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2101
2102         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2103                 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2104                                                  n_sq_elems, n_rq_elems);
2105         else
2106                 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2107                                                 n_sq_elems, n_rq_elems);
2108         if (rc)
2109                 qedr_cleanup_kernel(dev, qp);
2110
2111         return rc;
2112 }
2113
2114 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
2115                              struct ib_qp_init_attr *attrs,
2116                              struct ib_udata *udata)
2117 {
2118         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2119         struct qedr_pd *pd = get_qedr_pd(ibpd);
2120         struct qedr_qp *qp;
2121         struct ib_qp *ibqp;
2122         int rc = 0;
2123
2124         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2125                  udata ? "user library" : "kernel", pd);
2126
2127         rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2128         if (rc)
2129                 return ERR_PTR(rc);
2130
2131         DP_DEBUG(dev, QEDR_MSG_QP,
2132                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2133                  udata ? "user library" : "kernel", attrs->event_handler, pd,
2134                  get_qedr_cq(attrs->send_cq),
2135                  get_qedr_cq(attrs->send_cq)->icid,
2136                  get_qedr_cq(attrs->recv_cq),
2137                  attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2138
2139         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2140         if (!qp) {
2141                 DP_ERR(dev, "create qp: failed allocating memory\n");
2142                 return ERR_PTR(-ENOMEM);
2143         }
2144
2145         qedr_set_common_qp_params(dev, qp, pd, attrs);
2146
2147         if (attrs->qp_type == IB_QPT_GSI) {
2148                 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2149                 if (IS_ERR(ibqp))
2150                         kfree(qp);
2151                 return ibqp;
2152         }
2153
2154         if (udata)
2155                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2156         else
2157                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2158
2159         if (rc)
2160                 goto err;
2161
2162         qp->ibqp.qp_num = qp->qp_id;
2163
2164         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2165                 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2166                 if (rc)
2167                         goto err;
2168         }
2169
2170         return &qp->ibqp;
2171
2172 err:
2173         kfree(qp);
2174
2175         return ERR_PTR(-EFAULT);
2176 }
2177
2178 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2179 {
2180         switch (qp_state) {
2181         case QED_ROCE_QP_STATE_RESET:
2182                 return IB_QPS_RESET;
2183         case QED_ROCE_QP_STATE_INIT:
2184                 return IB_QPS_INIT;
2185         case QED_ROCE_QP_STATE_RTR:
2186                 return IB_QPS_RTR;
2187         case QED_ROCE_QP_STATE_RTS:
2188                 return IB_QPS_RTS;
2189         case QED_ROCE_QP_STATE_SQD:
2190                 return IB_QPS_SQD;
2191         case QED_ROCE_QP_STATE_ERR:
2192                 return IB_QPS_ERR;
2193         case QED_ROCE_QP_STATE_SQE:
2194                 return IB_QPS_SQE;
2195         }
2196         return IB_QPS_ERR;
2197 }
2198
2199 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2200                                         enum ib_qp_state qp_state)
2201 {
2202         switch (qp_state) {
2203         case IB_QPS_RESET:
2204                 return QED_ROCE_QP_STATE_RESET;
2205         case IB_QPS_INIT:
2206                 return QED_ROCE_QP_STATE_INIT;
2207         case IB_QPS_RTR:
2208                 return QED_ROCE_QP_STATE_RTR;
2209         case IB_QPS_RTS:
2210                 return QED_ROCE_QP_STATE_RTS;
2211         case IB_QPS_SQD:
2212                 return QED_ROCE_QP_STATE_SQD;
2213         case IB_QPS_ERR:
2214                 return QED_ROCE_QP_STATE_ERR;
2215         default:
2216                 return QED_ROCE_QP_STATE_ERR;
2217         }
2218 }
2219
2220 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2221 {
2222         qed_chain_reset(&qph->pbl);
2223         qph->prod = 0;
2224         qph->cons = 0;
2225         qph->wqe_cons = 0;
2226         qph->db_data.data.value = cpu_to_le16(0);
2227 }
2228
2229 static int qedr_update_qp_state(struct qedr_dev *dev,
2230                                 struct qedr_qp *qp,
2231                                 enum qed_roce_qp_state cur_state,
2232                                 enum qed_roce_qp_state new_state)
2233 {
2234         int status = 0;
2235
2236         if (new_state == cur_state)
2237                 return 0;
2238
2239         switch (cur_state) {
2240         case QED_ROCE_QP_STATE_RESET:
2241                 switch (new_state) {
2242                 case QED_ROCE_QP_STATE_INIT:
2243                         qp->prev_wqe_size = 0;
2244                         qedr_reset_qp_hwq_info(&qp->sq);
2245                         qedr_reset_qp_hwq_info(&qp->rq);
2246                         break;
2247                 default:
2248                         status = -EINVAL;
2249                         break;
2250                 }
2251                 break;
2252         case QED_ROCE_QP_STATE_INIT:
2253                 switch (new_state) {
2254                 case QED_ROCE_QP_STATE_RTR:
2255                         /* Update doorbell (in case post_recv was
2256                          * done before move to RTR)
2257                          */
2258
2259                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2260                                 writel(qp->rq.db_data.raw, qp->rq.db);
2261                         }
2262                         break;
2263                 case QED_ROCE_QP_STATE_ERR:
2264                         break;
2265                 default:
2266                         /* Invalid state change. */
2267                         status = -EINVAL;
2268                         break;
2269                 }
2270                 break;
2271         case QED_ROCE_QP_STATE_RTR:
2272                 /* RTR->XXX */
2273                 switch (new_state) {
2274                 case QED_ROCE_QP_STATE_RTS:
2275                         break;
2276                 case QED_ROCE_QP_STATE_ERR:
2277                         break;
2278                 default:
2279                         /* Invalid state change. */
2280                         status = -EINVAL;
2281                         break;
2282                 }
2283                 break;
2284         case QED_ROCE_QP_STATE_RTS:
2285                 /* RTS->XXX */
2286                 switch (new_state) {
2287                 case QED_ROCE_QP_STATE_SQD:
2288                         break;
2289                 case QED_ROCE_QP_STATE_ERR:
2290                         break;
2291                 default:
2292                         /* Invalid state change. */
2293                         status = -EINVAL;
2294                         break;
2295                 }
2296                 break;
2297         case QED_ROCE_QP_STATE_SQD:
2298                 /* SQD->XXX */
2299                 switch (new_state) {
2300                 case QED_ROCE_QP_STATE_RTS:
2301                 case QED_ROCE_QP_STATE_ERR:
2302                         break;
2303                 default:
2304                         /* Invalid state change. */
2305                         status = -EINVAL;
2306                         break;
2307                 }
2308                 break;
2309         case QED_ROCE_QP_STATE_ERR:
2310                 /* ERR->XXX */
2311                 switch (new_state) {
2312                 case QED_ROCE_QP_STATE_RESET:
2313                         if ((qp->rq.prod != qp->rq.cons) ||
2314                             (qp->sq.prod != qp->sq.cons)) {
2315                                 DP_NOTICE(dev,
2316                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2317                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
2318                                           qp->sq.cons);
2319                                 status = -EINVAL;
2320                         }
2321                         break;
2322                 default:
2323                         status = -EINVAL;
2324                         break;
2325                 }
2326                 break;
2327         default:
2328                 status = -EINVAL;
2329                 break;
2330         }
2331
2332         return status;
2333 }
2334
2335 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2336                    int attr_mask, struct ib_udata *udata)
2337 {
2338         struct qedr_qp *qp = get_qedr_qp(ibqp);
2339         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2340         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2341         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2342         enum ib_qp_state old_qp_state, new_qp_state;
2343         enum qed_roce_qp_state cur_state;
2344         int rc = 0;
2345
2346         DP_DEBUG(dev, QEDR_MSG_QP,
2347                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2348                  attr->qp_state);
2349
2350         old_qp_state = qedr_get_ibqp_state(qp->state);
2351         if (attr_mask & IB_QP_STATE)
2352                 new_qp_state = attr->qp_state;
2353         else
2354                 new_qp_state = old_qp_state;
2355
2356         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2357                 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2358                                         ibqp->qp_type, attr_mask)) {
2359                         DP_ERR(dev,
2360                                "modify qp: invalid attribute mask=0x%x specified for\n"
2361                                "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2362                                attr_mask, qp->qp_id, ibqp->qp_type,
2363                                old_qp_state, new_qp_state);
2364                         rc = -EINVAL;
2365                         goto err;
2366                 }
2367         }
2368
2369         /* Translate the masks... */
2370         if (attr_mask & IB_QP_STATE) {
2371                 SET_FIELD(qp_params.modify_flags,
2372                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2373                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2374         }
2375
2376         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2377                 qp_params.sqd_async = true;
2378
2379         if (attr_mask & IB_QP_PKEY_INDEX) {
2380                 SET_FIELD(qp_params.modify_flags,
2381                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2382                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2383                         rc = -EINVAL;
2384                         goto err;
2385                 }
2386
2387                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2388         }
2389
2390         if (attr_mask & IB_QP_QKEY)
2391                 qp->qkey = attr->qkey;
2392
2393         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2394                 SET_FIELD(qp_params.modify_flags,
2395                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2396                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2397                                                   IB_ACCESS_REMOTE_READ;
2398                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2399                                                    IB_ACCESS_REMOTE_WRITE;
2400                 qp_params.incoming_atomic_en = attr->qp_access_flags &
2401                                                IB_ACCESS_REMOTE_ATOMIC;
2402         }
2403
2404         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2405                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2406                         return -EINVAL;
2407
2408                 if (attr_mask & IB_QP_PATH_MTU) {
2409                         if (attr->path_mtu < IB_MTU_256 ||
2410                             attr->path_mtu > IB_MTU_4096) {
2411                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2412                                 rc = -EINVAL;
2413                                 goto err;
2414                         }
2415                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2416                                       ib_mtu_enum_to_int(iboe_get_mtu
2417                                                          (dev->ndev->mtu)));
2418                 }
2419
2420                 if (!qp->mtu) {
2421                         qp->mtu =
2422                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2423                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2424                 }
2425
2426                 SET_FIELD(qp_params.modify_flags,
2427                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2428
2429                 qp_params.traffic_class_tos = grh->traffic_class;
2430                 qp_params.flow_label = grh->flow_label;
2431                 qp_params.hop_limit_ttl = grh->hop_limit;
2432
2433                 qp->sgid_idx = grh->sgid_index;
2434
2435                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2436                 if (rc) {
2437                         DP_ERR(dev,
2438                                "modify qp: problems with GID index %d (rc=%d)\n",
2439                                grh->sgid_index, rc);
2440                         return rc;
2441                 }
2442
2443                 rc = qedr_get_dmac(dev, &attr->ah_attr,
2444                                    qp_params.remote_mac_addr);
2445                 if (rc)
2446                         return rc;
2447
2448                 qp_params.use_local_mac = true;
2449                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2450
2451                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2452                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2453                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2454                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2455                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2456                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2457                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2458                          qp_params.remote_mac_addr);
2459
2460                 qp_params.mtu = qp->mtu;
2461                 qp_params.lb_indication = false;
2462         }
2463
2464         if (!qp_params.mtu) {
2465                 /* Stay with current MTU */
2466                 if (qp->mtu)
2467                         qp_params.mtu = qp->mtu;
2468                 else
2469                         qp_params.mtu =
2470                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2471         }
2472
2473         if (attr_mask & IB_QP_TIMEOUT) {
2474                 SET_FIELD(qp_params.modify_flags,
2475                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2476
2477                 /* The received timeout value is an exponent used like this:
2478                  *    "12.7.34 LOCAL ACK TIMEOUT
2479                  *    Value representing the transport (ACK) timeout for use by
2480                  *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2481                  * The FW expects timeout in msec so we need to divide the usec
2482                  * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2483                  * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2484                  * The value of zero means infinite so we use a 'max_t' to make
2485                  * sure that sub 1 msec values will be configured as 1 msec.
2486                  */
2487                 if (attr->timeout)
2488                         qp_params.ack_timeout =
2489                                         1 << max_t(int, attr->timeout - 8, 0);
2490                 else
2491                         qp_params.ack_timeout = 0;
2492         }
2493
2494         if (attr_mask & IB_QP_RETRY_CNT) {
2495                 SET_FIELD(qp_params.modify_flags,
2496                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2497                 qp_params.retry_cnt = attr->retry_cnt;
2498         }
2499
2500         if (attr_mask & IB_QP_RNR_RETRY) {
2501                 SET_FIELD(qp_params.modify_flags,
2502                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2503                 qp_params.rnr_retry_cnt = attr->rnr_retry;
2504         }
2505
2506         if (attr_mask & IB_QP_RQ_PSN) {
2507                 SET_FIELD(qp_params.modify_flags,
2508                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2509                 qp_params.rq_psn = attr->rq_psn;
2510                 qp->rq_psn = attr->rq_psn;
2511         }
2512
2513         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2514                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2515                         rc = -EINVAL;
2516                         DP_ERR(dev,
2517                                "unsupported max_rd_atomic=%d, supported=%d\n",
2518                                attr->max_rd_atomic,
2519                                dev->attr.max_qp_req_rd_atomic_resc);
2520                         goto err;
2521                 }
2522
2523                 SET_FIELD(qp_params.modify_flags,
2524                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2525                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2526         }
2527
2528         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2529                 SET_FIELD(qp_params.modify_flags,
2530                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2531                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2532         }
2533
2534         if (attr_mask & IB_QP_SQ_PSN) {
2535                 SET_FIELD(qp_params.modify_flags,
2536                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2537                 qp_params.sq_psn = attr->sq_psn;
2538                 qp->sq_psn = attr->sq_psn;
2539         }
2540
2541         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2542                 if (attr->max_dest_rd_atomic >
2543                     dev->attr.max_qp_resp_rd_atomic_resc) {
2544                         DP_ERR(dev,
2545                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2546                                attr->max_dest_rd_atomic,
2547                                dev->attr.max_qp_resp_rd_atomic_resc);
2548
2549                         rc = -EINVAL;
2550                         goto err;
2551                 }
2552
2553                 SET_FIELD(qp_params.modify_flags,
2554                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2555                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2556         }
2557
2558         if (attr_mask & IB_QP_DEST_QPN) {
2559                 SET_FIELD(qp_params.modify_flags,
2560                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2561
2562                 qp_params.dest_qp = attr->dest_qp_num;
2563                 qp->dest_qp_num = attr->dest_qp_num;
2564         }
2565
2566         cur_state = qp->state;
2567
2568         /* Update the QP state before the actual ramrod to prevent a race with
2569          * fast path. Modifying the QP state to error will cause the device to
2570          * flush the CQEs and while polling the flushed CQEs will considered as
2571          * a potential issue if the QP isn't in error state.
2572          */
2573         if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2574             !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2575                 qp->state = QED_ROCE_QP_STATE_ERR;
2576
2577         if (qp->qp_type != IB_QPT_GSI)
2578                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2579                                               qp->qed_qp, &qp_params);
2580
2581         if (attr_mask & IB_QP_STATE) {
2582                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2583                         rc = qedr_update_qp_state(dev, qp, cur_state,
2584                                                   qp_params.new_state);
2585                 qp->state = qp_params.new_state;
2586         }
2587
2588 err:
2589         return rc;
2590 }
2591
2592 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2593 {
2594         int ib_qp_acc_flags = 0;
2595
2596         if (params->incoming_rdma_write_en)
2597                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2598         if (params->incoming_rdma_read_en)
2599                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2600         if (params->incoming_atomic_en)
2601                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2602         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2603         return ib_qp_acc_flags;
2604 }
2605
2606 int qedr_query_qp(struct ib_qp *ibqp,
2607                   struct ib_qp_attr *qp_attr,
2608                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2609 {
2610         struct qed_rdma_query_qp_out_params params;
2611         struct qedr_qp *qp = get_qedr_qp(ibqp);
2612         struct qedr_dev *dev = qp->dev;
2613         int rc = 0;
2614
2615         memset(&params, 0, sizeof(params));
2616
2617         rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2618         if (rc)
2619                 goto err;
2620
2621         memset(qp_attr, 0, sizeof(*qp_attr));
2622         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2623
2624         qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2625         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2626         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2627         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2628         qp_attr->rq_psn = params.rq_psn;
2629         qp_attr->sq_psn = params.sq_psn;
2630         qp_attr->dest_qp_num = params.dest_qp;
2631
2632         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2633
2634         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2635         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2636         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2637         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2638         qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2639         qp_init_attr->cap = qp_attr->cap;
2640
2641         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2642         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2643                         params.flow_label, qp->sgid_idx,
2644                         params.hop_limit_ttl, params.traffic_class_tos);
2645         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2646         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2647         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2648         qp_attr->timeout = params.timeout;
2649         qp_attr->rnr_retry = params.rnr_retry;
2650         qp_attr->retry_cnt = params.retry_cnt;
2651         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2652         qp_attr->pkey_index = params.pkey_index;
2653         qp_attr->port_num = 1;
2654         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2655         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2656         qp_attr->alt_pkey_index = 0;
2657         qp_attr->alt_port_num = 0;
2658         qp_attr->alt_timeout = 0;
2659         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2660
2661         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2662         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2663         qp_attr->max_rd_atomic = params.max_rd_atomic;
2664         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2665
2666         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2667                  qp_attr->cap.max_inline_data);
2668
2669 err:
2670         return rc;
2671 }
2672
2673 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2674                                   struct ib_udata *udata)
2675 {
2676         struct qedr_ucontext *ctx =
2677                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2678                                           ibucontext);
2679         int rc;
2680
2681         if (qp->qp_type != IB_QPT_GSI) {
2682                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2683                 if (rc)
2684                         return rc;
2685         }
2686
2687         if (qp->create_type == QEDR_QP_CREATE_USER)
2688                 qedr_cleanup_user(dev, ctx, qp);
2689         else
2690                 qedr_cleanup_kernel(dev, qp);
2691
2692         return 0;
2693 }
2694
2695 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2696 {
2697         struct qedr_qp *qp = get_qedr_qp(ibqp);
2698         struct qedr_dev *dev = qp->dev;
2699         struct ib_qp_attr attr;
2700         int attr_mask = 0;
2701
2702         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2703                  qp, qp->qp_type);
2704
2705         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2706                 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2707                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
2708                     (qp->state != QED_ROCE_QP_STATE_INIT)) {
2709
2710                         attr.qp_state = IB_QPS_ERR;
2711                         attr_mask |= IB_QP_STATE;
2712
2713                         /* Change the QP state to ERROR */
2714                         qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2715                 }
2716         } else {
2717                 /* If connection establishment started the WAIT_FOR_CONNECT
2718                  * bit will be on and we need to Wait for the establishment
2719                  * to complete before destroying the qp.
2720                  */
2721                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2722                                      &qp->iwarp_cm_flags))
2723                         wait_for_completion(&qp->iwarp_cm_comp);
2724
2725                 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2726                  * bit will be on, and we need to wait for the disconnect to
2727                  * complete before continuing. We can use the same completion,
2728                  * iwarp_cm_comp, since this is the only place that waits for
2729                  * this completion and it is sequential. In addition,
2730                  * disconnect can't occur before the connection is fully
2731                  * established, therefore if WAIT_FOR_DISCONNECT is on it
2732                  * means WAIT_FOR_CONNECT is also on and the completion for
2733                  * CONNECT already occurred.
2734                  */
2735                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2736                                      &qp->iwarp_cm_flags))
2737                         wait_for_completion(&qp->iwarp_cm_comp);
2738         }
2739
2740         if (qp->qp_type == IB_QPT_GSI)
2741                 qedr_destroy_gsi_qp(dev);
2742
2743         /* We need to remove the entry from the xarray before we release the
2744          * qp_id to avoid a race of the qp_id being reallocated and failing
2745          * on xa_insert
2746          */
2747         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2748                 xa_erase(&dev->qps, qp->qp_id);
2749
2750         qedr_free_qp_resources(dev, qp, udata);
2751
2752         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2753                 qedr_iw_qp_rem_ref(&qp->ibqp);
2754
2755         return 0;
2756 }
2757
2758 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2759                    struct ib_udata *udata)
2760 {
2761         struct qedr_ah *ah = get_qedr_ah(ibah);
2762
2763         rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2764
2765         return 0;
2766 }
2767
2768 void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2769 {
2770         struct qedr_ah *ah = get_qedr_ah(ibah);
2771
2772         rdma_destroy_ah_attr(&ah->attr);
2773 }
2774
2775 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2776 {
2777         struct qedr_pbl *pbl, *tmp;
2778
2779         if (info->pbl_table)
2780                 list_add_tail(&info->pbl_table->list_entry,
2781                               &info->free_pbl_list);
2782
2783         if (!list_empty(&info->inuse_pbl_list))
2784                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2785
2786         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2787                 list_del(&pbl->list_entry);
2788                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2789         }
2790 }
2791
2792 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2793                         size_t page_list_len, bool two_layered)
2794 {
2795         struct qedr_pbl *tmp;
2796         int rc;
2797
2798         INIT_LIST_HEAD(&info->free_pbl_list);
2799         INIT_LIST_HEAD(&info->inuse_pbl_list);
2800
2801         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2802                                   page_list_len, two_layered);
2803         if (rc)
2804                 goto done;
2805
2806         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2807         if (IS_ERR(info->pbl_table)) {
2808                 rc = PTR_ERR(info->pbl_table);
2809                 goto done;
2810         }
2811
2812         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2813                  &info->pbl_table->pa);
2814
2815         /* in usual case we use 2 PBLs, so we add one to free
2816          * list and allocating another one
2817          */
2818         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2819         if (IS_ERR(tmp)) {
2820                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2821                 goto done;
2822         }
2823
2824         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2825
2826         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2827
2828 done:
2829         if (rc)
2830                 free_mr_info(dev, info);
2831
2832         return rc;
2833 }
2834
2835 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2836                                u64 usr_addr, int acc, struct ib_udata *udata)
2837 {
2838         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2839         struct qedr_mr *mr;
2840         struct qedr_pd *pd;
2841         int rc = -ENOMEM;
2842
2843         pd = get_qedr_pd(ibpd);
2844         DP_DEBUG(dev, QEDR_MSG_MR,
2845                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2846                  pd->pd_id, start, len, usr_addr, acc);
2847
2848         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2849                 return ERR_PTR(-EINVAL);
2850
2851         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2852         if (!mr)
2853                 return ERR_PTR(rc);
2854
2855         mr->type = QEDR_MR_USER;
2856
2857         mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2858         if (IS_ERR(mr->umem)) {
2859                 rc = -EFAULT;
2860                 goto err0;
2861         }
2862
2863         rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2864         if (rc)
2865                 goto err1;
2866
2867         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2868                            &mr->info.pbl_info, PAGE_SHIFT);
2869
2870         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2871         if (rc) {
2872                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2873                 goto err1;
2874         }
2875
2876         /* Index only, 18 bit long, lkey = itid << 8 | key */
2877         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2878         mr->hw_mr.key = 0;
2879         mr->hw_mr.pd = pd->pd_id;
2880         mr->hw_mr.local_read = 1;
2881         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2882         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2883         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2884         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2885         mr->hw_mr.mw_bind = false;
2886         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2887         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2888         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2889         mr->hw_mr.page_size_log = PAGE_SHIFT;
2890         mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2891         mr->hw_mr.length = len;
2892         mr->hw_mr.vaddr = usr_addr;
2893         mr->hw_mr.zbva = false;
2894         mr->hw_mr.phy_mr = false;
2895         mr->hw_mr.dma_mr = false;
2896
2897         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2898         if (rc) {
2899                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2900                 goto err2;
2901         }
2902
2903         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2904         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2905             mr->hw_mr.remote_atomic)
2906                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2907
2908         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2909                  mr->ibmr.lkey);
2910         return &mr->ibmr;
2911
2912 err2:
2913         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2914 err1:
2915         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2916 err0:
2917         kfree(mr);
2918         return ERR_PTR(rc);
2919 }
2920
2921 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
2922 {
2923         struct qedr_mr *mr = get_qedr_mr(ib_mr);
2924         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2925         int rc = 0;
2926
2927         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2928         if (rc)
2929                 return rc;
2930
2931         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2932
2933         if (mr->type != QEDR_MR_DMA)
2934                 free_mr_info(dev, &mr->info);
2935
2936         /* it could be user registered memory. */
2937         ib_umem_release(mr->umem);
2938
2939         kfree(mr);
2940
2941         return rc;
2942 }
2943
2944 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2945                                        int max_page_list_len)
2946 {
2947         struct qedr_pd *pd = get_qedr_pd(ibpd);
2948         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2949         struct qedr_mr *mr;
2950         int rc = -ENOMEM;
2951
2952         DP_DEBUG(dev, QEDR_MSG_MR,
2953                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2954                  max_page_list_len);
2955
2956         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2957         if (!mr)
2958                 return ERR_PTR(rc);
2959
2960         mr->dev = dev;
2961         mr->type = QEDR_MR_FRMR;
2962
2963         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2964         if (rc)
2965                 goto err0;
2966
2967         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2968         if (rc) {
2969                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2970                 goto err0;
2971         }
2972
2973         /* Index only, 18 bit long, lkey = itid << 8 | key */
2974         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2975         mr->hw_mr.key = 0;
2976         mr->hw_mr.pd = pd->pd_id;
2977         mr->hw_mr.local_read = 1;
2978         mr->hw_mr.local_write = 0;
2979         mr->hw_mr.remote_read = 0;
2980         mr->hw_mr.remote_write = 0;
2981         mr->hw_mr.remote_atomic = 0;
2982         mr->hw_mr.mw_bind = false;
2983         mr->hw_mr.pbl_ptr = 0;
2984         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2985         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2986         mr->hw_mr.fbo = 0;
2987         mr->hw_mr.length = 0;
2988         mr->hw_mr.vaddr = 0;
2989         mr->hw_mr.zbva = false;
2990         mr->hw_mr.phy_mr = true;
2991         mr->hw_mr.dma_mr = false;
2992
2993         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2994         if (rc) {
2995                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2996                 goto err1;
2997         }
2998
2999         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3000         mr->ibmr.rkey = mr->ibmr.lkey;
3001
3002         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3003         return mr;
3004
3005 err1:
3006         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3007 err0:
3008         kfree(mr);
3009         return ERR_PTR(rc);
3010 }
3011
3012 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3013                             u32 max_num_sg)
3014 {
3015         struct qedr_mr *mr;
3016
3017         if (mr_type != IB_MR_TYPE_MEM_REG)
3018                 return ERR_PTR(-EINVAL);
3019
3020         mr = __qedr_alloc_mr(ibpd, max_num_sg);
3021
3022         if (IS_ERR(mr))
3023                 return ERR_PTR(-EINVAL);
3024
3025         return &mr->ibmr;
3026 }
3027
3028 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3029 {
3030         struct qedr_mr *mr = get_qedr_mr(ibmr);
3031         struct qedr_pbl *pbl_table;
3032         struct regpair *pbe;
3033         u32 pbes_in_page;
3034
3035         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3036                 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3037                 return -ENOMEM;
3038         }
3039
3040         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3041                  mr->npages, addr);
3042
3043         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3044         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3045         pbe = (struct regpair *)pbl_table->va;
3046         pbe +=  mr->npages % pbes_in_page;
3047         pbe->lo = cpu_to_le32((u32)addr);
3048         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3049
3050         mr->npages++;
3051
3052         return 0;
3053 }
3054
3055 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3056 {
3057         int work = info->completed - info->completed_handled - 1;
3058
3059         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3060         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3061                 struct qedr_pbl *pbl;
3062
3063                 /* Free all the page list that are possible to be freed
3064                  * (all the ones that were invalidated), under the assumption
3065                  * that if an FMR was completed successfully that means that
3066                  * if there was an invalidate operation before it also ended
3067                  */
3068                 pbl = list_first_entry(&info->inuse_pbl_list,
3069                                        struct qedr_pbl, list_entry);
3070                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3071                 info->completed_handled++;
3072         }
3073 }
3074
3075 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3076                    int sg_nents, unsigned int *sg_offset)
3077 {
3078         struct qedr_mr *mr = get_qedr_mr(ibmr);
3079
3080         mr->npages = 0;
3081
3082         handle_completed_mrs(mr->dev, &mr->info);
3083         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3084 }
3085
3086 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3087 {
3088         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3089         struct qedr_pd *pd = get_qedr_pd(ibpd);
3090         struct qedr_mr *mr;
3091         int rc;
3092
3093         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3094         if (!mr)
3095                 return ERR_PTR(-ENOMEM);
3096
3097         mr->type = QEDR_MR_DMA;
3098
3099         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3100         if (rc) {
3101                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3102                 goto err1;
3103         }
3104
3105         /* index only, 18 bit long, lkey = itid << 8 | key */
3106         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3107         mr->hw_mr.pd = pd->pd_id;
3108         mr->hw_mr.local_read = 1;
3109         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3110         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3111         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3112         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3113         mr->hw_mr.dma_mr = true;
3114
3115         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3116         if (rc) {
3117                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3118                 goto err2;
3119         }
3120
3121         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3122         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3123             mr->hw_mr.remote_atomic)
3124                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3125
3126         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3127         return &mr->ibmr;
3128
3129 err2:
3130         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3131 err1:
3132         kfree(mr);
3133         return ERR_PTR(rc);
3134 }
3135
3136 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3137 {
3138         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3139 }
3140
3141 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3142 {
3143         int i, len = 0;
3144
3145         for (i = 0; i < num_sge; i++)
3146                 len += sg_list[i].length;
3147
3148         return len;
3149 }
3150
3151 static void swap_wqe_data64(u64 *p)
3152 {
3153         int i;
3154
3155         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3156                 *p = cpu_to_be64(cpu_to_le64(*p));
3157 }
3158
3159 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3160                                        struct qedr_qp *qp, u8 *wqe_size,
3161                                        const struct ib_send_wr *wr,
3162                                        const struct ib_send_wr **bad_wr,
3163                                        u8 *bits, u8 bit)
3164 {
3165         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3166         char *seg_prt, *wqe;
3167         int i, seg_siz;
3168
3169         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3170                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3171                 *bad_wr = wr;
3172                 return 0;
3173         }
3174
3175         if (!data_size)
3176                 return data_size;
3177
3178         *bits |= bit;
3179
3180         seg_prt = NULL;
3181         wqe = NULL;
3182         seg_siz = 0;
3183
3184         /* Copy data inline */
3185         for (i = 0; i < wr->num_sge; i++) {
3186                 u32 len = wr->sg_list[i].length;
3187                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3188
3189                 while (len > 0) {
3190                         u32 cur;
3191
3192                         /* New segment required */
3193                         if (!seg_siz) {
3194                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3195                                 seg_prt = wqe;
3196                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
3197                                 (*wqe_size)++;
3198                         }
3199
3200                         /* Calculate currently allowed length */
3201                         cur = min_t(u32, len, seg_siz);
3202                         memcpy(seg_prt, src, cur);
3203
3204                         /* Update segment variables */
3205                         seg_prt += cur;
3206                         seg_siz -= cur;
3207
3208                         /* Update sge variables */
3209                         src += cur;
3210                         len -= cur;
3211
3212                         /* Swap fully-completed segments */
3213                         if (!seg_siz)
3214                                 swap_wqe_data64((u64 *)wqe);
3215                 }
3216         }
3217
3218         /* swap last not completed segment */
3219         if (seg_siz)
3220                 swap_wqe_data64((u64 *)wqe);
3221
3222         return data_size;
3223 }
3224
3225 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
3226         do {                                                    \
3227                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3228                 (sge)->length = cpu_to_le32(vlength);           \
3229                 (sge)->flags = cpu_to_le32(vflags);             \
3230         } while (0)
3231
3232 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
3233         do {                                                    \
3234                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
3235                 (hdr)->num_sges = num_sge;                      \
3236         } while (0)
3237
3238 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
3239         do {                                                    \
3240                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3241                 (sge)->length = cpu_to_le32(vlength);           \
3242                 (sge)->l_key = cpu_to_le32(vlkey);              \
3243         } while (0)
3244
3245 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3246                                 const struct ib_send_wr *wr)
3247 {
3248         u32 data_size = 0;
3249         int i;
3250
3251         for (i = 0; i < wr->num_sge; i++) {
3252                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3253
3254                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3255                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3256                 sge->length = cpu_to_le32(wr->sg_list[i].length);
3257                 data_size += wr->sg_list[i].length;
3258         }
3259
3260         if (wqe_size)
3261                 *wqe_size += wr->num_sge;
3262
3263         return data_size;
3264 }
3265
3266 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3267                                      struct qedr_qp *qp,
3268                                      struct rdma_sq_rdma_wqe_1st *rwqe,
3269                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
3270                                      const struct ib_send_wr *wr,
3271                                      const struct ib_send_wr **bad_wr)
3272 {
3273         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3274         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3275
3276         if (wr->send_flags & IB_SEND_INLINE &&
3277             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3278              wr->opcode == IB_WR_RDMA_WRITE)) {
3279                 u8 flags = 0;
3280
3281                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3282                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3283                                                    bad_wr, &rwqe->flags, flags);
3284         }
3285
3286         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3287 }
3288
3289 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3290                                      struct qedr_qp *qp,
3291                                      struct rdma_sq_send_wqe_1st *swqe,
3292                                      struct rdma_sq_send_wqe_2st *swqe2,
3293                                      const struct ib_send_wr *wr,
3294                                      const struct ib_send_wr **bad_wr)
3295 {
3296         memset(swqe2, 0, sizeof(*swqe2));
3297         if (wr->send_flags & IB_SEND_INLINE) {
3298                 u8 flags = 0;
3299
3300                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3301                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3302                                                    bad_wr, &swqe->flags, flags);
3303         }
3304
3305         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3306 }
3307
3308 static int qedr_prepare_reg(struct qedr_qp *qp,
3309                             struct rdma_sq_fmr_wqe_1st *fwqe1,
3310                             const struct ib_reg_wr *wr)
3311 {
3312         struct qedr_mr *mr = get_qedr_mr(wr->mr);
3313         struct rdma_sq_fmr_wqe_2nd *fwqe2;
3314
3315         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3316         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3317         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3318         fwqe1->l_key = wr->key;
3319
3320         fwqe2->access_ctrl = 0;
3321
3322         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3323                    !!(wr->access & IB_ACCESS_REMOTE_READ));
3324         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3325                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3326         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3327                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3328         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3329         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3330                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3331         fwqe2->fmr_ctrl = 0;
3332
3333         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3334                    ilog2(mr->ibmr.page_size) - 12);
3335
3336         fwqe2->length_hi = 0;
3337         fwqe2->length_lo = mr->ibmr.length;
3338         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3339         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3340
3341         qp->wqe_wr_id[qp->sq.prod].mr = mr;
3342
3343         return 0;
3344 }
3345
3346 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3347 {
3348         switch (opcode) {
3349         case IB_WR_RDMA_WRITE:
3350         case IB_WR_RDMA_WRITE_WITH_IMM:
3351                 return IB_WC_RDMA_WRITE;
3352         case IB_WR_SEND_WITH_IMM:
3353         case IB_WR_SEND:
3354         case IB_WR_SEND_WITH_INV:
3355                 return IB_WC_SEND;
3356         case IB_WR_RDMA_READ:
3357         case IB_WR_RDMA_READ_WITH_INV:
3358                 return IB_WC_RDMA_READ;
3359         case IB_WR_ATOMIC_CMP_AND_SWP:
3360                 return IB_WC_COMP_SWAP;
3361         case IB_WR_ATOMIC_FETCH_AND_ADD:
3362                 return IB_WC_FETCH_ADD;
3363         case IB_WR_REG_MR:
3364                 return IB_WC_REG_MR;
3365         case IB_WR_LOCAL_INV:
3366                 return IB_WC_LOCAL_INV;
3367         default:
3368                 return IB_WC_SEND;
3369         }
3370 }
3371
3372 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3373                                       const struct ib_send_wr *wr)
3374 {
3375         int wq_is_full, err_wr, pbl_is_full;
3376         struct qedr_dev *dev = qp->dev;
3377
3378         /* prevent SQ overflow and/or processing of a bad WR */
3379         err_wr = wr->num_sge > qp->sq.max_sges;
3380         wq_is_full = qedr_wq_is_full(&qp->sq);
3381         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3382                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3383         if (wq_is_full || err_wr || pbl_is_full) {
3384                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3385                         DP_ERR(dev,
3386                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3387                                qp);
3388                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3389                 }
3390
3391                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3392                         DP_ERR(dev,
3393                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3394                                qp);
3395                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3396                 }
3397
3398                 if (pbl_is_full &&
3399                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3400                         DP_ERR(dev,
3401                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3402                                qp);
3403                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3404                 }
3405                 return false;
3406         }
3407         return true;
3408 }
3409
3410 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3411                             const struct ib_send_wr **bad_wr)
3412 {
3413         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3414         struct qedr_qp *qp = get_qedr_qp(ibqp);
3415         struct rdma_sq_atomic_wqe_1st *awqe1;
3416         struct rdma_sq_atomic_wqe_2nd *awqe2;
3417         struct rdma_sq_atomic_wqe_3rd *awqe3;
3418         struct rdma_sq_send_wqe_2st *swqe2;
3419         struct rdma_sq_local_inv_wqe *iwqe;
3420         struct rdma_sq_rdma_wqe_2nd *rwqe2;
3421         struct rdma_sq_send_wqe_1st *swqe;
3422         struct rdma_sq_rdma_wqe_1st *rwqe;
3423         struct rdma_sq_fmr_wqe_1st *fwqe1;
3424         struct rdma_sq_common_wqe *wqe;
3425         u32 length;
3426         int rc = 0;
3427         bool comp;
3428
3429         if (!qedr_can_post_send(qp, wr)) {
3430                 *bad_wr = wr;
3431                 return -ENOMEM;
3432         }
3433
3434         wqe = qed_chain_produce(&qp->sq.pbl);
3435         qp->wqe_wr_id[qp->sq.prod].signaled =
3436                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3437
3438         wqe->flags = 0;
3439         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3440                    !!(wr->send_flags & IB_SEND_SOLICITED));
3441         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3442         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3443         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3444                    !!(wr->send_flags & IB_SEND_FENCE));
3445         wqe->prev_wqe_size = qp->prev_wqe_size;
3446
3447         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3448
3449         switch (wr->opcode) {
3450         case IB_WR_SEND_WITH_IMM:
3451                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3452                         rc = -EINVAL;
3453                         *bad_wr = wr;
3454                         break;
3455                 }
3456                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3457                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3458                 swqe->wqe_size = 2;
3459                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3460
3461                 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3462                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3463                                                    wr, bad_wr);
3464                 swqe->length = cpu_to_le32(length);
3465                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3466                 qp->prev_wqe_size = swqe->wqe_size;
3467                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3468                 break;
3469         case IB_WR_SEND:
3470                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3471                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3472
3473                 swqe->wqe_size = 2;
3474                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3475                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3476                                                    wr, bad_wr);
3477                 swqe->length = cpu_to_le32(length);
3478                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3479                 qp->prev_wqe_size = swqe->wqe_size;
3480                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3481                 break;
3482         case IB_WR_SEND_WITH_INV:
3483                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3484                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3485                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3486                 swqe->wqe_size = 2;
3487                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3488                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3489                                                    wr, bad_wr);
3490                 swqe->length = cpu_to_le32(length);
3491                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3492                 qp->prev_wqe_size = swqe->wqe_size;
3493                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3494                 break;
3495
3496         case IB_WR_RDMA_WRITE_WITH_IMM:
3497                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3498                         rc = -EINVAL;
3499                         *bad_wr = wr;
3500                         break;
3501                 }
3502                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3503                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3504
3505                 rwqe->wqe_size = 2;
3506                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3507                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3508                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3509                                                    wr, bad_wr);
3510                 rwqe->length = cpu_to_le32(length);
3511                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3512                 qp->prev_wqe_size = rwqe->wqe_size;
3513                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3514                 break;
3515         case IB_WR_RDMA_WRITE:
3516                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3517                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3518
3519                 rwqe->wqe_size = 2;
3520                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3521                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3522                                                    wr, bad_wr);
3523                 rwqe->length = cpu_to_le32(length);
3524                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3525                 qp->prev_wqe_size = rwqe->wqe_size;
3526                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3527                 break;
3528         case IB_WR_RDMA_READ_WITH_INV:
3529                 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3530                 /* fallthrough -- same is identical to RDMA READ */
3531
3532         case IB_WR_RDMA_READ:
3533                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3534                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3535
3536                 rwqe->wqe_size = 2;
3537                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3538                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3539                                                    wr, bad_wr);
3540                 rwqe->length = cpu_to_le32(length);
3541                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3542                 qp->prev_wqe_size = rwqe->wqe_size;
3543                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3544                 break;
3545
3546         case IB_WR_ATOMIC_CMP_AND_SWP:
3547         case IB_WR_ATOMIC_FETCH_AND_ADD:
3548                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3549                 awqe1->wqe_size = 4;
3550
3551                 awqe2 = qed_chain_produce(&qp->sq.pbl);
3552                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3553                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3554
3555                 awqe3 = qed_chain_produce(&qp->sq.pbl);
3556
3557                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3558                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3559                         DMA_REGPAIR_LE(awqe3->swap_data,
3560                                        atomic_wr(wr)->compare_add);
3561                 } else {
3562                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3563                         DMA_REGPAIR_LE(awqe3->swap_data,
3564                                        atomic_wr(wr)->swap);
3565                         DMA_REGPAIR_LE(awqe3->cmp_data,
3566                                        atomic_wr(wr)->compare_add);
3567                 }
3568
3569                 qedr_prepare_sq_sges(qp, NULL, wr);
3570
3571                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3572                 qp->prev_wqe_size = awqe1->wqe_size;
3573                 break;
3574
3575         case IB_WR_LOCAL_INV:
3576                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3577                 iwqe->wqe_size = 1;
3578
3579                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3580                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3581                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3582                 qp->prev_wqe_size = iwqe->wqe_size;
3583                 break;
3584         case IB_WR_REG_MR:
3585                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3586                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3587                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3588                 fwqe1->wqe_size = 2;
3589
3590                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3591                 if (rc) {
3592                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3593                         *bad_wr = wr;
3594                         break;
3595                 }
3596
3597                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3598                 qp->prev_wqe_size = fwqe1->wqe_size;
3599                 break;
3600         default:
3601                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3602                 rc = -EINVAL;
3603                 *bad_wr = wr;
3604                 break;
3605         }
3606
3607         if (*bad_wr) {
3608                 u16 value;
3609
3610                 /* Restore prod to its position before
3611                  * this WR was processed
3612                  */
3613                 value = le16_to_cpu(qp->sq.db_data.data.value);
3614                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3615
3616                 /* Restore prev_wqe_size */
3617                 qp->prev_wqe_size = wqe->prev_wqe_size;
3618                 rc = -EINVAL;
3619                 DP_ERR(dev, "POST SEND FAILED\n");
3620         }
3621
3622         return rc;
3623 }
3624
3625 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3626                    const struct ib_send_wr **bad_wr)
3627 {
3628         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3629         struct qedr_qp *qp = get_qedr_qp(ibqp);
3630         unsigned long flags;
3631         int rc = 0;
3632
3633         *bad_wr = NULL;
3634
3635         if (qp->qp_type == IB_QPT_GSI)
3636                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3637
3638         spin_lock_irqsave(&qp->q_lock, flags);
3639
3640         if (rdma_protocol_roce(&dev->ibdev, 1)) {
3641                 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3642                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
3643                     (qp->state != QED_ROCE_QP_STATE_SQD)) {
3644                         spin_unlock_irqrestore(&qp->q_lock, flags);
3645                         *bad_wr = wr;
3646                         DP_DEBUG(dev, QEDR_MSG_CQ,
3647                                  "QP in wrong state! QP icid=0x%x state %d\n",
3648                                  qp->icid, qp->state);
3649                         return -EINVAL;
3650                 }
3651         }
3652
3653         while (wr) {
3654                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3655                 if (rc)
3656                         break;
3657
3658                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3659
3660                 qedr_inc_sw_prod(&qp->sq);
3661
3662                 qp->sq.db_data.data.value++;
3663
3664                 wr = wr->next;
3665         }
3666
3667         /* Trigger doorbell
3668          * If there was a failure in the first WR then it will be triggered in
3669          * vane. However this is not harmful (as long as the producer value is
3670          * unchanged). For performance reasons we avoid checking for this
3671          * redundant doorbell.
3672          *
3673          * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3674          * soon as we give the doorbell, we could get a completion
3675          * for this wr, therefore we need to make sure that the
3676          * memory is updated before giving the doorbell.
3677          * During qedr_poll_cq, rmb is called before accessing the
3678          * cqe. This covers for the smp_rmb as well.
3679          */
3680         smp_wmb();
3681         writel(qp->sq.db_data.raw, qp->sq.db);
3682
3683         spin_unlock_irqrestore(&qp->q_lock, flags);
3684
3685         return rc;
3686 }
3687
3688 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3689 {
3690         u32 used;
3691
3692         /* Calculate number of elements used based on producer
3693          * count and consumer count and subtract it from max
3694          * work request supported so that we get elements left.
3695          */
3696         used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3697
3698         return hw_srq->max_wr - used;
3699 }
3700
3701 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3702                        const struct ib_recv_wr **bad_wr)
3703 {
3704         struct qedr_srq *srq = get_qedr_srq(ibsrq);
3705         struct qedr_srq_hwq_info *hw_srq;
3706         struct qedr_dev *dev = srq->dev;
3707         struct qed_chain *pbl;
3708         unsigned long flags;
3709         int status = 0;
3710         u32 num_sge;
3711
3712         spin_lock_irqsave(&srq->lock, flags);
3713
3714         hw_srq = &srq->hw_srq;
3715         pbl = &srq->hw_srq.pbl;
3716         while (wr) {
3717                 struct rdma_srq_wqe_header *hdr;
3718                 int i;
3719
3720                 if (!qedr_srq_elem_left(hw_srq) ||
3721                     wr->num_sge > srq->hw_srq.max_sges) {
3722                         DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3723                                hw_srq->wr_prod_cnt,
3724                                atomic_read(&hw_srq->wr_cons_cnt),
3725                                wr->num_sge, srq->hw_srq.max_sges);
3726                         status = -ENOMEM;
3727                         *bad_wr = wr;
3728                         break;
3729                 }
3730
3731                 hdr = qed_chain_produce(pbl);
3732                 num_sge = wr->num_sge;
3733                 /* Set number of sge and work request id in header */
3734                 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3735
3736                 srq->hw_srq.wr_prod_cnt++;
3737                 hw_srq->wqe_prod++;
3738                 hw_srq->sge_prod++;
3739
3740                 DP_DEBUG(dev, QEDR_MSG_SRQ,
3741                          "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3742                          wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3743
3744                 for (i = 0; i < wr->num_sge; i++) {
3745                         struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3746
3747                         /* Set SGE length, lkey and address */
3748                         SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3749                                     wr->sg_list[i].length, wr->sg_list[i].lkey);
3750
3751                         DP_DEBUG(dev, QEDR_MSG_SRQ,
3752                                  "[%d]: len %d key %x addr %x:%x\n",
3753                                  i, srq_sge->length, srq_sge->l_key,
3754                                  srq_sge->addr.hi, srq_sge->addr.lo);
3755                         hw_srq->sge_prod++;
3756                 }
3757
3758                 /* Update WQE and SGE information before
3759                  * updating producer.
3760                  */
3761                 dma_wmb();
3762
3763                 /* SRQ producer is 8 bytes. Need to update SGE producer index
3764                  * in first 4 bytes and need to update WQE producer in
3765                  * next 4 bytes.
3766                  */
3767                 srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod;
3768                 /* Make sure sge producer is updated first */
3769                 dma_wmb();
3770                 srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod;
3771
3772                 wr = wr->next;
3773         }
3774
3775         DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3776                  qed_chain_get_elem_left(pbl));
3777         spin_unlock_irqrestore(&srq->lock, flags);
3778
3779         return status;
3780 }
3781
3782 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3783                    const struct ib_recv_wr **bad_wr)
3784 {
3785         struct qedr_qp *qp = get_qedr_qp(ibqp);
3786         struct qedr_dev *dev = qp->dev;
3787         unsigned long flags;
3788         int status = 0;
3789
3790         if (qp->qp_type == IB_QPT_GSI)
3791                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3792
3793         spin_lock_irqsave(&qp->q_lock, flags);
3794
3795         if (qp->state == QED_ROCE_QP_STATE_RESET) {
3796                 spin_unlock_irqrestore(&qp->q_lock, flags);
3797                 *bad_wr = wr;
3798                 return -EINVAL;
3799         }
3800
3801         while (wr) {
3802                 int i;
3803
3804                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3805                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3806                     wr->num_sge > qp->rq.max_sges) {
3807                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3808                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3809                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3810                                qp->rq.max_sges);
3811                         status = -ENOMEM;
3812                         *bad_wr = wr;
3813                         break;
3814                 }
3815                 for (i = 0; i < wr->num_sge; i++) {
3816                         u32 flags = 0;
3817                         struct rdma_rq_sge *rqe =
3818                             qed_chain_produce(&qp->rq.pbl);
3819
3820                         /* First one must include the number
3821                          * of SGE in the list
3822                          */
3823                         if (!i)
3824                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3825                                           wr->num_sge);
3826
3827                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3828                                   wr->sg_list[i].lkey);
3829
3830                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3831                                    wr->sg_list[i].length, flags);
3832                 }
3833
3834                 /* Special case of no sges. FW requires between 1-4 sges...
3835                  * in this case we need to post 1 sge with length zero. this is
3836                  * because rdma write with immediate consumes an RQ.
3837                  */
3838                 if (!wr->num_sge) {
3839                         u32 flags = 0;
3840                         struct rdma_rq_sge *rqe =
3841                             qed_chain_produce(&qp->rq.pbl);
3842
3843                         /* First one must include the number
3844                          * of SGE in the list
3845                          */
3846                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3847                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3848
3849                         RQ_SGE_SET(rqe, 0, 0, flags);
3850                         i = 1;
3851                 }
3852
3853                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3854                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3855
3856                 qedr_inc_sw_prod(&qp->rq);
3857
3858                 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3859                  * soon as we give the doorbell, we could get a completion
3860                  * for this wr, therefore we need to make sure that the
3861                  * memory is update before giving the doorbell.
3862                  * During qedr_poll_cq, rmb is called before accessing the
3863                  * cqe. This covers for the smp_rmb as well.
3864                  */
3865                 smp_wmb();
3866
3867                 qp->rq.db_data.data.value++;
3868
3869                 writel(qp->rq.db_data.raw, qp->rq.db);
3870
3871                 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3872                         writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3873                 }
3874
3875                 wr = wr->next;
3876         }
3877
3878         spin_unlock_irqrestore(&qp->q_lock, flags);
3879
3880         return status;
3881 }
3882
3883 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3884 {
3885         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3886
3887         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3888                 cq->pbl_toggle;
3889 }
3890
3891 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3892 {
3893         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3894         struct qedr_qp *qp;
3895
3896         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3897                                                    resp_cqe->qp_handle.lo,
3898                                                    u64);
3899         return qp;
3900 }
3901
3902 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3903 {
3904         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3905
3906         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3907 }
3908
3909 /* Return latest CQE (needs processing) */
3910 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3911 {
3912         return cq->latest_cqe;
3913 }
3914
3915 /* In fmr we need to increase the number of fmr completed counter for the fmr
3916  * algorithm determining whether we can free a pbl or not.
3917  * we need to perform this whether the work request was signaled or not. for
3918  * this purpose we call this function from the condition that checks if a wr
3919  * should be skipped, to make sure we don't miss it ( possibly this fmr
3920  * operation was not signalted)
3921  */
3922 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3923 {
3924         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3925                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3926 }
3927
3928 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3929                        struct qedr_cq *cq, int num_entries,
3930                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3931                        int force)
3932 {
3933         u16 cnt = 0;
3934
3935         while (num_entries && qp->sq.wqe_cons != hw_cons) {
3936                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3937                         qedr_chk_if_fmr(qp);
3938                         /* skip WC */
3939                         goto next_cqe;
3940                 }
3941
3942                 /* fill WC */
3943                 wc->status = status;
3944                 wc->vendor_err = 0;
3945                 wc->wc_flags = 0;
3946                 wc->src_qp = qp->id;
3947                 wc->qp = &qp->ibqp;
3948
3949                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3950                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3951
3952                 switch (wc->opcode) {
3953                 case IB_WC_RDMA_WRITE:
3954                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3955                         break;
3956                 case IB_WC_COMP_SWAP:
3957                 case IB_WC_FETCH_ADD:
3958                         wc->byte_len = 8;
3959                         break;
3960                 case IB_WC_REG_MR:
3961                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3962                         break;
3963                 case IB_WC_RDMA_READ:
3964                 case IB_WC_SEND:
3965                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3966                         break;
3967                 default:
3968                         break;
3969                 }
3970
3971                 num_entries--;
3972                 wc++;
3973                 cnt++;
3974 next_cqe:
3975                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3976                         qed_chain_consume(&qp->sq.pbl);
3977                 qedr_inc_sw_cons(&qp->sq);
3978         }
3979
3980         return cnt;
3981 }
3982
3983 static int qedr_poll_cq_req(struct qedr_dev *dev,
3984                             struct qedr_qp *qp, struct qedr_cq *cq,
3985                             int num_entries, struct ib_wc *wc,
3986                             struct rdma_cqe_requester *req)
3987 {
3988         int cnt = 0;
3989
3990         switch (req->status) {
3991         case RDMA_CQE_REQ_STS_OK:
3992                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3993                                   IB_WC_SUCCESS, 0);
3994                 break;
3995         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3996                 if (qp->state != QED_ROCE_QP_STATE_ERR)
3997                         DP_DEBUG(dev, QEDR_MSG_CQ,
3998                                  "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3999                                  cq->icid, qp->icid);
4000                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4001                                   IB_WC_WR_FLUSH_ERR, 1);
4002                 break;
4003         default:
4004                 /* process all WQE before the cosumer */
4005                 qp->state = QED_ROCE_QP_STATE_ERR;
4006                 cnt = process_req(dev, qp, cq, num_entries, wc,
4007                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
4008                 wc += cnt;
4009                 /* if we have extra WC fill it with actual error info */
4010                 if (cnt < num_entries) {
4011                         enum ib_wc_status wc_status;
4012
4013                         switch (req->status) {
4014                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4015                                 DP_ERR(dev,
4016                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4017                                        cq->icid, qp->icid);
4018                                 wc_status = IB_WC_BAD_RESP_ERR;
4019                                 break;
4020                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4021                                 DP_ERR(dev,
4022                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4023                                        cq->icid, qp->icid);
4024                                 wc_status = IB_WC_LOC_LEN_ERR;
4025                                 break;
4026                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4027                                 DP_ERR(dev,
4028                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4029                                        cq->icid, qp->icid);
4030                                 wc_status = IB_WC_LOC_QP_OP_ERR;
4031                                 break;
4032                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4033                                 DP_ERR(dev,
4034                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4035                                        cq->icid, qp->icid);
4036                                 wc_status = IB_WC_LOC_PROT_ERR;
4037                                 break;
4038                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4039                                 DP_ERR(dev,
4040                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4041                                        cq->icid, qp->icid);
4042                                 wc_status = IB_WC_MW_BIND_ERR;
4043                                 break;
4044                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4045                                 DP_ERR(dev,
4046                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4047                                        cq->icid, qp->icid);
4048                                 wc_status = IB_WC_REM_INV_REQ_ERR;
4049                                 break;
4050                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4051                                 DP_ERR(dev,
4052                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4053                                        cq->icid, qp->icid);
4054                                 wc_status = IB_WC_REM_ACCESS_ERR;
4055                                 break;
4056                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4057                                 DP_ERR(dev,
4058                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4059                                        cq->icid, qp->icid);
4060                                 wc_status = IB_WC_REM_OP_ERR;
4061                                 break;
4062                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4063                                 DP_ERR(dev,
4064                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4065                                        cq->icid, qp->icid);
4066                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4067                                 break;
4068                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4069                                 DP_ERR(dev,
4070                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4071                                        cq->icid, qp->icid);
4072                                 wc_status = IB_WC_RETRY_EXC_ERR;
4073                                 break;
4074                         default:
4075                                 DP_ERR(dev,
4076                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4077                                        cq->icid, qp->icid);
4078                                 wc_status = IB_WC_GENERAL_ERR;
4079                         }
4080                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4081                                            wc_status, 1);
4082                 }
4083         }
4084
4085         return cnt;
4086 }
4087
4088 static inline int qedr_cqe_resp_status_to_ib(u8 status)
4089 {
4090         switch (status) {
4091         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4092                 return IB_WC_LOC_ACCESS_ERR;
4093         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4094                 return IB_WC_LOC_LEN_ERR;
4095         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4096                 return IB_WC_LOC_QP_OP_ERR;
4097         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4098                 return IB_WC_LOC_PROT_ERR;
4099         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4100                 return IB_WC_MW_BIND_ERR;
4101         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4102                 return IB_WC_REM_INV_RD_REQ_ERR;
4103         case RDMA_CQE_RESP_STS_OK:
4104                 return IB_WC_SUCCESS;
4105         default:
4106                 return IB_WC_GENERAL_ERR;
4107         }
4108 }
4109
4110 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4111                                           struct ib_wc *wc)
4112 {
4113         wc->status = IB_WC_SUCCESS;
4114         wc->byte_len = le32_to_cpu(resp->length);
4115
4116         if (resp->flags & QEDR_RESP_IMM) {
4117                 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4118                 wc->wc_flags |= IB_WC_WITH_IMM;
4119
4120                 if (resp->flags & QEDR_RESP_RDMA)
4121                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4122
4123                 if (resp->flags & QEDR_RESP_INV)
4124                         return -EINVAL;
4125
4126         } else if (resp->flags & QEDR_RESP_INV) {
4127                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4128                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4129
4130                 if (resp->flags & QEDR_RESP_RDMA)
4131                         return -EINVAL;
4132
4133         } else if (resp->flags & QEDR_RESP_RDMA) {
4134                 return -EINVAL;
4135         }
4136
4137         return 0;
4138 }
4139
4140 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4141                                struct qedr_cq *cq, struct ib_wc *wc,
4142                                struct rdma_cqe_responder *resp, u64 wr_id)
4143 {
4144         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4145         wc->opcode = IB_WC_RECV;
4146         wc->wc_flags = 0;
4147
4148         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4149                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4150                         DP_ERR(dev,
4151                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4152                                cq, cq->icid, resp->flags);
4153
4154         } else {
4155                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4156                 if (wc->status == IB_WC_GENERAL_ERR)
4157                         DP_ERR(dev,
4158                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4159                                cq, cq->icid, resp->status);
4160         }
4161
4162         /* Fill the rest of the WC */
4163         wc->vendor_err = 0;
4164         wc->src_qp = qp->id;
4165         wc->qp = &qp->ibqp;
4166         wc->wr_id = wr_id;
4167 }
4168
4169 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4170                                 struct qedr_cq *cq, struct ib_wc *wc,
4171                                 struct rdma_cqe_responder *resp)
4172 {
4173         struct qedr_srq *srq = qp->srq;
4174         u64 wr_id;
4175
4176         wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4177                          le32_to_cpu(resp->srq_wr_id.lo), u64);
4178
4179         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4180                 wc->status = IB_WC_WR_FLUSH_ERR;
4181                 wc->vendor_err = 0;
4182                 wc->wr_id = wr_id;
4183                 wc->byte_len = 0;
4184                 wc->src_qp = qp->id;
4185                 wc->qp = &qp->ibqp;
4186                 wc->wr_id = wr_id;
4187         } else {
4188                 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4189         }
4190         atomic_inc(&srq->hw_srq.wr_cons_cnt);
4191
4192         return 1;
4193 }
4194 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4195                             struct qedr_cq *cq, struct ib_wc *wc,
4196                             struct rdma_cqe_responder *resp)
4197 {
4198         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4199
4200         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4201
4202         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4203                 qed_chain_consume(&qp->rq.pbl);
4204         qedr_inc_sw_cons(&qp->rq);
4205
4206         return 1;
4207 }
4208
4209 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4210                               int num_entries, struct ib_wc *wc, u16 hw_cons)
4211 {
4212         u16 cnt = 0;
4213
4214         while (num_entries && qp->rq.wqe_cons != hw_cons) {
4215                 /* fill WC */
4216                 wc->status = IB_WC_WR_FLUSH_ERR;
4217                 wc->vendor_err = 0;
4218                 wc->wc_flags = 0;
4219                 wc->src_qp = qp->id;
4220                 wc->byte_len = 0;
4221                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4222                 wc->qp = &qp->ibqp;
4223                 num_entries--;
4224                 wc++;
4225                 cnt++;
4226                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4227                         qed_chain_consume(&qp->rq.pbl);
4228                 qedr_inc_sw_cons(&qp->rq);
4229         }
4230
4231         return cnt;
4232 }
4233
4234 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4235                                  struct rdma_cqe_responder *resp, int *update)
4236 {
4237         if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4238                 consume_cqe(cq);
4239                 *update |= 1;
4240         }
4241 }
4242
4243 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4244                                  struct qedr_cq *cq, int num_entries,
4245                                  struct ib_wc *wc,
4246                                  struct rdma_cqe_responder *resp)
4247 {
4248         int cnt;
4249
4250         cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4251         consume_cqe(cq);
4252
4253         return cnt;
4254 }
4255
4256 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4257                              struct qedr_cq *cq, int num_entries,
4258                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
4259                              int *update)
4260 {
4261         int cnt;
4262
4263         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4264                 cnt = process_resp_flush(qp, cq, num_entries, wc,
4265                                          resp->rq_cons_or_srq_id);
4266                 try_consume_resp_cqe(cq, qp, resp, update);
4267         } else {
4268                 cnt = process_resp_one(dev, qp, cq, wc, resp);
4269                 consume_cqe(cq);
4270                 *update |= 1;
4271         }
4272
4273         return cnt;
4274 }
4275
4276 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4277                                 struct rdma_cqe_requester *req, int *update)
4278 {
4279         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4280                 consume_cqe(cq);
4281                 *update |= 1;
4282         }
4283 }
4284
4285 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4286 {
4287         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4288         struct qedr_cq *cq = get_qedr_cq(ibcq);
4289         union rdma_cqe *cqe;
4290         u32 old_cons, new_cons;
4291         unsigned long flags;
4292         int update = 0;
4293         int done = 0;
4294
4295         if (cq->destroyed) {
4296                 DP_ERR(dev,
4297                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4298                        cq, cq->icid);
4299                 return 0;
4300         }
4301
4302         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4303                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4304
4305         spin_lock_irqsave(&cq->cq_lock, flags);
4306         cqe = cq->latest_cqe;
4307         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4308         while (num_entries && is_valid_cqe(cq, cqe)) {
4309                 struct qedr_qp *qp;
4310                 int cnt = 0;
4311
4312                 /* prevent speculative reads of any field of CQE */
4313                 rmb();
4314
4315                 qp = cqe_get_qp(cqe);
4316                 if (!qp) {
4317                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4318                         break;
4319                 }
4320
4321                 wc->qp = &qp->ibqp;
4322
4323                 switch (cqe_get_type(cqe)) {
4324                 case RDMA_CQE_TYPE_REQUESTER:
4325                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4326                                                &cqe->req);
4327                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
4328                         break;
4329                 case RDMA_CQE_TYPE_RESPONDER_RQ:
4330                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4331                                                 &cqe->resp, &update);
4332                         break;
4333                 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4334                         cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4335                                                     wc, &cqe->resp);
4336                         update = 1;
4337                         break;
4338                 case RDMA_CQE_TYPE_INVALID:
4339                 default:
4340                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
4341                                cqe_get_type(cqe));
4342                 }
4343                 num_entries -= cnt;
4344                 wc += cnt;
4345                 done += cnt;
4346
4347                 cqe = get_cqe(cq);
4348         }
4349         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4350
4351         cq->cq_cons += new_cons - old_cons;
4352
4353         if (update)
4354                 /* doorbell notifies abount latest VALID entry,
4355                  * but chain already point to the next INVALID one
4356                  */
4357                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4358
4359         spin_unlock_irqrestore(&cq->cq_lock, flags);
4360         return done;
4361 }
4362
4363 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4364                      u8 port_num, const struct ib_wc *in_wc,
4365                      const struct ib_grh *in_grh, const struct ib_mad *in,
4366                      struct ib_mad *out_mad, size_t *out_mad_size,
4367                      u16 *out_mad_pkey_index)
4368 {
4369         return IB_MAD_RESULT_SUCCESS;
4370 }