drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45 #include <rdma/uverbs_ioctl.h>
  46
  47 #include <linux/qed/common_hsi.h>
  48 #include "qedr_hsi_rdma.h"
  49 #include <linux/qed/qed_if.h>
  50 #include "qedr.h"
  51 #include "verbs.h"
  52 #include <rdma/qedr-abi.h>
  53 #include "qedr_roce_cm.h"
  54 #include "qedr_iw_cm.h"
  55
  56 #define QEDR_SRQ_WQE_ELEM_SIZE  sizeof(union rdma_srq_elm)
  57 #define RDMA_MAX_SGE_PER_SRQ    (4)
  58 #define RDMA_MAX_SRQ_WQE_SIZE   (RDMA_MAX_SGE_PER_SRQ + 1)
  59
  60 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  61
  62 enum {
  63         QEDR_USER_MMAP_IO_WC = 0,
  64         QEDR_USER_MMAP_PHYS_PAGE,
  65 };
  66
  67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  68                                         size_t len)
  69 {
  70         size_t min_len = min_t(size_t, len, udata->outlen);
  71
  72         return ib_copy_to_udata(udata, src, min_len);
  73 }
  74
  75 int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
  76 {
  77         if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  78                 return -EINVAL;
  79
  80         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  81         return 0;
  82 }
  83
  84 int qedr_iw_query_gid(struct ib_device *ibdev, u32 port,
  85                       int index, union ib_gid *sgid)
  86 {
  87         struct qedr_dev *dev = get_qedr_dev(ibdev);
  88
  89         memset(sgid->raw, 0, sizeof(sgid->raw));
  90         ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  91
  92         DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  93                  sgid->global.interface_id, sgid->global.subnet_prefix);
  94
  95         return 0;
  96 }
  97
  98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  99 {
 100         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
 101         struct qedr_device_attr *qattr = &dev->attr;
 102         struct qedr_srq *srq = get_qedr_srq(ibsrq);
 103
 104         srq_attr->srq_limit = srq->srq_limit;
 105         srq_attr->max_wr = qattr->max_srq_wr;
 106         srq_attr->max_sge = qattr->max_sge;
 107
 108         return 0;
 109 }
 110
 111 int qedr_query_device(struct ib_device *ibdev,
 112                       struct ib_device_attr *attr, struct ib_udata *udata)
 113 {
 114         struct qedr_dev *dev = get_qedr_dev(ibdev);
 115         struct qedr_device_attr *qattr = &dev->attr;
 116
 117         if (!dev->rdma_ctx) {
 118                 DP_ERR(dev,
 119                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 120                        dev->rdma_ctx);
 121                 return -EINVAL;
 122         }
 123
 124         memset(attr, 0, sizeof(*attr));
 125
 126         attr->fw_ver = qattr->fw_ver;
 127         attr->sys_image_guid = qattr->sys_image_guid;
 128         attr->max_mr_size = qattr->max_mr_size;
 129         attr->page_size_cap = qattr->page_size_caps;
 130         attr->vendor_id = qattr->vendor_id;
 131         attr->vendor_part_id = qattr->vendor_part_id;
 132         attr->hw_ver = qattr->hw_ver;
 133         attr->max_qp = qattr->max_qp;
 134         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 135         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 136             IB_DEVICE_RC_RNR_NAK_GEN |
 137             IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
 138
 139         if (!rdma_protocol_iwarp(&dev->ibdev, 1))
 140                 attr->device_cap_flags |= IB_DEVICE_XRC;
 141         attr->max_send_sge = qattr->max_sge;
 142         attr->max_recv_sge = qattr->max_sge;
 143         attr->max_sge_rd = qattr->max_sge;
 144         attr->max_cq = qattr->max_cq;
 145         attr->max_cqe = qattr->max_cqe;
 146         attr->max_mr = qattr->max_mr;
 147         attr->max_mw = qattr->max_mw;
 148         attr->max_pd = qattr->max_pd;
 149         attr->atomic_cap = dev->atomic_cap;
 150         attr->max_qp_init_rd_atom =
 151             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 152         attr->max_qp_rd_atom =
 153             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 154                 attr->max_qp_init_rd_atom);
 155
 156         attr->max_srq = qattr->max_srq;
 157         attr->max_srq_sge = qattr->max_srq_sge;
 158         attr->max_srq_wr = qattr->max_srq_wr;
 159
 160         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 161         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 162         attr->max_pkeys = qattr->max_pkey;
 163         attr->max_ah = qattr->max_ah;
 164
 165         return 0;
 166 }
 167
 168 static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
 169                                             u8 *ib_width)
 170 {
 171         switch (speed) {
 172         case 1000:
 173                 *ib_speed = IB_SPEED_SDR;
 174                 *ib_width = IB_WIDTH_1X;
 175                 break;
 176         case 10000:
 177                 *ib_speed = IB_SPEED_QDR;
 178                 *ib_width = IB_WIDTH_1X;
 179                 break;
 180
 181         case 20000:
 182                 *ib_speed = IB_SPEED_DDR;
 183                 *ib_width = IB_WIDTH_4X;
 184                 break;
 185
 186         case 25000:
 187                 *ib_speed = IB_SPEED_EDR;
 188                 *ib_width = IB_WIDTH_1X;
 189                 break;
 190
 191         case 40000:
 192                 *ib_speed = IB_SPEED_QDR;
 193                 *ib_width = IB_WIDTH_4X;
 194                 break;
 195
 196         case 50000:
 197                 *ib_speed = IB_SPEED_HDR;
 198                 *ib_width = IB_WIDTH_1X;
 199                 break;
 200
 201         case 100000:
 202                 *ib_speed = IB_SPEED_EDR;
 203                 *ib_width = IB_WIDTH_4X;
 204                 break;
 205
 206         default:
 207                 /* Unsupported */
 208                 *ib_speed = IB_SPEED_SDR;
 209                 *ib_width = IB_WIDTH_1X;
 210         }
 211 }
 212
 213 int qedr_query_port(struct ib_device *ibdev, u32 port,
 214                     struct ib_port_attr *attr)
 215 {
 216         struct qedr_dev *dev;
 217         struct qed_rdma_port *rdma_port;
 218
 219         dev = get_qedr_dev(ibdev);
 220
 221         if (!dev->rdma_ctx) {
 222                 DP_ERR(dev, "rdma_ctx is NULL\n");
 223                 return -EINVAL;
 224         }
 225
 226         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 227
 228         /* *attr being zeroed by the caller, avoid zeroing it here */
 229         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 230                 attr->state = IB_PORT_ACTIVE;
 231                 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 232         } else {
 233                 attr->state = IB_PORT_DOWN;
 234                 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 235         }
 236         attr->max_mtu = IB_MTU_4096;
 237         attr->lid = 0;
 238         attr->lmc = 0;
 239         attr->sm_lid = 0;
 240         attr->sm_sl = 0;
 241         attr->ip_gids = true;
 242         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 243                 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
 244                 attr->gid_tbl_len = 1;
 245         } else {
 246                 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 247                 attr->gid_tbl_len = QEDR_MAX_SGID;
 248                 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 249         }
 250         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 251         attr->qkey_viol_cntr = 0;
 252         get_link_speed_and_width(rdma_port->link_speed,
 253                                  &attr->active_speed, &attr->active_width);
 254         attr->max_msg_sz = rdma_port->max_msg_size;
 255         attr->max_vl_num = 4;
 256
 257         return 0;
 258 }
 259
 260 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 261 {
 262         struct ib_device *ibdev = uctx->device;
 263         int rc;
 264         struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
 265         struct qedr_alloc_ucontext_resp uresp = {};
 266         struct qedr_alloc_ucontext_req ureq = {};
 267         struct qedr_dev *dev = get_qedr_dev(ibdev);
 268         struct qed_rdma_add_user_out_params oparams;
 269         struct qedr_user_mmap_entry *entry;
 270
 271         if (!udata)
 272                 return -EFAULT;
 273
 274         if (udata->inlen) {
 275                 rc = ib_copy_from_udata(&ureq, udata,
 276                                         min(sizeof(ureq), udata->inlen));
 277                 if (rc) {
 278                         DP_ERR(dev, "Problem copying data from user space\n");
 279                         return -EFAULT;
 280                 }
 281                 ctx->edpm_mode = !!(ureq.context_flags &
 282                                     QEDR_ALLOC_UCTX_EDPM_MODE);
 283                 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
 284         }
 285
 286         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 287         if (rc) {
 288                 DP_ERR(dev,
 289                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 290                        rc);
 291                 return rc;
 292         }
 293
 294         ctx->dpi = oparams.dpi;
 295         ctx->dpi_addr = oparams.dpi_addr;
 296         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 297         ctx->dpi_size = oparams.dpi_size;
 298         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 299         if (!entry) {
 300                 rc = -ENOMEM;
 301                 goto err;
 302         }
 303
 304         entry->io_address = ctx->dpi_phys_addr;
 305         entry->length = ctx->dpi_size;
 306         entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
 307         entry->dpi = ctx->dpi;
 308         entry->dev = dev;
 309         rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
 310                                          ctx->dpi_size);
 311         if (rc) {
 312                 kfree(entry);
 313                 goto err;
 314         }
 315         ctx->db_mmap_entry = &entry->rdma_entry;
 316
 317         if (!dev->user_dpm_enabled)
 318                 uresp.dpm_flags = 0;
 319         else if (rdma_protocol_iwarp(&dev->ibdev, 1))
 320                 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
 321         else
 322                 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
 323                                   QEDR_DPM_TYPE_ROCE_LEGACY |
 324                                   QEDR_DPM_TYPE_ROCE_EDPM_MODE;
 325
 326         if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
 327                 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
 328                 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
 329                 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
 330                 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
 331         }
 332
 333         uresp.wids_enabled = 1;
 334         uresp.wid_count = oparams.wid_count;
 335         uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
 336         uresp.db_size = ctx->dpi_size;
 337         uresp.max_send_wr = dev->attr.max_sqe;
 338         uresp.max_recv_wr = dev->attr.max_rqe;
 339         uresp.max_srq_wr = dev->attr.max_srq_wr;
 340         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 341         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 342         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 343         uresp.max_cqes = QEDR_MAX_CQES;
 344
 345         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 346         if (rc)
 347                 goto err;
 348
 349         ctx->dev = dev;
 350
 351         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 352                  &ctx->ibucontext);
 353         return 0;
 354
 355 err:
 356         if (!ctx->db_mmap_entry)
 357                 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
 358         else
 359                 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
 360
 361         return rc;
 362 }
 363
 364 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 365 {
 366         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 367
 368         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 369                  uctx);
 370
 371         rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
 372 }
 373
 374 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
 375 {
 376         struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
 377         struct qedr_dev *dev = entry->dev;
 378
 379         if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
 380                 free_page((unsigned long)entry->address);
 381         else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
 382                 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
 383
 384         kfree(entry);
 385 }
 386
 387 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
 388 {
 389         struct ib_device *dev = ucontext->device;
 390         size_t length = vma->vm_end - vma->vm_start;
 391         struct rdma_user_mmap_entry *rdma_entry;
 392         struct qedr_user_mmap_entry *entry;
 393         int rc = 0;
 394         u64 pfn;
 395
 396         ibdev_dbg(dev,
 397                   "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
 398                   vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
 399
 400         rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
 401         if (!rdma_entry) {
 402                 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
 403                           vma->vm_pgoff);
 404                 return -EINVAL;
 405         }
 406         entry = get_qedr_mmap_entry(rdma_entry);
 407         ibdev_dbg(dev,
 408                   "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
 409                   entry->io_address, length, entry->mmap_flag);
 410
 411         switch (entry->mmap_flag) {
 412         case QEDR_USER_MMAP_IO_WC:
 413                 pfn = entry->io_address >> PAGE_SHIFT;
 414                 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
 415                                        pgprot_writecombine(vma->vm_page_prot),
 416                                        rdma_entry);
 417                 break;
 418         case QEDR_USER_MMAP_PHYS_PAGE:
 419                 rc = vm_insert_page(vma, vma->vm_start,
 420                                     virt_to_page(entry->address));
 421                 break;
 422         default:
 423                 rc = -EINVAL;
 424         }
 425
 426         if (rc)
 427                 ibdev_dbg(dev,
 428                           "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 429                           entry->io_address, length, entry->mmap_flag, rc);
 430
 431         rdma_user_mmap_entry_put(rdma_entry);
 432         return rc;
 433 }
 434
 435 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 436 {
 437         struct ib_device *ibdev = ibpd->device;
 438         struct qedr_dev *dev = get_qedr_dev(ibdev);
 439         struct qedr_pd *pd = get_qedr_pd(ibpd);
 440         u16 pd_id;
 441         int rc;
 442
 443         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 444                  udata ? "User Lib" : "Kernel");
 445
 446         if (!dev->rdma_ctx) {
 447                 DP_ERR(dev, "invalid RDMA context\n");
 448                 return -EINVAL;
 449         }
 450
 451         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 452         if (rc)
 453                 return rc;
 454
 455         pd->pd_id = pd_id;
 456
 457         if (udata) {
 458                 struct qedr_alloc_pd_uresp uresp = {
 459                         .pd_id = pd_id,
 460                 };
 461                 struct qedr_ucontext *context = rdma_udata_to_drv_context(
 462                         udata, struct qedr_ucontext, ibucontext);
 463
 464                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 465                 if (rc) {
 466                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 467                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 468                         return rc;
 469                 }
 470
 471                 pd->uctx = context;
 472                 pd->uctx->pd = pd;
 473         }
 474
 475         return 0;
 476 }
 477
 478 int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 479 {
 480         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 481         struct qedr_pd *pd = get_qedr_pd(ibpd);
 482
 483         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 484         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 485         return 0;
 486 }
 487
 488
 489 int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 490 {
 491         struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 492         struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
 493
 494         return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
 495 }
 496
 497 int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 498 {
 499         struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 500         u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
 501
 502         dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
 503         return 0;
 504 }
 505 static void qedr_free_pbl(struct qedr_dev *dev,
 506                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 507 {
 508         struct pci_dev *pdev = dev->pdev;
 509         int i;
 510
 511         for (i = 0; i < pbl_info->num_pbls; i++) {
 512                 if (!pbl[i].va)
 513                         continue;
 514                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 515                                   pbl[i].va, pbl[i].pa);
 516         }
 517
 518         kfree(pbl);
 519 }
 520
 521 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 522 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 523
 524 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 525 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 526 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 527
 528 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 529                                            struct qedr_pbl_info *pbl_info,
 530                                            gfp_t flags)
 531 {
 532         struct pci_dev *pdev = dev->pdev;
 533         struct qedr_pbl *pbl_table;
 534         dma_addr_t *pbl_main_tbl;
 535         dma_addr_t pa;
 536         void *va;
 537         int i;
 538
 539         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 540         if (!pbl_table)
 541                 return ERR_PTR(-ENOMEM);
 542
 543         for (i = 0; i < pbl_info->num_pbls; i++) {
 544                 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
 545                                         flags);
 546                 if (!va)
 547                         goto err;
 548
 549                 pbl_table[i].va = va;
 550                 pbl_table[i].pa = pa;
 551         }
 552
 553         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 554          * the first one with physical pointers to all of the rest
 555          */
 556         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 557         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 558                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 559
 560         return pbl_table;
 561
 562 err:
 563         for (i--; i >= 0; i--)
 564                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 565                                   pbl_table[i].va, pbl_table[i].pa);
 566
 567         qedr_free_pbl(dev, pbl_info, pbl_table);
 568
 569         return ERR_PTR(-ENOMEM);
 570 }
 571
 572 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 573                                 struct qedr_pbl_info *pbl_info,
 574                                 u32 num_pbes, int two_layer_capable)
 575 {
 576         u32 pbl_capacity;
 577         u32 pbl_size;
 578         u32 num_pbls;
 579
 580         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 581                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 582                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 583                                num_pbes);
 584                         return -EINVAL;
 585                 }
 586
 587                 /* calculate required pbl page size */
 588                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 589                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 590                                NUM_PBES_ON_PAGE(pbl_size);
 591
 592                 while (pbl_capacity < num_pbes) {
 593                         pbl_size *= 2;
 594                         pbl_capacity = pbl_size / sizeof(u64);
 595                         pbl_capacity = pbl_capacity * pbl_capacity;
 596                 }
 597
 598                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 599                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 600                 pbl_info->two_layered = true;
 601         } else {
 602                 /* One layered PBL */
 603                 num_pbls = 1;
 604                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 605                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 606                 pbl_info->two_layered = false;
 607         }
 608
 609         pbl_info->num_pbls = num_pbls;
 610         pbl_info->pbl_size = pbl_size;
 611         pbl_info->num_pbes = num_pbes;
 612
 613         DP_DEBUG(dev, QEDR_MSG_MR,
 614                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 615                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 616
 617         return 0;
 618 }
 619
 620 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 621                                struct qedr_pbl *pbl,
 622                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 623 {
 624         int pbe_cnt, total_num_pbes = 0;
 625         struct qedr_pbl *pbl_tbl;
 626         struct ib_block_iter biter;
 627         struct regpair *pbe;
 628
 629         if (!pbl_info->num_pbes)
 630                 return;
 631
 632         /* If we have a two layered pbl, the first pbl points to the rest
 633          * of the pbls and the first entry lays on the second pbl in the table
 634          */
 635         if (pbl_info->two_layered)
 636                 pbl_tbl = &pbl[1];
 637         else
 638                 pbl_tbl = pbl;
 639
 640         pbe = (struct regpair *)pbl_tbl->va;
 641         if (!pbe) {
 642                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 643                 return;
 644         }
 645
 646         pbe_cnt = 0;
 647
 648         rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
 649                 u64 pg_addr = rdma_block_iter_dma_address(&biter);
 650
 651                 pbe->lo = cpu_to_le32(pg_addr);
 652                 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 653
 654                 pbe_cnt++;
 655                 total_num_pbes++;
 656                 pbe++;
 657
 658                 if (total_num_pbes == pbl_info->num_pbes)
 659                         return;
 660
 661                 /* If the given pbl is full storing the pbes, move to next pbl.
 662                  */
 663                 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
 664                         pbl_tbl++;
 665                         pbe = (struct regpair *)pbl_tbl->va;
 666                         pbe_cnt = 0;
 667                 }
 668         }
 669 }
 670
 671 static int qedr_db_recovery_add(struct qedr_dev *dev,
 672                                 void __iomem *db_addr,
 673                                 void *db_data,
 674                                 enum qed_db_rec_width db_width,
 675                                 enum qed_db_rec_space db_space)
 676 {
 677         if (!db_data) {
 678                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 679                 return 0;
 680         }
 681
 682         return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
 683                                                  db_width, db_space);
 684 }
 685
 686 static void qedr_db_recovery_del(struct qedr_dev *dev,
 687                                  void __iomem *db_addr,
 688                                  void *db_data)
 689 {
 690         if (!db_data) {
 691                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 692                 return;
 693         }
 694
 695         /* Ignore return code as there is not much we can do about it. Error
 696          * log will be printed inside.
 697          */
 698         dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
 699 }
 700
 701 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 702                               struct qedr_cq *cq, struct ib_udata *udata,
 703                               u32 db_offset)
 704 {
 705         struct qedr_create_cq_uresp uresp;
 706         int rc;
 707
 708         memset(&uresp, 0, sizeof(uresp));
 709
 710         uresp.db_offset = db_offset;
 711         uresp.icid = cq->icid;
 712         if (cq->q.db_mmap_entry)
 713                 uresp.db_rec_addr =
 714                         rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
 715
 716         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 717         if (rc)
 718                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 719
 720         return rc;
 721 }
 722
 723 static void consume_cqe(struct qedr_cq *cq)
 724 {
 725         if (cq->latest_cqe == cq->toggle_cqe)
 726                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 727
 728         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 729 }
 730
 731 static inline int qedr_align_cq_entries(int entries)
 732 {
 733         u64 size, aligned_size;
 734
 735         /* We allocate an extra entry that we don't report to the FW. */
 736         size = (entries + 1) * QEDR_CQE_SIZE;
 737         aligned_size = ALIGN(size, PAGE_SIZE);
 738
 739         return aligned_size / QEDR_CQE_SIZE;
 740 }
 741
 742 static int qedr_init_user_db_rec(struct ib_udata *udata,
 743                                  struct qedr_dev *dev, struct qedr_userq *q,
 744                                  bool requires_db_rec)
 745 {
 746         struct qedr_ucontext *uctx =
 747                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
 748                                           ibucontext);
 749         struct qedr_user_mmap_entry *entry;
 750         int rc;
 751
 752         /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
 753         if (requires_db_rec == 0 || !uctx->db_rec)
 754                 return 0;
 755
 756         /* Allocate a page for doorbell recovery, add to mmap */
 757         q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
 758         if (!q->db_rec_data) {
 759                 DP_ERR(dev, "get_zeroed_page failed\n");
 760                 return -ENOMEM;
 761         }
 762
 763         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 764         if (!entry)
 765                 goto err_free_db_data;
 766
 767         entry->address = q->db_rec_data;
 768         entry->length = PAGE_SIZE;
 769         entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
 770         rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
 771                                          &entry->rdma_entry,
 772                                          PAGE_SIZE);
 773         if (rc)
 774                 goto err_free_entry;
 775
 776         q->db_mmap_entry = &entry->rdma_entry;
 777
 778         return 0;
 779
 780 err_free_entry:
 781         kfree(entry);
 782
 783 err_free_db_data:
 784         free_page((unsigned long)q->db_rec_data);
 785         q->db_rec_data = NULL;
 786         return -ENOMEM;
 787 }
 788
 789 static inline int qedr_init_user_queue(struct ib_udata *udata,
 790                                        struct qedr_dev *dev,
 791                                        struct qedr_userq *q, u64 buf_addr,
 792                                        size_t buf_len, bool requires_db_rec,
 793                                        int access,
 794                                        int alloc_and_init)
 795 {
 796         u32 fw_pages;
 797         int rc;
 798
 799         q->buf_addr = buf_addr;
 800         q->buf_len = buf_len;
 801         q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
 802         if (IS_ERR(q->umem)) {
 803                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 804                        PTR_ERR(q->umem));
 805                 return PTR_ERR(q->umem);
 806         }
 807
 808         fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
 809         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 810         if (rc)
 811                 goto err0;
 812
 813         if (alloc_and_init) {
 814                 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 815                 if (IS_ERR(q->pbl_tbl)) {
 816                         rc = PTR_ERR(q->pbl_tbl);
 817                         goto err0;
 818                 }
 819                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 820                                    FW_PAGE_SHIFT);
 821         } else {
 822                 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 823                 if (!q->pbl_tbl) {
 824                         rc = -ENOMEM;
 825                         goto err0;
 826                 }
 827         }
 828
 829         /* mmap the user address used to store doorbell data for recovery */
 830         return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
 831
 832 err0:
 833         ib_umem_release(q->umem);
 834         q->umem = NULL;
 835
 836         return rc;
 837 }
 838
 839 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 840                                        struct qedr_ucontext *ctx,
 841                                        struct qedr_dev *dev, int vector,
 842                                        int chain_entries, int page_cnt,
 843                                        u64 pbl_ptr,
 844                                        struct qed_rdma_create_cq_in_params
 845                                        *params)
 846 {
 847         memset(params, 0, sizeof(*params));
 848         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 849         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 850         params->cnq_id = vector;
 851         params->cq_size = chain_entries - 1;
 852         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 853         params->pbl_num_pages = page_cnt;
 854         params->pbl_ptr = pbl_ptr;
 855         params->pbl_two_level = 0;
 856 }
 857
 858 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 859 {
 860         cq->db.data.agg_flags = flags;
 861         cq->db.data.value = cpu_to_le32(cons);
 862         writeq(cq->db.raw, cq->db_addr);
 863 }
 864
 865 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 866 {
 867         struct qedr_cq *cq = get_qedr_cq(ibcq);
 868         unsigned long sflags;
 869         struct qedr_dev *dev;
 870
 871         dev = get_qedr_dev(ibcq->device);
 872
 873         if (cq->destroyed) {
 874                 DP_ERR(dev,
 875                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 876                        cq, cq->icid);
 877                 return -EINVAL;
 878         }
 879
 880
 881         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 882                 return 0;
 883
 884         spin_lock_irqsave(&cq->cq_lock, sflags);
 885
 886         cq->arm_flags = 0;
 887
 888         if (flags & IB_CQ_SOLICITED)
 889                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 890
 891         if (flags & IB_CQ_NEXT_COMP)
 892                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 893
 894         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 895
 896         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 897
 898         return 0;
 899 }
 900
 901 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 902                    struct ib_udata *udata)
 903 {
 904         struct ib_device *ibdev = ibcq->device;
 905         struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
 906                 udata, struct qedr_ucontext, ibucontext);
 907         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 908         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 909         struct qed_chain_init_params chain_params = {
 910                 .mode           = QED_CHAIN_MODE_PBL,
 911                 .intended_use   = QED_CHAIN_USE_TO_CONSUME,
 912                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
 913                 .elem_size      = sizeof(union rdma_cqe),
 914         };
 915         struct qedr_dev *dev = get_qedr_dev(ibdev);
 916         struct qed_rdma_create_cq_in_params params;
 917         struct qedr_create_cq_ureq ureq = {};
 918         int vector = attr->comp_vector;
 919         int entries = attr->cqe;
 920         struct qedr_cq *cq = get_qedr_cq(ibcq);
 921         int chain_entries;
 922         u32 db_offset;
 923         int page_cnt;
 924         u64 pbl_ptr;
 925         u16 icid;
 926         int rc;
 927
 928         DP_DEBUG(dev, QEDR_MSG_INIT,
 929                  "create_cq: called from %s. entries=%d, vector=%d\n",
 930                  udata ? "User Lib" : "Kernel", entries, vector);
 931
 932         if (attr->flags)
 933                 return -EOPNOTSUPP;
 934
 935         if (entries > QEDR_MAX_CQES) {
 936                 DP_ERR(dev,
 937                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 938                        entries, QEDR_MAX_CQES);
 939                 return -EINVAL;
 940         }
 941
 942         chain_entries = qedr_align_cq_entries(entries);
 943         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 944         chain_params.num_elems = chain_entries;
 945
 946         /* calc db offset. user will add DPI base, kernel will add db addr */
 947         db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 948
 949         if (udata) {
 950                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
 951                                                          udata->inlen))) {
 952                         DP_ERR(dev,
 953                                "create cq: problem copying data from user space\n");
 954                         goto err0;
 955                 }
 956
 957                 if (!ureq.len) {
 958                         DP_ERR(dev,
 959                                "create cq: cannot create a cq with 0 entries\n");
 960                         goto err0;
 961                 }
 962
 963                 cq->cq_type = QEDR_CQ_TYPE_USER;
 964
 965                 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 966                                           ureq.len, true, IB_ACCESS_LOCAL_WRITE,
 967                                           1);
 968                 if (rc)
 969                         goto err0;
 970
 971                 pbl_ptr = cq->q.pbl_tbl->pa;
 972                 page_cnt = cq->q.pbl_info.num_pbes;
 973
 974                 cq->ibcq.cqe = chain_entries;
 975                 cq->q.db_addr = ctx->dpi_addr + db_offset;
 976         } else {
 977                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 978
 979                 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
 980                                                    &chain_params);
 981                 if (rc)
 982                         goto err0;
 983
 984                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 985                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 986                 cq->ibcq.cqe = cq->pbl.capacity;
 987         }
 988
 989         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 990                             pbl_ptr, &params);
 991
 992         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 993         if (rc)
 994                 goto err1;
 995
 996         cq->icid = icid;
 997         cq->sig = QEDR_CQ_MAGIC_NUMBER;
 998         spin_lock_init(&cq->cq_lock);
 999
1000         if (udata) {
1001                 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1002                 if (rc)
1003                         goto err2;
1004
1005                 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1006                                           &cq->q.db_rec_data->db_data,
1007                                           DB_REC_WIDTH_64B,
1008                                           DB_REC_USER);
1009                 if (rc)
1010                         goto err2;
1011
1012         } else {
1013                 /* Generate doorbell address. */
1014                 cq->db.data.icid = cq->icid;
1015                 cq->db_addr = dev->db_addr + db_offset;
1016                 cq->db.data.params = DB_AGG_CMD_MAX <<
1017                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1018
1019                 /* point to the very last element, passing it we will toggle */
1020                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1021                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1022                 cq->latest_cqe = NULL;
1023                 consume_cqe(cq);
1024                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1025
1026                 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1027                                           DB_REC_WIDTH_64B, DB_REC_KERNEL);
1028                 if (rc)
1029                         goto err2;
1030         }
1031
1032         DP_DEBUG(dev, QEDR_MSG_CQ,
1033                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1034                  cq->icid, cq, params.cq_size);
1035
1036         return 0;
1037
1038 err2:
1039         destroy_iparams.icid = cq->icid;
1040         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1041                                   &destroy_oparams);
1042 err1:
1043         if (udata) {
1044                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1045                 ib_umem_release(cq->q.umem);
1046                 if (cq->q.db_mmap_entry)
1047                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1048         } else {
1049                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1050         }
1051 err0:
1052         return -EINVAL;
1053 }
1054
1055 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1056 {
1057         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1058         struct qedr_cq *cq = get_qedr_cq(ibcq);
1059
1060         DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1061
1062         return 0;
1063 }
1064
1065 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1066 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1067
1068 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1069 {
1070         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1071         struct qed_rdma_destroy_cq_out_params oparams;
1072         struct qed_rdma_destroy_cq_in_params iparams;
1073         struct qedr_cq *cq = get_qedr_cq(ibcq);
1074         int iter;
1075
1076         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1077
1078         cq->destroyed = 1;
1079
1080         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1081         if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1082                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1083                 return 0;
1084         }
1085
1086         iparams.icid = cq->icid;
1087         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1088         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1089
1090         if (udata) {
1091                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1092                 ib_umem_release(cq->q.umem);
1093
1094                 if (cq->q.db_rec_data) {
1095                         qedr_db_recovery_del(dev, cq->q.db_addr,
1096                                              &cq->q.db_rec_data->db_data);
1097                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1098                 }
1099         } else {
1100                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1101         }
1102
1103         /* We don't want the IRQ handler to handle a non-existing CQ so we
1104          * wait until all CNQ interrupts, if any, are received. This will always
1105          * happen and will always happen very fast. If not, then a serious error
1106          * has occured. That is why we can use a long delay.
1107          * We spin for a short time so we don’t lose time on context switching
1108          * in case all the completions are handled in that span. Otherwise
1109          * we sleep for a while and check again. Since the CNQ may be
1110          * associated with (only) the current CPU we use msleep to allow the
1111          * current CPU to be freed.
1112          * The CNQ notification is increased in qedr_irq_handler().
1113          */
1114         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1115         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1116                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1117                 iter--;
1118         }
1119
1120         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1121         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1122                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1123                 iter--;
1124         }
1125
1126         /* Note that we don't need to have explicit code to wait for the
1127          * completion of the event handler because it is invoked from the EQ.
1128          * Since the destroy CQ ramrod has also been received on the EQ we can
1129          * be certain that there's no event handler in process.
1130          */
1131         return 0;
1132 }
1133
1134 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1135                                           struct ib_qp_attr *attr,
1136                                           int attr_mask,
1137                                           struct qed_rdma_modify_qp_in_params
1138                                           *qp_params)
1139 {
1140         const struct ib_gid_attr *gid_attr;
1141         enum rdma_network_type nw_type;
1142         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1143         u32 ipv4_addr;
1144         int ret;
1145         int i;
1146
1147         gid_attr = grh->sgid_attr;
1148         ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1149         if (ret)
1150                 return ret;
1151
1152         nw_type = rdma_gid_attr_network_type(gid_attr);
1153         switch (nw_type) {
1154         case RDMA_NETWORK_IPV6:
1155                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1156                        sizeof(qp_params->sgid));
1157                 memcpy(&qp_params->dgid.bytes[0],
1158                        &grh->dgid,
1159                        sizeof(qp_params->dgid));
1160                 qp_params->roce_mode = ROCE_V2_IPV6;
1161                 SET_FIELD(qp_params->modify_flags,
1162                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1163                 break;
1164         case RDMA_NETWORK_ROCE_V1:
1165                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1166                        sizeof(qp_params->sgid));
1167                 memcpy(&qp_params->dgid.bytes[0],
1168                        &grh->dgid,
1169                        sizeof(qp_params->dgid));
1170                 qp_params->roce_mode = ROCE_V1;
1171                 break;
1172         case RDMA_NETWORK_IPV4:
1173                 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1174                 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1175                 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1176                 qp_params->sgid.ipv4_addr = ipv4_addr;
1177                 ipv4_addr =
1178                     qedr_get_ipv4_from_gid(grh->dgid.raw);
1179                 qp_params->dgid.ipv4_addr = ipv4_addr;
1180                 SET_FIELD(qp_params->modify_flags,
1181                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1182                 qp_params->roce_mode = ROCE_V2_IPV4;
1183                 break;
1184         default:
1185                 return -EINVAL;
1186         }
1187
1188         for (i = 0; i < 4; i++) {
1189                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1190                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1191         }
1192
1193         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1194                 qp_params->vlan_id = 0;
1195
1196         return 0;
1197 }
1198
1199 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1200                                struct ib_qp_init_attr *attrs,
1201                                struct ib_udata *udata)
1202 {
1203         struct qedr_device_attr *qattr = &dev->attr;
1204
1205         /* QP0... attrs->qp_type == IB_QPT_GSI */
1206         if (attrs->qp_type != IB_QPT_RC &&
1207             attrs->qp_type != IB_QPT_GSI &&
1208             attrs->qp_type != IB_QPT_XRC_INI &&
1209             attrs->qp_type != IB_QPT_XRC_TGT) {
1210                 DP_DEBUG(dev, QEDR_MSG_QP,
1211                          "create qp: unsupported qp type=0x%x requested\n",
1212                          attrs->qp_type);
1213                 return -EOPNOTSUPP;
1214         }
1215
1216         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1217                 DP_ERR(dev,
1218                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1219                        attrs->cap.max_send_wr, qattr->max_sqe);
1220                 return -EINVAL;
1221         }
1222
1223         if (attrs->cap.max_inline_data > qattr->max_inline) {
1224                 DP_ERR(dev,
1225                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1226                        attrs->cap.max_inline_data, qattr->max_inline);
1227                 return -EINVAL;
1228         }
1229
1230         if (attrs->cap.max_send_sge > qattr->max_sge) {
1231                 DP_ERR(dev,
1232                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1233                        attrs->cap.max_send_sge, qattr->max_sge);
1234                 return -EINVAL;
1235         }
1236
1237         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1238                 DP_ERR(dev,
1239                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1240                        attrs->cap.max_recv_sge, qattr->max_sge);
1241                 return -EINVAL;
1242         }
1243
1244         /* verify consumer QPs are not trying to use GSI QP's CQ.
1245          * TGT QP isn't associated with RQ/SQ
1246          */
1247         if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1248             (attrs->qp_type != IB_QPT_XRC_TGT) &&
1249             (attrs->qp_type != IB_QPT_XRC_INI)) {
1250                 struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);
1251                 struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);
1252
1253                 if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1254                     (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1255                         DP_ERR(dev,
1256                                "create qp: consumer QP cannot use GSI CQs.\n");
1257                         return -EINVAL;
1258                 }
1259         }
1260
1261         return 0;
1262 }
1263
1264 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1265                                struct qedr_srq *srq, struct ib_udata *udata)
1266 {
1267         struct qedr_create_srq_uresp uresp = {};
1268         int rc;
1269
1270         uresp.srq_id = srq->srq_id;
1271
1272         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1273         if (rc)
1274                 DP_ERR(dev, "create srq: problem copying data to user space\n");
1275
1276         return rc;
1277 }
1278
1279 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1280                                struct qedr_create_qp_uresp *uresp,
1281                                struct qedr_qp *qp)
1282 {
1283         /* iWARP requires two doorbells per RQ. */
1284         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1285                 uresp->rq_db_offset =
1286                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1287                 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1288         } else {
1289                 uresp->rq_db_offset =
1290                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1291         }
1292
1293         uresp->rq_icid = qp->icid;
1294         if (qp->urq.db_mmap_entry)
1295                 uresp->rq_db_rec_addr =
1296                         rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1297 }
1298
1299 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1300                                struct qedr_create_qp_uresp *uresp,
1301                                struct qedr_qp *qp)
1302 {
1303         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1304
1305         /* iWARP uses the same cid for rq and sq */
1306         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1307                 uresp->sq_icid = qp->icid;
1308         else
1309                 uresp->sq_icid = qp->icid + 1;
1310
1311         if (qp->usq.db_mmap_entry)
1312                 uresp->sq_db_rec_addr =
1313                         rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1314 }
1315
1316 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1317                               struct qedr_qp *qp, struct ib_udata *udata,
1318                               struct qedr_create_qp_uresp *uresp)
1319 {
1320         int rc;
1321
1322         memset(uresp, 0, sizeof(*uresp));
1323
1324         if (qedr_qp_has_sq(qp))
1325                 qedr_copy_sq_uresp(dev, uresp, qp);
1326
1327         if (qedr_qp_has_rq(qp))
1328                 qedr_copy_rq_uresp(dev, uresp, qp);
1329
1330         uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1331         uresp->qp_id = qp->qp_id;
1332
1333         rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1334         if (rc)
1335                 DP_ERR(dev,
1336                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1337                        qp->icid);
1338
1339         return rc;
1340 }
1341
1342 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1343                                       struct qedr_qp *qp,
1344                                       struct qedr_pd *pd,
1345                                       struct ib_qp_init_attr *attrs)
1346 {
1347         spin_lock_init(&qp->q_lock);
1348         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1349                 kref_init(&qp->refcnt);
1350                 init_completion(&qp->iwarp_cm_comp);
1351         }
1352
1353         qp->pd = pd;
1354         qp->qp_type = attrs->qp_type;
1355         qp->max_inline_data = attrs->cap.max_inline_data;
1356         qp->state = QED_ROCE_QP_STATE_RESET;
1357         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1358         qp->dev = dev;
1359         if (qedr_qp_has_sq(qp)) {
1360                 qp->sq.max_sges = attrs->cap.max_send_sge;
1361                 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1362                 DP_DEBUG(dev, QEDR_MSG_QP,
1363                          "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1364                          qp->sq.max_sges, qp->sq_cq->icid);
1365         }
1366
1367         if (attrs->srq)
1368                 qp->srq = get_qedr_srq(attrs->srq);
1369
1370         if (qedr_qp_has_rq(qp)) {
1371                 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1372                 qp->rq.max_sges = attrs->cap.max_recv_sge;
1373                 DP_DEBUG(dev, QEDR_MSG_QP,
1374                          "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1375                          qp->rq.max_sges, qp->rq_cq->icid);
1376         }
1377
1378         DP_DEBUG(dev, QEDR_MSG_QP,
1379                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1380                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1381                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1382         DP_DEBUG(dev, QEDR_MSG_QP,
1383                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1384                  qp->sq.max_sges, qp->sq_cq->icid);
1385 }
1386
1387 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1388 {
1389         int rc = 0;
1390
1391         if (qedr_qp_has_sq(qp)) {
1392                 qp->sq.db = dev->db_addr +
1393                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1394                 qp->sq.db_data.data.icid = qp->icid + 1;
1395                 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1396                                           DB_REC_WIDTH_32B, DB_REC_KERNEL);
1397                 if (rc)
1398                         return rc;
1399         }
1400
1401         if (qedr_qp_has_rq(qp)) {
1402                 qp->rq.db = dev->db_addr +
1403                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1404                 qp->rq.db_data.data.icid = qp->icid;
1405                 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1406                                           DB_REC_WIDTH_32B, DB_REC_KERNEL);
1407                 if (rc && qedr_qp_has_sq(qp))
1408                         qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1409         }
1410
1411         return rc;
1412 }
1413
1414 static int qedr_check_srq_params(struct qedr_dev *dev,
1415                                  struct ib_srq_init_attr *attrs,
1416                                  struct ib_udata *udata)
1417 {
1418         struct qedr_device_attr *qattr = &dev->attr;
1419
1420         if (attrs->attr.max_wr > qattr->max_srq_wr) {
1421                 DP_ERR(dev,
1422                        "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1423                        attrs->attr.max_wr, qattr->max_srq_wr);
1424                 return -EINVAL;
1425         }
1426
1427         if (attrs->attr.max_sge > qattr->max_sge) {
1428                 DP_ERR(dev,
1429                        "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1430                        attrs->attr.max_sge, qattr->max_sge);
1431         }
1432
1433         if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1434                 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1435                 return -EINVAL;
1436         }
1437
1438         return 0;
1439 }
1440
1441 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1442 {
1443         qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1444         ib_umem_release(srq->usrq.umem);
1445         ib_umem_release(srq->prod_umem);
1446 }
1447
1448 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1449 {
1450         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1451         struct qedr_dev *dev = srq->dev;
1452
1453         dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1454
1455         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1456                           hw_srq->virt_prod_pair_addr,
1457                           hw_srq->phy_prod_pair_addr);
1458 }
1459
1460 static int qedr_init_srq_user_params(struct ib_udata *udata,
1461                                      struct qedr_srq *srq,
1462                                      struct qedr_create_srq_ureq *ureq,
1463                                      int access)
1464 {
1465         struct scatterlist *sg;
1466         int rc;
1467
1468         rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1469                                   ureq->srq_len, false, access, 1);
1470         if (rc)
1471                 return rc;
1472
1473         srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1474                                      sizeof(struct rdma_srq_producers), access);
1475         if (IS_ERR(srq->prod_umem)) {
1476                 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1477                 ib_umem_release(srq->usrq.umem);
1478                 DP_ERR(srq->dev,
1479                        "create srq: failed ib_umem_get for producer, got %ld\n",
1480                        PTR_ERR(srq->prod_umem));
1481                 return PTR_ERR(srq->prod_umem);
1482         }
1483
1484         sg = srq->prod_umem->sg_head.sgl;
1485         srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1486
1487         return 0;
1488 }
1489
1490 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1491                                         struct qedr_dev *dev,
1492                                         struct ib_srq_init_attr *init_attr)
1493 {
1494         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1495         struct qed_chain_init_params params = {
1496                 .mode           = QED_CHAIN_MODE_PBL,
1497                 .intended_use   = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1498                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
1499                 .elem_size      = QEDR_SRQ_WQE_ELEM_SIZE,
1500         };
1501         dma_addr_t phy_prod_pair_addr;
1502         u32 num_elems;
1503         void *va;
1504         int rc;
1505
1506         va = dma_alloc_coherent(&dev->pdev->dev,
1507                                 sizeof(struct rdma_srq_producers),
1508                                 &phy_prod_pair_addr, GFP_KERNEL);
1509         if (!va) {
1510                 DP_ERR(dev,
1511                        "create srq: failed to allocate dma memory for producer\n");
1512                 return -ENOMEM;
1513         }
1514
1515         hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1516         hw_srq->virt_prod_pair_addr = va;
1517
1518         num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1519         params.num_elems = num_elems;
1520
1521         rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
1522         if (rc)
1523                 goto err0;
1524
1525         hw_srq->num_elems = num_elems;
1526
1527         return 0;
1528
1529 err0:
1530         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1531                           va, phy_prod_pair_addr);
1532         return rc;
1533 }
1534
1535 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1536                     struct ib_udata *udata)
1537 {
1538         struct qed_rdma_destroy_srq_in_params destroy_in_params;
1539         struct qed_rdma_create_srq_in_params in_params = {};
1540         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1541         struct qed_rdma_create_srq_out_params out_params;
1542         struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1543         struct qedr_create_srq_ureq ureq = {};
1544         u64 pbl_base_addr, phy_prod_pair_addr;
1545         struct qedr_srq_hwq_info *hw_srq;
1546         u32 page_cnt, page_size;
1547         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1548         int rc = 0;
1549
1550         DP_DEBUG(dev, QEDR_MSG_QP,
1551                  "create SRQ called from %s (pd %p)\n",
1552                  (udata) ? "User lib" : "kernel", pd);
1553
1554         if (init_attr->srq_type != IB_SRQT_BASIC &&
1555             init_attr->srq_type != IB_SRQT_XRC)
1556                 return -EOPNOTSUPP;
1557
1558         rc = qedr_check_srq_params(dev, init_attr, udata);
1559         if (rc)
1560                 return -EINVAL;
1561
1562         srq->dev = dev;
1563         srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
1564         hw_srq = &srq->hw_srq;
1565         spin_lock_init(&srq->lock);
1566
1567         hw_srq->max_wr = init_attr->attr.max_wr;
1568         hw_srq->max_sges = init_attr->attr.max_sge;
1569
1570         if (udata) {
1571                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1572                                                          udata->inlen))) {
1573                         DP_ERR(dev,
1574                                "create srq: problem copying data from user space\n");
1575                         goto err0;
1576                 }
1577
1578                 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1579                 if (rc)
1580                         goto err0;
1581
1582                 page_cnt = srq->usrq.pbl_info.num_pbes;
1583                 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1584                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1585                 page_size = PAGE_SIZE;
1586         } else {
1587                 struct qed_chain *pbl;
1588
1589                 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1590                 if (rc)
1591                         goto err0;
1592
1593                 pbl = &hw_srq->pbl;
1594                 page_cnt = qed_chain_get_page_cnt(pbl);
1595                 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1596                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1597                 page_size = QED_CHAIN_PAGE_SIZE;
1598         }
1599
1600         in_params.pd_id = pd->pd_id;
1601         in_params.pbl_base_addr = pbl_base_addr;
1602         in_params.prod_pair_addr = phy_prod_pair_addr;
1603         in_params.num_pages = page_cnt;
1604         in_params.page_size = page_size;
1605         if (srq->is_xrc) {
1606                 struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);
1607                 struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);
1608
1609                 in_params.is_xrc = 1;
1610                 in_params.xrcd_id = xrcd->xrcd_id;
1611                 in_params.cq_cid = cq->icid;
1612         }
1613
1614         rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1615         if (rc)
1616                 goto err1;
1617
1618         srq->srq_id = out_params.srq_id;
1619
1620         if (udata) {
1621                 rc = qedr_copy_srq_uresp(dev, srq, udata);
1622                 if (rc)
1623                         goto err2;
1624         }
1625
1626         rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1627         if (rc)
1628                 goto err2;
1629
1630         DP_DEBUG(dev, QEDR_MSG_SRQ,
1631                  "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1632         return 0;
1633
1634 err2:
1635         destroy_in_params.srq_id = srq->srq_id;
1636
1637         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1638 err1:
1639         if (udata)
1640                 qedr_free_srq_user_params(srq);
1641         else
1642                 qedr_free_srq_kernel_params(srq);
1643 err0:
1644         return -EFAULT;
1645 }
1646
1647 int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1648 {
1649         struct qed_rdma_destroy_srq_in_params in_params = {};
1650         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1651         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1652
1653         xa_erase_irq(&dev->srqs, srq->srq_id);
1654         in_params.srq_id = srq->srq_id;
1655         in_params.is_xrc = srq->is_xrc;
1656         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1657
1658         if (ibsrq->uobject)
1659                 qedr_free_srq_user_params(srq);
1660         else
1661                 qedr_free_srq_kernel_params(srq);
1662
1663         DP_DEBUG(dev, QEDR_MSG_SRQ,
1664                  "destroy srq: destroyed srq with srq_id=0x%0x\n",
1665                  srq->srq_id);
1666         return 0;
1667 }
1668
1669 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1670                     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1671 {
1672         struct qed_rdma_modify_srq_in_params in_params = {};
1673         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1674         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1675         int rc;
1676
1677         if (attr_mask & IB_SRQ_MAX_WR) {
1678                 DP_ERR(dev,
1679                        "modify srq: invalid attribute mask=0x%x specified for %p\n",
1680                        attr_mask, srq);
1681                 return -EINVAL;
1682         }
1683
1684         if (attr_mask & IB_SRQ_LIMIT) {
1685                 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1686                         DP_ERR(dev,
1687                                "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1688                                attr->srq_limit, srq->hw_srq.max_wr);
1689                         return -EINVAL;
1690                 }
1691
1692                 in_params.srq_id = srq->srq_id;
1693                 in_params.wqe_limit = attr->srq_limit;
1694                 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1695                 if (rc)
1696                         return rc;
1697         }
1698
1699         srq->srq_limit = attr->srq_limit;
1700
1701         DP_DEBUG(dev, QEDR_MSG_SRQ,
1702                  "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1703
1704         return 0;
1705 }
1706
1707 static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1708 {
1709         switch (ib_qp_type) {
1710         case IB_QPT_RC:
1711                 return QED_RDMA_QP_TYPE_RC;
1712         case IB_QPT_XRC_INI:
1713                 return QED_RDMA_QP_TYPE_XRC_INI;
1714         case IB_QPT_XRC_TGT:
1715                 return QED_RDMA_QP_TYPE_XRC_TGT;
1716         default:
1717                 return QED_RDMA_QP_TYPE_INVAL;
1718         }
1719 }
1720
1721 static inline void
1722 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1723                               struct qedr_pd *pd,
1724                               struct qedr_qp *qp,
1725                               struct ib_qp_init_attr *attrs,
1726                               bool fmr_and_reserved_lkey,
1727                               struct qed_rdma_create_qp_in_params *params)
1728 {
1729         /* QP handle to be written in an async event */
1730         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1731         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1732
1733         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1734         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1735         params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);
1736         params->stats_queue = 0;
1737
1738         if (pd) {
1739                 params->pd = pd->pd_id;
1740                 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1741         }
1742
1743         if (qedr_qp_has_sq(qp))
1744                 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1745
1746         if (qedr_qp_has_rq(qp))
1747                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1748
1749         if (qedr_qp_has_srq(qp)) {
1750                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1751                 params->srq_id = qp->srq->srq_id;
1752                 params->use_srq = true;
1753         } else {
1754                 params->srq_id = 0;
1755                 params->use_srq = false;
1756         }
1757 }
1758
1759 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1760 {
1761         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1762                  "qp=%p. "
1763                  "sq_addr=0x%llx, "
1764                  "sq_len=%zd, "
1765                  "rq_addr=0x%llx, "
1766                  "rq_len=%zd"
1767                  "\n",
1768                  qp,
1769                  qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1770                  qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1771                  qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1772                  qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1773 }
1774
1775 static inline void
1776 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1777                             struct qedr_qp *qp,
1778                             struct qed_rdma_create_qp_out_params *out_params)
1779 {
1780         qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1781         qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1782
1783         qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1784                            &qp->usq.pbl_info, FW_PAGE_SHIFT);
1785         if (!qp->srq) {
1786                 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1787                 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1788         }
1789
1790         qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1791                            &qp->urq.pbl_info, FW_PAGE_SHIFT);
1792 }
1793
1794 static void qedr_cleanup_user(struct qedr_dev *dev,
1795                               struct qedr_ucontext *ctx,
1796                               struct qedr_qp *qp)
1797 {
1798         if (qedr_qp_has_sq(qp)) {
1799                 ib_umem_release(qp->usq.umem);
1800                 qp->usq.umem = NULL;
1801         }
1802
1803         if (qedr_qp_has_rq(qp)) {
1804                 ib_umem_release(qp->urq.umem);
1805                 qp->urq.umem = NULL;
1806         }
1807
1808         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1809                 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1810                 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1811         } else {
1812                 kfree(qp->usq.pbl_tbl);
1813                 kfree(qp->urq.pbl_tbl);
1814         }
1815
1816         if (qp->usq.db_rec_data) {
1817                 qedr_db_recovery_del(dev, qp->usq.db_addr,
1818                                      &qp->usq.db_rec_data->db_data);
1819                 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1820         }
1821
1822         if (qp->urq.db_rec_data) {
1823                 qedr_db_recovery_del(dev, qp->urq.db_addr,
1824                                      &qp->urq.db_rec_data->db_data);
1825                 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1826         }
1827
1828         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1829                 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1830                                      &qp->urq.db_rec_db2_data);
1831 }
1832
1833 static int qedr_create_user_qp(struct qedr_dev *dev,
1834                                struct qedr_qp *qp,
1835                                struct ib_pd *ibpd,
1836                                struct ib_udata *udata,
1837                                struct ib_qp_init_attr *attrs)
1838 {
1839         struct qed_rdma_create_qp_in_params in_params;
1840         struct qed_rdma_create_qp_out_params out_params;
1841         struct qedr_create_qp_uresp uresp = {};
1842         struct qedr_create_qp_ureq ureq = {};
1843         int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1844         struct qedr_ucontext *ctx = NULL;
1845         struct qedr_pd *pd = NULL;
1846         int rc = 0;
1847
1848         qp->create_type = QEDR_QP_CREATE_USER;
1849
1850         if (ibpd) {
1851                 pd = get_qedr_pd(ibpd);
1852                 ctx = pd->uctx;
1853         }
1854
1855         if (udata) {
1856                 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1857                                         udata->inlen));
1858                 if (rc) {
1859                         DP_ERR(dev, "Problem copying data from user space\n");
1860                         return rc;
1861                 }
1862         }
1863
1864         if (qedr_qp_has_sq(qp)) {
1865                 /* SQ - read access only (0) */
1866                 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1867                                           ureq.sq_len, true, 0, alloc_and_init);
1868                 if (rc)
1869                         return rc;
1870         }
1871
1872         if (qedr_qp_has_rq(qp)) {
1873                 /* RQ - read access only (0) */
1874                 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1875                                           ureq.rq_len, true, 0, alloc_and_init);
1876                 if (rc)
1877                         return rc;
1878         }
1879
1880         memset(&in_params, 0, sizeof(in_params));
1881         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1882         in_params.qp_handle_lo = ureq.qp_handle_lo;
1883         in_params.qp_handle_hi = ureq.qp_handle_hi;
1884
1885         if (qp->qp_type == IB_QPT_XRC_TGT) {
1886                 struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);
1887
1888                 in_params.xrcd_id = xrcd->xrcd_id;
1889                 in_params.qp_handle_lo = qp->qp_id;
1890                 in_params.use_srq = 1;
1891         }
1892
1893         if (qedr_qp_has_sq(qp)) {
1894                 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1895                 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1896         }
1897
1898         if (qedr_qp_has_rq(qp)) {
1899                 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1900                 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1901         }
1902
1903         if (ctx)
1904                 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1905
1906         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1907                                               &in_params, &out_params);
1908
1909         if (!qp->qed_qp) {
1910                 rc = -ENOMEM;
1911                 goto err1;
1912         }
1913
1914         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1915                 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1916
1917         qp->qp_id = out_params.qp_id;
1918         qp->icid = out_params.icid;
1919
1920         if (udata) {
1921                 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1922                 if (rc)
1923                         goto err;
1924         }
1925
1926         /* db offset was calculated in copy_qp_uresp, now set in the user q */
1927         if (qedr_qp_has_sq(qp)) {
1928                 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1929                 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1930                                           &qp->usq.db_rec_data->db_data,
1931                                           DB_REC_WIDTH_32B,
1932                                           DB_REC_USER);
1933                 if (rc)
1934                         goto err;
1935         }
1936
1937         if (qedr_qp_has_rq(qp)) {
1938                 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1939                 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1940                                           &qp->urq.db_rec_data->db_data,
1941                                           DB_REC_WIDTH_32B,
1942                                           DB_REC_USER);
1943                 if (rc)
1944                         goto err;
1945         }
1946
1947         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1948                 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1949
1950                 /* calculate the db_rec_db2 data since it is constant so no
1951                  * need to reflect from user
1952                  */
1953                 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1954                 qp->urq.db_rec_db2_data.data.value =
1955                         cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1956
1957                 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1958                                           &qp->urq.db_rec_db2_data,
1959                                           DB_REC_WIDTH_32B,
1960                                           DB_REC_USER);
1961                 if (rc)
1962                         goto err;
1963         }
1964         qedr_qp_user_print(dev, qp);
1965         return rc;
1966 err:
1967         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1968         if (rc)
1969                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1970
1971 err1:
1972         qedr_cleanup_user(dev, ctx, qp);
1973         return rc;
1974 }
1975
1976 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1977 {
1978         int rc;
1979
1980         qp->sq.db = dev->db_addr +
1981             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1982         qp->sq.db_data.data.icid = qp->icid;
1983
1984         rc = qedr_db_recovery_add(dev, qp->sq.db,
1985                                   &qp->sq.db_data,
1986                                   DB_REC_WIDTH_32B,
1987                                   DB_REC_KERNEL);
1988         if (rc)
1989                 return rc;
1990
1991         qp->rq.db = dev->db_addr +
1992                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1993         qp->rq.db_data.data.icid = qp->icid;
1994         qp->rq.iwarp_db2 = dev->db_addr +
1995                            DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1996         qp->rq.iwarp_db2_data.data.icid = qp->icid;
1997         qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1998
1999         rc = qedr_db_recovery_add(dev, qp->rq.db,
2000                                   &qp->rq.db_data,
2001                                   DB_REC_WIDTH_32B,
2002                                   DB_REC_KERNEL);
2003         if (rc)
2004                 return rc;
2005
2006         rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2007                                   &qp->rq.iwarp_db2_data,
2008                                   DB_REC_WIDTH_32B,
2009                                   DB_REC_KERNEL);
2010         return rc;
2011 }
2012
2013 static int
2014 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2015                            struct qedr_qp *qp,
2016                            struct qed_rdma_create_qp_in_params *in_params,
2017                            u32 n_sq_elems, u32 n_rq_elems)
2018 {
2019         struct qed_rdma_create_qp_out_params out_params;
2020         struct qed_chain_init_params params = {
2021                 .mode           = QED_CHAIN_MODE_PBL,
2022                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
2023         };
2024         int rc;
2025
2026         params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2027         params.num_elems = n_sq_elems;
2028         params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2029
2030         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2031         if (rc)
2032                 return rc;
2033
2034         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2035         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
2036
2037         params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2038         params.num_elems = n_rq_elems;
2039         params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2040
2041         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2042         if (rc)
2043                 return rc;
2044
2045         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2046         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
2047
2048         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2049                                               in_params, &out_params);
2050
2051         if (!qp->qed_qp)
2052                 return -EINVAL;
2053
2054         qp->qp_id = out_params.qp_id;
2055         qp->icid = out_params.icid;
2056
2057         return qedr_set_roce_db_info(dev, qp);
2058 }
2059
2060 static int
2061 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2062                             struct qedr_qp *qp,
2063                             struct qed_rdma_create_qp_in_params *in_params,
2064                             u32 n_sq_elems, u32 n_rq_elems)
2065 {
2066         struct qed_rdma_create_qp_out_params out_params;
2067         struct qed_chain_init_params params = {
2068                 .mode           = QED_CHAIN_MODE_PBL,
2069                 .cnt_type       = QED_CHAIN_CNT_TYPE_U32,
2070         };
2071         int rc;
2072
2073         in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2074                                                      QEDR_SQE_ELEMENT_SIZE,
2075                                                      QED_CHAIN_PAGE_SIZE,
2076                                                      QED_CHAIN_MODE_PBL);
2077         in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2078                                                      QEDR_RQE_ELEMENT_SIZE,
2079                                                      QED_CHAIN_PAGE_SIZE,
2080                                                      QED_CHAIN_MODE_PBL);
2081
2082         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2083                                               in_params, &out_params);
2084
2085         if (!qp->qed_qp)
2086                 return -EINVAL;
2087
2088         /* Now we allocate the chain */
2089
2090         params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2091         params.num_elems = n_sq_elems;
2092         params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2093         params.ext_pbl_virt = out_params.sq_pbl_virt;
2094         params.ext_pbl_phys = out_params.sq_pbl_phys;
2095
2096         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2097         if (rc)
2098                 goto err;
2099
2100         params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2101         params.num_elems = n_rq_elems;
2102         params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2103         params.ext_pbl_virt = out_params.rq_pbl_virt;
2104         params.ext_pbl_phys = out_params.rq_pbl_phys;
2105
2106         rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2107         if (rc)
2108                 goto err;
2109
2110         qp->qp_id = out_params.qp_id;
2111         qp->icid = out_params.icid;
2112
2113         return qedr_set_iwarp_db_info(dev, qp);
2114
2115 err:
2116         dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2117
2118         return rc;
2119 }
2120
2121 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2122 {
2123         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2124         kfree(qp->wqe_wr_id);
2125
2126         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2127         kfree(qp->rqe_wr_id);
2128
2129         /* GSI qp is not registered to db mechanism so no need to delete */
2130         if (qp->qp_type == IB_QPT_GSI)
2131                 return;
2132
2133         qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2134
2135         if (!qp->srq) {
2136                 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2137
2138                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2139                         qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2140                                              &qp->rq.iwarp_db2_data);
2141         }
2142 }
2143
2144 static int qedr_create_kernel_qp(struct qedr_dev *dev,
2145                                  struct qedr_qp *qp,
2146                                  struct ib_pd *ibpd,
2147                                  struct ib_qp_init_attr *attrs)
2148 {
2149         struct qed_rdma_create_qp_in_params in_params;
2150         struct qedr_pd *pd = get_qedr_pd(ibpd);
2151         int rc = -EINVAL;
2152         u32 n_rq_elems;
2153         u32 n_sq_elems;
2154         u32 n_sq_entries;
2155
2156         memset(&in_params, 0, sizeof(in_params));
2157         qp->create_type = QEDR_QP_CREATE_KERNEL;
2158
2159         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2160          * the ring. The ring should allow at least a single WR, even if the
2161          * user requested none, due to allocation issues.
2162          * We should add an extra WR since the prod and cons indices of
2163          * wqe_wr_id are managed in such a way that the WQ is considered full
2164          * when (prod+1)%max_wr==cons. We currently don't do that because we
2165          * double the number of entries due an iSER issue that pushes far more
2166          * WRs than indicated. If we decline its ib_post_send() then we get
2167          * error prints in the dmesg we'd like to avoid.
2168          */
2169         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2170                               dev->attr.max_sqe);
2171
2172         qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2173                                 GFP_KERNEL);
2174         if (!qp->wqe_wr_id) {
2175                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2176                 return -ENOMEM;
2177         }
2178
2179         /* QP handle to be written in CQE */
2180         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2181         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2182
2183         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2184          * the ring. There ring should allow at least a single WR, even if the
2185          * user requested none, due to allocation issues.
2186          */
2187         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2188
2189         /* Allocate driver internal RQ array */
2190         qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2191                                 GFP_KERNEL);
2192         if (!qp->rqe_wr_id) {
2193                 DP_ERR(dev,
2194                        "create qp: failed RQ shadow memory allocation\n");
2195                 kfree(qp->wqe_wr_id);
2196                 return -ENOMEM;
2197         }
2198
2199         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2200
2201         n_sq_entries = attrs->cap.max_send_wr;
2202         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2203         n_sq_entries = max_t(u32, n_sq_entries, 1);
2204         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2205
2206         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2207
2208         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2209                 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2210                                                  n_sq_elems, n_rq_elems);
2211         else
2212                 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2213                                                 n_sq_elems, n_rq_elems);
2214         if (rc)
2215                 qedr_cleanup_kernel(dev, qp);
2216
2217         return rc;
2218 }
2219
2220 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2221                                   struct ib_udata *udata)
2222 {
2223         struct qedr_ucontext *ctx =
2224                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2225                                           ibucontext);
2226         int rc;
2227
2228         if (qp->qp_type != IB_QPT_GSI) {
2229                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2230                 if (rc)
2231                         return rc;
2232         }
2233
2234         if (qp->create_type == QEDR_QP_CREATE_USER)
2235                 qedr_cleanup_user(dev, ctx, qp);
2236         else
2237                 qedr_cleanup_kernel(dev, qp);
2238
2239         return 0;
2240 }
2241
2242 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
2243                              struct ib_qp_init_attr *attrs,
2244                              struct ib_udata *udata)
2245 {
2246         struct qedr_xrcd *xrcd = NULL;
2247         struct qedr_pd *pd = NULL;
2248         struct qedr_dev *dev;
2249         struct qedr_qp *qp;
2250         struct ib_qp *ibqp;
2251         int rc = 0;
2252
2253         if (attrs->create_flags)
2254                 return ERR_PTR(-EOPNOTSUPP);
2255
2256         if (attrs->qp_type == IB_QPT_XRC_TGT) {
2257                 xrcd = get_qedr_xrcd(attrs->xrcd);
2258                 dev = get_qedr_dev(xrcd->ibxrcd.device);
2259         } else {
2260                 pd = get_qedr_pd(ibpd);
2261                 dev = get_qedr_dev(ibpd->device);
2262         }
2263
2264         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2265                  udata ? "user library" : "kernel", pd);
2266
2267         rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2268         if (rc)
2269                 return ERR_PTR(rc);
2270
2271         DP_DEBUG(dev, QEDR_MSG_QP,
2272                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2273                  udata ? "user library" : "kernel", attrs->event_handler, pd,
2274                  get_qedr_cq(attrs->send_cq),
2275                  get_qedr_cq(attrs->send_cq)->icid,
2276                  get_qedr_cq(attrs->recv_cq),
2277                  attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2278
2279         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2280         if (!qp) {
2281                 DP_ERR(dev, "create qp: failed allocating memory\n");
2282                 return ERR_PTR(-ENOMEM);
2283         }
2284
2285         qedr_set_common_qp_params(dev, qp, pd, attrs);
2286
2287         if (attrs->qp_type == IB_QPT_GSI) {
2288                 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2289                 if (IS_ERR(ibqp))
2290                         kfree(qp);
2291                 return ibqp;
2292         }
2293
2294         if (udata || xrcd)
2295                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2296         else
2297                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2298
2299         if (rc)
2300                 goto out_free_qp;
2301
2302         qp->ibqp.qp_num = qp->qp_id;
2303
2304         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2305                 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2306                 if (rc)
2307                         goto out_free_qp_resources;
2308         }
2309
2310         return &qp->ibqp;
2311
2312 out_free_qp_resources:
2313         qedr_free_qp_resources(dev, qp, udata);
2314 out_free_qp:
2315         kfree(qp);
2316
2317         return ERR_PTR(-EFAULT);
2318 }
2319
2320 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2321 {
2322         switch (qp_state) {
2323         case QED_ROCE_QP_STATE_RESET:
2324                 return IB_QPS_RESET;
2325         case QED_ROCE_QP_STATE_INIT:
2326                 return IB_QPS_INIT;
2327         case QED_ROCE_QP_STATE_RTR:
2328                 return IB_QPS_RTR;
2329         case QED_ROCE_QP_STATE_RTS:
2330                 return IB_QPS_RTS;
2331         case QED_ROCE_QP_STATE_SQD:
2332                 return IB_QPS_SQD;
2333         case QED_ROCE_QP_STATE_ERR:
2334                 return IB_QPS_ERR;
2335         case QED_ROCE_QP_STATE_SQE:
2336                 return IB_QPS_SQE;
2337         }
2338         return IB_QPS_ERR;
2339 }
2340
2341 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2342                                         enum ib_qp_state qp_state)
2343 {
2344         switch (qp_state) {
2345         case IB_QPS_RESET:
2346                 return QED_ROCE_QP_STATE_RESET;
2347         case IB_QPS_INIT:
2348                 return QED_ROCE_QP_STATE_INIT;
2349         case IB_QPS_RTR:
2350                 return QED_ROCE_QP_STATE_RTR;
2351         case IB_QPS_RTS:
2352                 return QED_ROCE_QP_STATE_RTS;
2353         case IB_QPS_SQD:
2354                 return QED_ROCE_QP_STATE_SQD;
2355         case IB_QPS_ERR:
2356                 return QED_ROCE_QP_STATE_ERR;
2357         default:
2358                 return QED_ROCE_QP_STATE_ERR;
2359         }
2360 }
2361
2362 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2363 {
2364         qed_chain_reset(&qph->pbl);
2365         qph->prod = 0;
2366         qph->cons = 0;
2367         qph->wqe_cons = 0;
2368         qph->db_data.data.value = cpu_to_le16(0);
2369 }
2370
2371 static int qedr_update_qp_state(struct qedr_dev *dev,
2372                                 struct qedr_qp *qp,
2373                                 enum qed_roce_qp_state cur_state,
2374                                 enum qed_roce_qp_state new_state)
2375 {
2376         int status = 0;
2377
2378         if (new_state == cur_state)
2379                 return 0;
2380
2381         switch (cur_state) {
2382         case QED_ROCE_QP_STATE_RESET:
2383                 switch (new_state) {
2384                 case QED_ROCE_QP_STATE_INIT:
2385                         qp->prev_wqe_size = 0;
2386                         qedr_reset_qp_hwq_info(&qp->sq);
2387                         qedr_reset_qp_hwq_info(&qp->rq);
2388                         break;
2389                 default:
2390                         status = -EINVAL;
2391                         break;
2392                 }
2393                 break;
2394         case QED_ROCE_QP_STATE_INIT:
2395                 switch (new_state) {
2396                 case QED_ROCE_QP_STATE_RTR:
2397                         /* Update doorbell (in case post_recv was
2398                          * done before move to RTR)
2399                          */
2400
2401                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2402                                 writel(qp->rq.db_data.raw, qp->rq.db);
2403                         }
2404                         break;
2405                 case QED_ROCE_QP_STATE_ERR:
2406                         break;
2407                 default:
2408                         /* Invalid state change. */
2409                         status = -EINVAL;
2410                         break;
2411                 }
2412                 break;
2413         case QED_ROCE_QP_STATE_RTR:
2414                 /* RTR->XXX */
2415                 switch (new_state) {
2416                 case QED_ROCE_QP_STATE_RTS:
2417                         break;
2418                 case QED_ROCE_QP_STATE_ERR:
2419                         break;
2420                 default:
2421                         /* Invalid state change. */
2422                         status = -EINVAL;
2423                         break;
2424                 }
2425                 break;
2426         case QED_ROCE_QP_STATE_RTS:
2427                 /* RTS->XXX */
2428                 switch (new_state) {
2429                 case QED_ROCE_QP_STATE_SQD:
2430                         break;
2431                 case QED_ROCE_QP_STATE_ERR:
2432                         break;
2433                 default:
2434                         /* Invalid state change. */
2435                         status = -EINVAL;
2436                         break;
2437                 }
2438                 break;
2439         case QED_ROCE_QP_STATE_SQD:
2440                 /* SQD->XXX */
2441                 switch (new_state) {
2442                 case QED_ROCE_QP_STATE_RTS:
2443                 case QED_ROCE_QP_STATE_ERR:
2444                         break;
2445                 default:
2446                         /* Invalid state change. */
2447                         status = -EINVAL;
2448                         break;
2449                 }
2450                 break;
2451         case QED_ROCE_QP_STATE_ERR:
2452                 /* ERR->XXX */
2453                 switch (new_state) {
2454                 case QED_ROCE_QP_STATE_RESET:
2455                         if ((qp->rq.prod != qp->rq.cons) ||
2456                             (qp->sq.prod != qp->sq.cons)) {
2457                                 DP_NOTICE(dev,
2458                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2459                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
2460                                           qp->sq.cons);
2461                                 status = -EINVAL;
2462                         }
2463                         break;
2464                 default:
2465                         status = -EINVAL;
2466                         break;
2467                 }
2468                 break;
2469         default:
2470                 status = -EINVAL;
2471                 break;
2472         }
2473
2474         return status;
2475 }
2476
2477 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2478                    int attr_mask, struct ib_udata *udata)
2479 {
2480         struct qedr_qp *qp = get_qedr_qp(ibqp);
2481         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2482         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2483         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2484         enum ib_qp_state old_qp_state, new_qp_state;
2485         enum qed_roce_qp_state cur_state;
2486         int rc = 0;
2487
2488         DP_DEBUG(dev, QEDR_MSG_QP,
2489                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2490                  attr->qp_state);
2491
2492         if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2493                 return -EOPNOTSUPP;
2494
2495         old_qp_state = qedr_get_ibqp_state(qp->state);
2496         if (attr_mask & IB_QP_STATE)
2497                 new_qp_state = attr->qp_state;
2498         else
2499                 new_qp_state = old_qp_state;
2500
2501         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2502                 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2503                                         ibqp->qp_type, attr_mask)) {
2504                         DP_ERR(dev,
2505                                "modify qp: invalid attribute mask=0x%x specified for\n"
2506                                "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2507                                attr_mask, qp->qp_id, ibqp->qp_type,
2508                                old_qp_state, new_qp_state);
2509                         rc = -EINVAL;
2510                         goto err;
2511                 }
2512         }
2513
2514         /* Translate the masks... */
2515         if (attr_mask & IB_QP_STATE) {
2516                 SET_FIELD(qp_params.modify_flags,
2517                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2518                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2519         }
2520
2521         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2522                 qp_params.sqd_async = true;
2523
2524         if (attr_mask & IB_QP_PKEY_INDEX) {
2525                 SET_FIELD(qp_params.modify_flags,
2526                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2527                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2528                         rc = -EINVAL;
2529                         goto err;
2530                 }
2531
2532                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2533         }
2534
2535         if (attr_mask & IB_QP_QKEY)
2536                 qp->qkey = attr->qkey;
2537
2538         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2539                 SET_FIELD(qp_params.modify_flags,
2540                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2541                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2542                                                   IB_ACCESS_REMOTE_READ;
2543                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2544                                                    IB_ACCESS_REMOTE_WRITE;
2545                 qp_params.incoming_atomic_en = attr->qp_access_flags &
2546                                                IB_ACCESS_REMOTE_ATOMIC;
2547         }
2548
2549         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2550                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2551                         return -EINVAL;
2552
2553                 if (attr_mask & IB_QP_PATH_MTU) {
2554                         if (attr->path_mtu < IB_MTU_256 ||
2555                             attr->path_mtu > IB_MTU_4096) {
2556                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2557                                 rc = -EINVAL;
2558                                 goto err;
2559                         }
2560                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2561                                       ib_mtu_enum_to_int(iboe_get_mtu
2562                                                          (dev->ndev->mtu)));
2563                 }
2564
2565                 if (!qp->mtu) {
2566                         qp->mtu =
2567                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2568                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2569                 }
2570
2571                 SET_FIELD(qp_params.modify_flags,
2572                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2573
2574                 qp_params.traffic_class_tos = grh->traffic_class;
2575                 qp_params.flow_label = grh->flow_label;
2576                 qp_params.hop_limit_ttl = grh->hop_limit;
2577
2578                 qp->sgid_idx = grh->sgid_index;
2579
2580                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2581                 if (rc) {
2582                         DP_ERR(dev,
2583                                "modify qp: problems with GID index %d (rc=%d)\n",
2584                                grh->sgid_index, rc);
2585                         return rc;
2586                 }
2587
2588                 rc = qedr_get_dmac(dev, &attr->ah_attr,
2589                                    qp_params.remote_mac_addr);
2590                 if (rc)
2591                         return rc;
2592
2593                 qp_params.use_local_mac = true;
2594                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2595
2596                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2597                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2598                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2599                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2600                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2601                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2602                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2603                          qp_params.remote_mac_addr);
2604
2605                 qp_params.mtu = qp->mtu;
2606                 qp_params.lb_indication = false;
2607         }
2608
2609         if (!qp_params.mtu) {
2610                 /* Stay with current MTU */
2611                 if (qp->mtu)
2612                         qp_params.mtu = qp->mtu;
2613                 else
2614                         qp_params.mtu =
2615                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2616         }
2617
2618         if (attr_mask & IB_QP_TIMEOUT) {
2619                 SET_FIELD(qp_params.modify_flags,
2620                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2621
2622                 /* The received timeout value is an exponent used like this:
2623                  *    "12.7.34 LOCAL ACK TIMEOUT
2624                  *    Value representing the transport (ACK) timeout for use by
2625                  *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2626                  * The FW expects timeout in msec so we need to divide the usec
2627                  * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2628                  * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2629                  * The value of zero means infinite so we use a 'max_t' to make
2630                  * sure that sub 1 msec values will be configured as 1 msec.
2631                  */
2632                 if (attr->timeout)
2633                         qp_params.ack_timeout =
2634                                         1 << max_t(int, attr->timeout - 8, 0);
2635                 else
2636                         qp_params.ack_timeout = 0;
2637         }
2638
2639         if (attr_mask & IB_QP_RETRY_CNT) {
2640                 SET_FIELD(qp_params.modify_flags,
2641                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2642                 qp_params.retry_cnt = attr->retry_cnt;
2643         }
2644
2645         if (attr_mask & IB_QP_RNR_RETRY) {
2646                 SET_FIELD(qp_params.modify_flags,
2647                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2648                 qp_params.rnr_retry_cnt = attr->rnr_retry;
2649         }
2650
2651         if (attr_mask & IB_QP_RQ_PSN) {
2652                 SET_FIELD(qp_params.modify_flags,
2653                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2654                 qp_params.rq_psn = attr->rq_psn;
2655                 qp->rq_psn = attr->rq_psn;
2656         }
2657
2658         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2659                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2660                         rc = -EINVAL;
2661                         DP_ERR(dev,
2662                                "unsupported max_rd_atomic=%d, supported=%d\n",
2663                                attr->max_rd_atomic,
2664                                dev->attr.max_qp_req_rd_atomic_resc);
2665                         goto err;
2666                 }
2667
2668                 SET_FIELD(qp_params.modify_flags,
2669                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2670                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2671         }
2672
2673         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2674                 SET_FIELD(qp_params.modify_flags,
2675                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2676                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2677         }
2678
2679         if (attr_mask & IB_QP_SQ_PSN) {
2680                 SET_FIELD(qp_params.modify_flags,
2681                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2682                 qp_params.sq_psn = attr->sq_psn;
2683                 qp->sq_psn = attr->sq_psn;
2684         }
2685
2686         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2687                 if (attr->max_dest_rd_atomic >
2688                     dev->attr.max_qp_resp_rd_atomic_resc) {
2689                         DP_ERR(dev,
2690                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2691                                attr->max_dest_rd_atomic,
2692                                dev->attr.max_qp_resp_rd_atomic_resc);
2693
2694                         rc = -EINVAL;
2695                         goto err;
2696                 }
2697
2698                 SET_FIELD(qp_params.modify_flags,
2699                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2700                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2701         }
2702
2703         if (attr_mask & IB_QP_DEST_QPN) {
2704                 SET_FIELD(qp_params.modify_flags,
2705                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2706
2707                 qp_params.dest_qp = attr->dest_qp_num;
2708                 qp->dest_qp_num = attr->dest_qp_num;
2709         }
2710
2711         cur_state = qp->state;
2712
2713         /* Update the QP state before the actual ramrod to prevent a race with
2714          * fast path. Modifying the QP state to error will cause the device to
2715          * flush the CQEs and while polling the flushed CQEs will considered as
2716          * a potential issue if the QP isn't in error state.
2717          */
2718         if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2719             !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2720                 qp->state = QED_ROCE_QP_STATE_ERR;
2721
2722         if (qp->qp_type != IB_QPT_GSI)
2723                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2724                                               qp->qed_qp, &qp_params);
2725
2726         if (attr_mask & IB_QP_STATE) {
2727                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2728                         rc = qedr_update_qp_state(dev, qp, cur_state,
2729                                                   qp_params.new_state);
2730                 qp->state = qp_params.new_state;
2731         }
2732
2733 err:
2734         return rc;
2735 }
2736
2737 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2738 {
2739         int ib_qp_acc_flags = 0;
2740
2741         if (params->incoming_rdma_write_en)
2742                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2743         if (params->incoming_rdma_read_en)
2744                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2745         if (params->incoming_atomic_en)
2746                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2747         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2748         return ib_qp_acc_flags;
2749 }
2750
2751 int qedr_query_qp(struct ib_qp *ibqp,
2752                   struct ib_qp_attr *qp_attr,
2753                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2754 {
2755         struct qed_rdma_query_qp_out_params params;
2756         struct qedr_qp *qp = get_qedr_qp(ibqp);
2757         struct qedr_dev *dev = qp->dev;
2758         int rc = 0;
2759
2760         memset(&params, 0, sizeof(params));
2761
2762         rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2763         if (rc)
2764                 goto err;
2765
2766         memset(qp_attr, 0, sizeof(*qp_attr));
2767         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2768
2769         qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2770         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2771         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2772         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2773         qp_attr->rq_psn = params.rq_psn;
2774         qp_attr->sq_psn = params.sq_psn;
2775         qp_attr->dest_qp_num = params.dest_qp;
2776
2777         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2778
2779         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2780         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2781         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2782         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2783         qp_attr->cap.max_inline_data = dev->attr.max_inline;
2784         qp_init_attr->cap = qp_attr->cap;
2785
2786         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2787         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2788                         params.flow_label, qp->sgid_idx,
2789                         params.hop_limit_ttl, params.traffic_class_tos);
2790         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2791         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2792         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2793         qp_attr->timeout = params.timeout;
2794         qp_attr->rnr_retry = params.rnr_retry;
2795         qp_attr->retry_cnt = params.retry_cnt;
2796         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2797         qp_attr->pkey_index = params.pkey_index;
2798         qp_attr->port_num = 1;
2799         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2800         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2801         qp_attr->alt_pkey_index = 0;
2802         qp_attr->alt_port_num = 0;
2803         qp_attr->alt_timeout = 0;
2804         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2805
2806         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2807         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2808         qp_attr->max_rd_atomic = params.max_rd_atomic;
2809         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2810
2811         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2812                  qp_attr->cap.max_inline_data);
2813
2814 err:
2815         return rc;
2816 }
2817
2818 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2819 {
2820         struct qedr_qp *qp = get_qedr_qp(ibqp);
2821         struct qedr_dev *dev = qp->dev;
2822         struct ib_qp_attr attr;
2823         int attr_mask = 0;
2824
2825         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2826                  qp, qp->qp_type);
2827
2828         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2829                 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2830                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
2831                     (qp->state != QED_ROCE_QP_STATE_INIT)) {
2832
2833                         attr.qp_state = IB_QPS_ERR;
2834                         attr_mask |= IB_QP_STATE;
2835
2836                         /* Change the QP state to ERROR */
2837                         qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2838                 }
2839         } else {
2840                 /* If connection establishment started the WAIT_FOR_CONNECT
2841                  * bit will be on and we need to Wait for the establishment
2842                  * to complete before destroying the qp.
2843                  */
2844                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2845                                      &qp->iwarp_cm_flags))
2846                         wait_for_completion(&qp->iwarp_cm_comp);
2847
2848                 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2849                  * bit will be on, and we need to wait for the disconnect to
2850                  * complete before continuing. We can use the same completion,
2851                  * iwarp_cm_comp, since this is the only place that waits for
2852                  * this completion and it is sequential. In addition,
2853                  * disconnect can't occur before the connection is fully
2854                  * established, therefore if WAIT_FOR_DISCONNECT is on it
2855                  * means WAIT_FOR_CONNECT is also on and the completion for
2856                  * CONNECT already occurred.
2857                  */
2858                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2859                                      &qp->iwarp_cm_flags))
2860                         wait_for_completion(&qp->iwarp_cm_comp);
2861         }
2862
2863         if (qp->qp_type == IB_QPT_GSI)
2864                 qedr_destroy_gsi_qp(dev);
2865
2866         /* We need to remove the entry from the xarray before we release the
2867          * qp_id to avoid a race of the qp_id being reallocated and failing
2868          * on xa_insert
2869          */
2870         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2871                 xa_erase(&dev->qps, qp->qp_id);
2872
2873         qedr_free_qp_resources(dev, qp, udata);
2874
2875         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2876                 qedr_iw_qp_rem_ref(&qp->ibqp);
2877         else
2878                 kfree(qp);
2879
2880         return 0;
2881 }
2882
2883 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2884                    struct ib_udata *udata)
2885 {
2886         struct qedr_ah *ah = get_qedr_ah(ibah);
2887
2888         rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2889
2890         return 0;
2891 }
2892
2893 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2894 {
2895         struct qedr_ah *ah = get_qedr_ah(ibah);
2896
2897         rdma_destroy_ah_attr(&ah->attr);
2898         return 0;
2899 }
2900
2901 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2902 {
2903         struct qedr_pbl *pbl, *tmp;
2904
2905         if (info->pbl_table)
2906                 list_add_tail(&info->pbl_table->list_entry,
2907                               &info->free_pbl_list);
2908
2909         if (!list_empty(&info->inuse_pbl_list))
2910                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2911
2912         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2913                 list_del(&pbl->list_entry);
2914                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2915         }
2916 }
2917
2918 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2919                         size_t page_list_len, bool two_layered)
2920 {
2921         struct qedr_pbl *tmp;
2922         int rc;
2923
2924         INIT_LIST_HEAD(&info->free_pbl_list);
2925         INIT_LIST_HEAD(&info->inuse_pbl_list);
2926
2927         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2928                                   page_list_len, two_layered);
2929         if (rc)
2930                 goto done;
2931
2932         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2933         if (IS_ERR(info->pbl_table)) {
2934                 rc = PTR_ERR(info->pbl_table);
2935                 goto done;
2936         }
2937
2938         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2939                  &info->pbl_table->pa);
2940
2941         /* in usual case we use 2 PBLs, so we add one to free
2942          * list and allocating another one
2943          */
2944         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2945         if (IS_ERR(tmp)) {
2946                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2947                 goto done;
2948         }
2949
2950         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2951
2952         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2953
2954 done:
2955         if (rc)
2956                 free_mr_info(dev, info);
2957
2958         return rc;
2959 }
2960
2961 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2962                                u64 usr_addr, int acc, struct ib_udata *udata)
2963 {
2964         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2965         struct qedr_mr *mr;
2966         struct qedr_pd *pd;
2967         int rc = -ENOMEM;
2968
2969         pd = get_qedr_pd(ibpd);
2970         DP_DEBUG(dev, QEDR_MSG_MR,
2971                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2972                  pd->pd_id, start, len, usr_addr, acc);
2973
2974         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2975                 return ERR_PTR(-EINVAL);
2976
2977         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2978         if (!mr)
2979                 return ERR_PTR(rc);
2980
2981         mr->type = QEDR_MR_USER;
2982
2983         mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2984         if (IS_ERR(mr->umem)) {
2985                 rc = -EFAULT;
2986                 goto err0;
2987         }
2988
2989         rc = init_mr_info(dev, &mr->info,
2990                           ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
2991         if (rc)
2992                 goto err1;
2993
2994         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2995                            &mr->info.pbl_info, PAGE_SHIFT);
2996
2997         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2998         if (rc) {
2999                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3000                 goto err1;
3001         }
3002
3003         /* Index only, 18 bit long, lkey = itid << 8 | key */
3004         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3005         mr->hw_mr.key = 0;
3006         mr->hw_mr.pd = pd->pd_id;
3007         mr->hw_mr.local_read = 1;
3008         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3009         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3010         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3011         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3012         mr->hw_mr.mw_bind = false;
3013         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3014         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3015         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3016         mr->hw_mr.page_size_log = PAGE_SHIFT;
3017         mr->hw_mr.length = len;
3018         mr->hw_mr.vaddr = usr_addr;
3019         mr->hw_mr.phy_mr = false;
3020         mr->hw_mr.dma_mr = false;
3021
3022         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3023         if (rc) {
3024                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3025                 goto err2;
3026         }
3027
3028         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3029         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3030             mr->hw_mr.remote_atomic)
3031                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3032
3033         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3034                  mr->ibmr.lkey);
3035         return &mr->ibmr;
3036
3037 err2:
3038         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3039 err1:
3040         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3041 err0:
3042         kfree(mr);
3043         return ERR_PTR(rc);
3044 }
3045
3046 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3047 {
3048         struct qedr_mr *mr = get_qedr_mr(ib_mr);
3049         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3050         int rc = 0;
3051
3052         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3053         if (rc)
3054                 return rc;
3055
3056         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3057
3058         if (mr->type != QEDR_MR_DMA)
3059                 free_mr_info(dev, &mr->info);
3060
3061         /* it could be user registered memory. */
3062         ib_umem_release(mr->umem);
3063
3064         kfree(mr);
3065
3066         return rc;
3067 }
3068
3069 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3070                                        int max_page_list_len)
3071 {
3072         struct qedr_pd *pd = get_qedr_pd(ibpd);
3073         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3074         struct qedr_mr *mr;
3075         int rc = -ENOMEM;
3076
3077         DP_DEBUG(dev, QEDR_MSG_MR,
3078                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3079                  max_page_list_len);
3080
3081         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3082         if (!mr)
3083                 return ERR_PTR(rc);
3084
3085         mr->dev = dev;
3086         mr->type = QEDR_MR_FRMR;
3087
3088         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3089         if (rc)
3090                 goto err0;
3091
3092         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3093         if (rc) {
3094                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3095                 goto err0;
3096         }
3097
3098         /* Index only, 18 bit long, lkey = itid << 8 | key */
3099         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3100         mr->hw_mr.key = 0;
3101         mr->hw_mr.pd = pd->pd_id;
3102         mr->hw_mr.local_read = 1;
3103         mr->hw_mr.local_write = 0;
3104         mr->hw_mr.remote_read = 0;
3105         mr->hw_mr.remote_write = 0;
3106         mr->hw_mr.remote_atomic = 0;
3107         mr->hw_mr.mw_bind = false;
3108         mr->hw_mr.pbl_ptr = 0;
3109         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3110         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3111         mr->hw_mr.length = 0;
3112         mr->hw_mr.vaddr = 0;
3113         mr->hw_mr.phy_mr = true;
3114         mr->hw_mr.dma_mr = false;
3115
3116         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3117         if (rc) {
3118                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3119                 goto err1;
3120         }
3121
3122         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3123         mr->ibmr.rkey = mr->ibmr.lkey;
3124
3125         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3126         return mr;
3127
3128 err1:
3129         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3130 err0:
3131         kfree(mr);
3132         return ERR_PTR(rc);
3133 }
3134
3135 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3136                             u32 max_num_sg)
3137 {
3138         struct qedr_mr *mr;
3139
3140         if (mr_type != IB_MR_TYPE_MEM_REG)
3141                 return ERR_PTR(-EINVAL);
3142
3143         mr = __qedr_alloc_mr(ibpd, max_num_sg);
3144
3145         if (IS_ERR(mr))
3146                 return ERR_PTR(-EINVAL);
3147
3148         return &mr->ibmr;
3149 }
3150
3151 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3152 {
3153         struct qedr_mr *mr = get_qedr_mr(ibmr);
3154         struct qedr_pbl *pbl_table;
3155         struct regpair *pbe;
3156         u32 pbes_in_page;
3157
3158         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3159                 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3160                 return -ENOMEM;
3161         }
3162
3163         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3164                  mr->npages, addr);
3165
3166         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3167         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3168         pbe = (struct regpair *)pbl_table->va;
3169         pbe +=  mr->npages % pbes_in_page;
3170         pbe->lo = cpu_to_le32((u32)addr);
3171         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3172
3173         mr->npages++;
3174
3175         return 0;
3176 }
3177
3178 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3179 {
3180         int work = info->completed - info->completed_handled - 1;
3181
3182         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3183         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3184                 struct qedr_pbl *pbl;
3185
3186                 /* Free all the page list that are possible to be freed
3187                  * (all the ones that were invalidated), under the assumption
3188                  * that if an FMR was completed successfully that means that
3189                  * if there was an invalidate operation before it also ended
3190                  */
3191                 pbl = list_first_entry(&info->inuse_pbl_list,
3192                                        struct qedr_pbl, list_entry);
3193                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3194                 info->completed_handled++;
3195         }
3196 }
3197
3198 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3199                    int sg_nents, unsigned int *sg_offset)
3200 {
3201         struct qedr_mr *mr = get_qedr_mr(ibmr);
3202
3203         mr->npages = 0;
3204
3205         handle_completed_mrs(mr->dev, &mr->info);
3206         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3207 }
3208
3209 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3210 {
3211         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3212         struct qedr_pd *pd = get_qedr_pd(ibpd);
3213         struct qedr_mr *mr;
3214         int rc;
3215
3216         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3217         if (!mr)
3218                 return ERR_PTR(-ENOMEM);
3219
3220         mr->type = QEDR_MR_DMA;
3221
3222         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3223         if (rc) {
3224                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3225                 goto err1;
3226         }
3227
3228         /* index only, 18 bit long, lkey = itid << 8 | key */
3229         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3230         mr->hw_mr.pd = pd->pd_id;
3231         mr->hw_mr.local_read = 1;
3232         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3233         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3234         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3235         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3236         mr->hw_mr.dma_mr = true;
3237
3238         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3239         if (rc) {
3240                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3241                 goto err2;
3242         }
3243
3244         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3245         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3246             mr->hw_mr.remote_atomic)
3247                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3248
3249         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3250         return &mr->ibmr;
3251
3252 err2:
3253         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3254 err1:
3255         kfree(mr);
3256         return ERR_PTR(rc);
3257 }
3258
3259 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3260 {
3261         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3262 }
3263
3264 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3265 {
3266         int i, len = 0;
3267
3268         for (i = 0; i < num_sge; i++)
3269                 len += sg_list[i].length;
3270
3271         return len;
3272 }
3273
3274 static void swap_wqe_data64(u64 *p)
3275 {
3276         int i;
3277
3278         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3279                 *p = cpu_to_be64(cpu_to_le64(*p));
3280 }
3281
3282 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3283                                        struct qedr_qp *qp, u8 *wqe_size,
3284                                        const struct ib_send_wr *wr,
3285                                        const struct ib_send_wr **bad_wr,
3286                                        u8 *bits, u8 bit)
3287 {
3288         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3289         char *seg_prt, *wqe;
3290         int i, seg_siz;
3291
3292         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3293                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3294                 *bad_wr = wr;
3295                 return 0;
3296         }
3297
3298         if (!data_size)
3299                 return data_size;
3300
3301         *bits |= bit;
3302
3303         seg_prt = NULL;
3304         wqe = NULL;
3305         seg_siz = 0;
3306
3307         /* Copy data inline */
3308         for (i = 0; i < wr->num_sge; i++) {
3309                 u32 len = wr->sg_list[i].length;
3310                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3311
3312                 while (len > 0) {
3313                         u32 cur;
3314
3315                         /* New segment required */
3316                         if (!seg_siz) {
3317                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3318                                 seg_prt = wqe;
3319                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
3320                                 (*wqe_size)++;
3321                         }
3322
3323                         /* Calculate currently allowed length */
3324                         cur = min_t(u32, len, seg_siz);
3325                         memcpy(seg_prt, src, cur);
3326
3327                         /* Update segment variables */
3328                         seg_prt += cur;
3329                         seg_siz -= cur;
3330
3331                         /* Update sge variables */
3332                         src += cur;
3333                         len -= cur;
3334
3335                         /* Swap fully-completed segments */
3336                         if (!seg_siz)
3337                                 swap_wqe_data64((u64 *)wqe);
3338                 }
3339         }
3340
3341         /* swap last not completed segment */
3342         if (seg_siz)
3343                 swap_wqe_data64((u64 *)wqe);
3344
3345         return data_size;
3346 }
3347
3348 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
3349         do {                                                    \
3350                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3351                 (sge)->length = cpu_to_le32(vlength);           \
3352                 (sge)->flags = cpu_to_le32(vflags);             \
3353         } while (0)
3354
3355 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
3356         do {                                                    \
3357                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
3358                 (hdr)->num_sges = num_sge;                      \
3359         } while (0)
3360
3361 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
3362         do {                                                    \
3363                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3364                 (sge)->length = cpu_to_le32(vlength);           \
3365                 (sge)->l_key = cpu_to_le32(vlkey);              \
3366         } while (0)
3367
3368 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3369                                 const struct ib_send_wr *wr)
3370 {
3371         u32 data_size = 0;
3372         int i;
3373
3374         for (i = 0; i < wr->num_sge; i++) {
3375                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3376
3377                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3378                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3379                 sge->length = cpu_to_le32(wr->sg_list[i].length);
3380                 data_size += wr->sg_list[i].length;
3381         }
3382
3383         if (wqe_size)
3384                 *wqe_size += wr->num_sge;
3385
3386         return data_size;
3387 }
3388
3389 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3390                                      struct qedr_qp *qp,
3391                                      struct rdma_sq_rdma_wqe_1st *rwqe,
3392                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
3393                                      const struct ib_send_wr *wr,
3394                                      const struct ib_send_wr **bad_wr)
3395 {
3396         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3397         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3398
3399         if (wr->send_flags & IB_SEND_INLINE &&
3400             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3401              wr->opcode == IB_WR_RDMA_WRITE)) {
3402                 u8 flags = 0;
3403
3404                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3405                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3406                                                    bad_wr, &rwqe->flags, flags);
3407         }
3408
3409         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3410 }
3411
3412 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3413                                      struct qedr_qp *qp,
3414                                      struct rdma_sq_send_wqe_1st *swqe,
3415                                      struct rdma_sq_send_wqe_2st *swqe2,
3416                                      const struct ib_send_wr *wr,
3417                                      const struct ib_send_wr **bad_wr)
3418 {
3419         memset(swqe2, 0, sizeof(*swqe2));
3420         if (wr->send_flags & IB_SEND_INLINE) {
3421                 u8 flags = 0;
3422
3423                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3424                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3425                                                    bad_wr, &swqe->flags, flags);
3426         }
3427
3428         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3429 }
3430
3431 static int qedr_prepare_reg(struct qedr_qp *qp,
3432                             struct rdma_sq_fmr_wqe_1st *fwqe1,
3433                             const struct ib_reg_wr *wr)
3434 {
3435         struct qedr_mr *mr = get_qedr_mr(wr->mr);
3436         struct rdma_sq_fmr_wqe_2nd *fwqe2;
3437
3438         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3439         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3440         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3441         fwqe1->l_key = wr->key;
3442
3443         fwqe2->access_ctrl = 0;
3444
3445         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3446                    !!(wr->access & IB_ACCESS_REMOTE_READ));
3447         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3448                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3449         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3450                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3451         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3452         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3453                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3454         fwqe2->fmr_ctrl = 0;
3455
3456         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3457                    ilog2(mr->ibmr.page_size) - 12);
3458
3459         fwqe2->length_hi = 0;
3460         fwqe2->length_lo = mr->ibmr.length;
3461         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3462         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3463
3464         qp->wqe_wr_id[qp->sq.prod].mr = mr;
3465
3466         return 0;
3467 }
3468
3469 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3470 {
3471         switch (opcode) {
3472         case IB_WR_RDMA_WRITE:
3473         case IB_WR_RDMA_WRITE_WITH_IMM:
3474                 return IB_WC_RDMA_WRITE;
3475         case IB_WR_SEND_WITH_IMM:
3476         case IB_WR_SEND:
3477         case IB_WR_SEND_WITH_INV:
3478                 return IB_WC_SEND;
3479         case IB_WR_RDMA_READ:
3480         case IB_WR_RDMA_READ_WITH_INV:
3481                 return IB_WC_RDMA_READ;
3482         case IB_WR_ATOMIC_CMP_AND_SWP:
3483                 return IB_WC_COMP_SWAP;
3484         case IB_WR_ATOMIC_FETCH_AND_ADD:
3485                 return IB_WC_FETCH_ADD;
3486         case IB_WR_REG_MR:
3487                 return IB_WC_REG_MR;
3488         case IB_WR_LOCAL_INV:
3489                 return IB_WC_LOCAL_INV;
3490         default:
3491                 return IB_WC_SEND;
3492         }
3493 }
3494
3495 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3496                                       const struct ib_send_wr *wr)
3497 {
3498         int wq_is_full, err_wr, pbl_is_full;
3499         struct qedr_dev *dev = qp->dev;
3500
3501         /* prevent SQ overflow and/or processing of a bad WR */
3502         err_wr = wr->num_sge > qp->sq.max_sges;
3503         wq_is_full = qedr_wq_is_full(&qp->sq);
3504         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3505                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3506         if (wq_is_full || err_wr || pbl_is_full) {
3507                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3508                         DP_ERR(dev,
3509                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3510                                qp);
3511                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3512                 }
3513
3514                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3515                         DP_ERR(dev,
3516                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3517                                qp);
3518                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3519                 }
3520
3521                 if (pbl_is_full &&
3522                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3523                         DP_ERR(dev,
3524                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3525                                qp);
3526                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3527                 }
3528                 return false;
3529         }
3530         return true;
3531 }
3532
3533 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3534                             const struct ib_send_wr **bad_wr)
3535 {
3536         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3537         struct qedr_qp *qp = get_qedr_qp(ibqp);
3538         struct rdma_sq_atomic_wqe_1st *awqe1;
3539         struct rdma_sq_atomic_wqe_2nd *awqe2;
3540         struct rdma_sq_atomic_wqe_3rd *awqe3;
3541         struct rdma_sq_send_wqe_2st *swqe2;
3542         struct rdma_sq_local_inv_wqe *iwqe;
3543         struct rdma_sq_rdma_wqe_2nd *rwqe2;
3544         struct rdma_sq_send_wqe_1st *swqe;
3545         struct rdma_sq_rdma_wqe_1st *rwqe;
3546         struct rdma_sq_fmr_wqe_1st *fwqe1;
3547         struct rdma_sq_common_wqe *wqe;
3548         u32 length;
3549         int rc = 0;
3550         bool comp;
3551
3552         if (!qedr_can_post_send(qp, wr)) {
3553                 *bad_wr = wr;
3554                 return -ENOMEM;
3555         }
3556
3557         wqe = qed_chain_produce(&qp->sq.pbl);
3558         qp->wqe_wr_id[qp->sq.prod].signaled =
3559                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3560
3561         wqe->flags = 0;
3562         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3563                    !!(wr->send_flags & IB_SEND_SOLICITED));
3564         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3565         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3566         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3567                    !!(wr->send_flags & IB_SEND_FENCE));
3568         wqe->prev_wqe_size = qp->prev_wqe_size;
3569
3570         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3571
3572         switch (wr->opcode) {
3573         case IB_WR_SEND_WITH_IMM:
3574                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3575                         rc = -EINVAL;
3576                         *bad_wr = wr;
3577                         break;
3578                 }
3579                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3580                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3581                 swqe->wqe_size = 2;
3582                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3583
3584                 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3585                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3586                                                    wr, bad_wr);
3587                 swqe->length = cpu_to_le32(length);
3588                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3589                 qp->prev_wqe_size = swqe->wqe_size;
3590                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3591                 break;
3592         case IB_WR_SEND:
3593                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3594                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3595
3596                 swqe->wqe_size = 2;
3597                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3598                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3599                                                    wr, bad_wr);
3600                 swqe->length = cpu_to_le32(length);
3601                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3602                 qp->prev_wqe_size = swqe->wqe_size;
3603                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3604                 break;
3605         case IB_WR_SEND_WITH_INV:
3606                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3607                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3608                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3609                 swqe->wqe_size = 2;
3610                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3611                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3612                                                    wr, bad_wr);
3613                 swqe->length = cpu_to_le32(length);
3614                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3615                 qp->prev_wqe_size = swqe->wqe_size;
3616                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3617                 break;
3618
3619         case IB_WR_RDMA_WRITE_WITH_IMM:
3620                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3621                         rc = -EINVAL;
3622                         *bad_wr = wr;
3623                         break;
3624                 }
3625                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3626                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3627
3628                 rwqe->wqe_size = 2;
3629                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3630                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3631                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3632                                                    wr, bad_wr);
3633                 rwqe->length = cpu_to_le32(length);
3634                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3635                 qp->prev_wqe_size = rwqe->wqe_size;
3636                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3637                 break;
3638         case IB_WR_RDMA_WRITE:
3639                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3640                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3641
3642                 rwqe->wqe_size = 2;
3643                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3644                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3645                                                    wr, bad_wr);
3646                 rwqe->length = cpu_to_le32(length);
3647                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3648                 qp->prev_wqe_size = rwqe->wqe_size;
3649                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3650                 break;
3651         case IB_WR_RDMA_READ_WITH_INV:
3652                 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3653                 fallthrough;    /* same is identical to RDMA READ */
3654
3655         case IB_WR_RDMA_READ:
3656                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3657                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3658
3659                 rwqe->wqe_size = 2;
3660                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3661                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3662                                                    wr, bad_wr);
3663                 rwqe->length = cpu_to_le32(length);
3664                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3665                 qp->prev_wqe_size = rwqe->wqe_size;
3666                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3667                 break;
3668
3669         case IB_WR_ATOMIC_CMP_AND_SWP:
3670         case IB_WR_ATOMIC_FETCH_AND_ADD:
3671                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3672                 awqe1->wqe_size = 4;
3673
3674                 awqe2 = qed_chain_produce(&qp->sq.pbl);
3675                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3676                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3677
3678                 awqe3 = qed_chain_produce(&qp->sq.pbl);
3679
3680                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3681                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3682                         DMA_REGPAIR_LE(awqe3->swap_data,
3683                                        atomic_wr(wr)->compare_add);
3684                 } else {
3685                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3686                         DMA_REGPAIR_LE(awqe3->swap_data,
3687                                        atomic_wr(wr)->swap);
3688                         DMA_REGPAIR_LE(awqe3->cmp_data,
3689                                        atomic_wr(wr)->compare_add);
3690                 }
3691
3692                 qedr_prepare_sq_sges(qp, NULL, wr);
3693
3694                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3695                 qp->prev_wqe_size = awqe1->wqe_size;
3696                 break;
3697
3698         case IB_WR_LOCAL_INV:
3699                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3700                 iwqe->wqe_size = 1;
3701
3702                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3703                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3704                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3705                 qp->prev_wqe_size = iwqe->wqe_size;
3706                 break;
3707         case IB_WR_REG_MR:
3708                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3709                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3710                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3711                 fwqe1->wqe_size = 2;
3712
3713                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3714                 if (rc) {
3715                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3716                         *bad_wr = wr;
3717                         break;
3718                 }
3719
3720                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3721                 qp->prev_wqe_size = fwqe1->wqe_size;
3722                 break;
3723         default:
3724                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3725                 rc = -EINVAL;
3726                 *bad_wr = wr;
3727                 break;
3728         }
3729
3730         if (*bad_wr) {
3731                 u16 value;
3732
3733                 /* Restore prod to its position before
3734                  * this WR was processed
3735                  */
3736                 value = le16_to_cpu(qp->sq.db_data.data.value);
3737                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3738
3739                 /* Restore prev_wqe_size */
3740                 qp->prev_wqe_size = wqe->prev_wqe_size;
3741                 rc = -EINVAL;
3742                 DP_ERR(dev, "POST SEND FAILED\n");
3743         }
3744
3745         return rc;
3746 }
3747
3748 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3749                    const struct ib_send_wr **bad_wr)
3750 {
3751         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3752         struct qedr_qp *qp = get_qedr_qp(ibqp);
3753         unsigned long flags;
3754         int rc = 0;
3755
3756         *bad_wr = NULL;
3757
3758         if (qp->qp_type == IB_QPT_GSI)
3759                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3760
3761         spin_lock_irqsave(&qp->q_lock, flags);
3762
3763         if (rdma_protocol_roce(&dev->ibdev, 1)) {
3764                 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3765                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
3766                     (qp->state != QED_ROCE_QP_STATE_SQD)) {
3767                         spin_unlock_irqrestore(&qp->q_lock, flags);
3768                         *bad_wr = wr;
3769                         DP_DEBUG(dev, QEDR_MSG_CQ,
3770                                  "QP in wrong state! QP icid=0x%x state %d\n",
3771                                  qp->icid, qp->state);
3772                         return -EINVAL;
3773                 }
3774         }
3775
3776         while (wr) {
3777                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3778                 if (rc)
3779                         break;
3780
3781                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3782
3783                 qedr_inc_sw_prod(&qp->sq);
3784
3785                 qp->sq.db_data.data.value++;
3786
3787                 wr = wr->next;
3788         }
3789
3790         /* Trigger doorbell
3791          * If there was a failure in the first WR then it will be triggered in
3792          * vane. However this is not harmful (as long as the producer value is
3793          * unchanged). For performance reasons we avoid checking for this
3794          * redundant doorbell.
3795          *
3796          * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3797          * soon as we give the doorbell, we could get a completion
3798          * for this wr, therefore we need to make sure that the
3799          * memory is updated before giving the doorbell.
3800          * During qedr_poll_cq, rmb is called before accessing the
3801          * cqe. This covers for the smp_rmb as well.
3802          */
3803         smp_wmb();
3804         writel(qp->sq.db_data.raw, qp->sq.db);
3805
3806         spin_unlock_irqrestore(&qp->q_lock, flags);
3807
3808         return rc;
3809 }
3810
3811 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3812 {
3813         u32 used;
3814
3815         /* Calculate number of elements used based on producer
3816          * count and consumer count and subtract it from max
3817          * work request supported so that we get elements left.
3818          */
3819         used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3820
3821         return hw_srq->max_wr - used;
3822 }
3823
3824 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3825                        const struct ib_recv_wr **bad_wr)
3826 {
3827         struct qedr_srq *srq = get_qedr_srq(ibsrq);
3828         struct qedr_srq_hwq_info *hw_srq;
3829         struct qedr_dev *dev = srq->dev;
3830         struct qed_chain *pbl;
3831         unsigned long flags;
3832         int status = 0;
3833         u32 num_sge;
3834
3835         spin_lock_irqsave(&srq->lock, flags);
3836
3837         hw_srq = &srq->hw_srq;
3838         pbl = &srq->hw_srq.pbl;
3839         while (wr) {
3840                 struct rdma_srq_wqe_header *hdr;
3841                 int i;
3842
3843                 if (!qedr_srq_elem_left(hw_srq) ||
3844                     wr->num_sge > srq->hw_srq.max_sges) {
3845                         DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3846                                hw_srq->wr_prod_cnt,
3847                                atomic_read(&hw_srq->wr_cons_cnt),
3848                                wr->num_sge, srq->hw_srq.max_sges);
3849                         status = -ENOMEM;
3850                         *bad_wr = wr;
3851                         break;
3852                 }
3853
3854                 hdr = qed_chain_produce(pbl);
3855                 num_sge = wr->num_sge;
3856                 /* Set number of sge and work request id in header */
3857                 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3858
3859                 srq->hw_srq.wr_prod_cnt++;
3860                 hw_srq->wqe_prod++;
3861                 hw_srq->sge_prod++;
3862
3863                 DP_DEBUG(dev, QEDR_MSG_SRQ,
3864                          "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3865                          wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3866
3867                 for (i = 0; i < wr->num_sge; i++) {
3868                         struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3869
3870                         /* Set SGE length, lkey and address */
3871                         SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3872                                     wr->sg_list[i].length, wr->sg_list[i].lkey);
3873
3874                         DP_DEBUG(dev, QEDR_MSG_SRQ,
3875                                  "[%d]: len %d key %x addr %x:%x\n",
3876                                  i, srq_sge->length, srq_sge->l_key,
3877                                  srq_sge->addr.hi, srq_sge->addr.lo);
3878                         hw_srq->sge_prod++;
3879                 }
3880
3881                 /* Update WQE and SGE information before
3882                  * updating producer.
3883                  */
3884                 dma_wmb();
3885
3886                 /* SRQ producer is 8 bytes. Need to update SGE producer index
3887                  * in first 4 bytes and need to update WQE producer in
3888                  * next 4 bytes.
3889                  */
3890                 srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
3891                 /* Make sure sge producer is updated first */
3892                 dma_wmb();
3893                 srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3894
3895                 wr = wr->next;
3896         }
3897
3898         DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3899                  qed_chain_get_elem_left(pbl));
3900         spin_unlock_irqrestore(&srq->lock, flags);
3901
3902         return status;
3903 }
3904
3905 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3906                    const struct ib_recv_wr **bad_wr)
3907 {
3908         struct qedr_qp *qp = get_qedr_qp(ibqp);
3909         struct qedr_dev *dev = qp->dev;
3910         unsigned long flags;
3911         int status = 0;
3912
3913         if (qp->qp_type == IB_QPT_GSI)
3914                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3915
3916         spin_lock_irqsave(&qp->q_lock, flags);
3917
3918         if (qp->state == QED_ROCE_QP_STATE_RESET) {
3919                 spin_unlock_irqrestore(&qp->q_lock, flags);
3920                 *bad_wr = wr;
3921                 return -EINVAL;
3922         }
3923
3924         while (wr) {
3925                 int i;
3926
3927                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3928                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3929                     wr->num_sge > qp->rq.max_sges) {
3930                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3931                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3932                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3933                                qp->rq.max_sges);
3934                         status = -ENOMEM;
3935                         *bad_wr = wr;
3936                         break;
3937                 }
3938                 for (i = 0; i < wr->num_sge; i++) {
3939                         u32 flags = 0;
3940                         struct rdma_rq_sge *rqe =
3941                             qed_chain_produce(&qp->rq.pbl);
3942
3943                         /* First one must include the number
3944                          * of SGE in the list
3945                          */
3946                         if (!i)
3947                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3948                                           wr->num_sge);
3949
3950                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3951                                   wr->sg_list[i].lkey);
3952
3953                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3954                                    wr->sg_list[i].length, flags);
3955                 }
3956
3957                 /* Special case of no sges. FW requires between 1-4 sges...
3958                  * in this case we need to post 1 sge with length zero. this is
3959                  * because rdma write with immediate consumes an RQ.
3960                  */
3961                 if (!wr->num_sge) {
3962                         u32 flags = 0;
3963                         struct rdma_rq_sge *rqe =
3964                             qed_chain_produce(&qp->rq.pbl);
3965
3966                         /* First one must include the number
3967                          * of SGE in the list
3968                          */
3969                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3970                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3971
3972                         RQ_SGE_SET(rqe, 0, 0, flags);
3973                         i = 1;
3974                 }
3975
3976                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3977                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3978
3979                 qedr_inc_sw_prod(&qp->rq);
3980
3981                 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3982                  * soon as we give the doorbell, we could get a completion
3983                  * for this wr, therefore we need to make sure that the
3984                  * memory is update before giving the doorbell.
3985                  * During qedr_poll_cq, rmb is called before accessing the
3986                  * cqe. This covers for the smp_rmb as well.
3987                  */
3988                 smp_wmb();
3989
3990                 qp->rq.db_data.data.value++;
3991
3992                 writel(qp->rq.db_data.raw, qp->rq.db);
3993
3994                 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3995                         writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3996                 }
3997
3998                 wr = wr->next;
3999         }
4000
4001         spin_unlock_irqrestore(&qp->q_lock, flags);
4002
4003         return status;
4004 }
4005
4006 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
4007 {
4008         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4009
4010         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4011                 cq->pbl_toggle;
4012 }
4013
4014 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4015 {
4016         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4017         struct qedr_qp *qp;
4018
4019         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4020                                                    resp_cqe->qp_handle.lo,
4021                                                    u64);
4022         return qp;
4023 }
4024
4025 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4026 {
4027         struct rdma_cqe_requester *resp_cqe = &cqe->req;
4028
4029         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4030 }
4031
4032 /* Return latest CQE (needs processing) */
4033 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4034 {
4035         return cq->latest_cqe;
4036 }
4037
4038 /* In fmr we need to increase the number of fmr completed counter for the fmr
4039  * algorithm determining whether we can free a pbl or not.
4040  * we need to perform this whether the work request was signaled or not. for
4041  * this purpose we call this function from the condition that checks if a wr
4042  * should be skipped, to make sure we don't miss it ( possibly this fmr
4043  * operation was not signalted)
4044  */
4045 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4046 {
4047         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4048                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4049 }
4050
4051 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4052                        struct qedr_cq *cq, int num_entries,
4053                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4054                        int force)
4055 {
4056         u16 cnt = 0;
4057
4058         while (num_entries && qp->sq.wqe_cons != hw_cons) {
4059                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4060                         qedr_chk_if_fmr(qp);
4061                         /* skip WC */
4062                         goto next_cqe;
4063                 }
4064
4065                 /* fill WC */
4066                 wc->status = status;
4067                 wc->vendor_err = 0;
4068                 wc->wc_flags = 0;
4069                 wc->src_qp = qp->id;
4070                 wc->qp = &qp->ibqp;
4071
4072                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4073                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4074
4075                 switch (wc->opcode) {
4076                 case IB_WC_RDMA_WRITE:
4077                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4078                         break;
4079                 case IB_WC_COMP_SWAP:
4080                 case IB_WC_FETCH_ADD:
4081                         wc->byte_len = 8;
4082                         break;
4083                 case IB_WC_REG_MR:
4084                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4085                         break;
4086                 case IB_WC_RDMA_READ:
4087                 case IB_WC_SEND:
4088                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4089                         break;
4090                 default:
4091                         break;
4092                 }
4093
4094                 num_entries--;
4095                 wc++;
4096                 cnt++;
4097 next_cqe:
4098                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4099                         qed_chain_consume(&qp->sq.pbl);
4100                 qedr_inc_sw_cons(&qp->sq);
4101         }
4102
4103         return cnt;
4104 }
4105
4106 static int qedr_poll_cq_req(struct qedr_dev *dev,
4107                             struct qedr_qp *qp, struct qedr_cq *cq,
4108                             int num_entries, struct ib_wc *wc,
4109                             struct rdma_cqe_requester *req)
4110 {
4111         int cnt = 0;
4112
4113         switch (req->status) {
4114         case RDMA_CQE_REQ_STS_OK:
4115                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4116                                   IB_WC_SUCCESS, 0);
4117                 break;
4118         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
4119                 if (qp->state != QED_ROCE_QP_STATE_ERR)
4120                         DP_DEBUG(dev, QEDR_MSG_CQ,
4121                                  "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4122                                  cq->icid, qp->icid);
4123                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4124                                   IB_WC_WR_FLUSH_ERR, 1);
4125                 break;
4126         default:
4127                 /* process all WQE before the cosumer */
4128                 qp->state = QED_ROCE_QP_STATE_ERR;
4129                 cnt = process_req(dev, qp, cq, num_entries, wc,
4130                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
4131                 wc += cnt;
4132                 /* if we have extra WC fill it with actual error info */
4133                 if (cnt < num_entries) {
4134                         enum ib_wc_status wc_status;
4135
4136                         switch (req->status) {
4137                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4138                                 DP_ERR(dev,
4139                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4140                                        cq->icid, qp->icid);
4141                                 wc_status = IB_WC_BAD_RESP_ERR;
4142                                 break;
4143                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4144                                 DP_ERR(dev,
4145                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4146                                        cq->icid, qp->icid);
4147                                 wc_status = IB_WC_LOC_LEN_ERR;
4148                                 break;
4149                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4150                                 DP_ERR(dev,
4151                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4152                                        cq->icid, qp->icid);
4153                                 wc_status = IB_WC_LOC_QP_OP_ERR;
4154                                 break;
4155                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4156                                 DP_ERR(dev,
4157                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4158                                        cq->icid, qp->icid);
4159                                 wc_status = IB_WC_LOC_PROT_ERR;
4160                                 break;
4161                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4162                                 DP_ERR(dev,
4163                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4164                                        cq->icid, qp->icid);
4165                                 wc_status = IB_WC_MW_BIND_ERR;
4166                                 break;
4167                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4168                                 DP_ERR(dev,
4169                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4170                                        cq->icid, qp->icid);
4171                                 wc_status = IB_WC_REM_INV_REQ_ERR;
4172                                 break;
4173                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4174                                 DP_ERR(dev,
4175                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4176                                        cq->icid, qp->icid);
4177                                 wc_status = IB_WC_REM_ACCESS_ERR;
4178                                 break;
4179                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4180                                 DP_ERR(dev,
4181                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4182                                        cq->icid, qp->icid);
4183                                 wc_status = IB_WC_REM_OP_ERR;
4184                                 break;
4185                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4186                                 DP_ERR(dev,
4187                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4188                                        cq->icid, qp->icid);
4189                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4190                                 break;
4191                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4192                                 DP_ERR(dev,
4193                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4194                                        cq->icid, qp->icid);
4195                                 wc_status = IB_WC_RETRY_EXC_ERR;
4196                                 break;
4197                         default:
4198                                 DP_ERR(dev,
4199                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4200                                        cq->icid, qp->icid);
4201                                 wc_status = IB_WC_GENERAL_ERR;
4202                         }
4203                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4204                                            wc_status, 1);
4205                 }
4206         }
4207
4208         return cnt;
4209 }
4210
4211 static inline int qedr_cqe_resp_status_to_ib(u8 status)
4212 {
4213         switch (status) {
4214         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4215                 return IB_WC_LOC_ACCESS_ERR;
4216         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4217                 return IB_WC_LOC_LEN_ERR;
4218         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4219                 return IB_WC_LOC_QP_OP_ERR;
4220         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4221                 return IB_WC_LOC_PROT_ERR;
4222         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4223                 return IB_WC_MW_BIND_ERR;
4224         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4225                 return IB_WC_REM_INV_RD_REQ_ERR;
4226         case RDMA_CQE_RESP_STS_OK:
4227                 return IB_WC_SUCCESS;
4228         default:
4229                 return IB_WC_GENERAL_ERR;
4230         }
4231 }
4232
4233 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4234                                           struct ib_wc *wc)
4235 {
4236         wc->status = IB_WC_SUCCESS;
4237         wc->byte_len = le32_to_cpu(resp->length);
4238
4239         if (resp->flags & QEDR_RESP_IMM) {
4240                 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4241                 wc->wc_flags |= IB_WC_WITH_IMM;
4242
4243                 if (resp->flags & QEDR_RESP_RDMA)
4244                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4245
4246                 if (resp->flags & QEDR_RESP_INV)
4247                         return -EINVAL;
4248
4249         } else if (resp->flags & QEDR_RESP_INV) {
4250                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4251                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4252
4253                 if (resp->flags & QEDR_RESP_RDMA)
4254                         return -EINVAL;
4255
4256         } else if (resp->flags & QEDR_RESP_RDMA) {
4257                 return -EINVAL;
4258         }
4259
4260         return 0;
4261 }
4262
4263 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4264                                struct qedr_cq *cq, struct ib_wc *wc,
4265                                struct rdma_cqe_responder *resp, u64 wr_id)
4266 {
4267         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4268         wc->opcode = IB_WC_RECV;
4269         wc->wc_flags = 0;
4270
4271         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4272                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4273                         DP_ERR(dev,
4274                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4275                                cq, cq->icid, resp->flags);
4276
4277         } else {
4278                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4279                 if (wc->status == IB_WC_GENERAL_ERR)
4280                         DP_ERR(dev,
4281                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4282                                cq, cq->icid, resp->status);
4283         }
4284
4285         /* Fill the rest of the WC */
4286         wc->vendor_err = 0;
4287         wc->src_qp = qp->id;
4288         wc->qp = &qp->ibqp;
4289         wc->wr_id = wr_id;
4290 }
4291
4292 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4293                                 struct qedr_cq *cq, struct ib_wc *wc,
4294                                 struct rdma_cqe_responder *resp)
4295 {
4296         struct qedr_srq *srq = qp->srq;
4297         u64 wr_id;
4298
4299         wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4300                          le32_to_cpu(resp->srq_wr_id.lo), u64);
4301
4302         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4303                 wc->status = IB_WC_WR_FLUSH_ERR;
4304                 wc->vendor_err = 0;
4305                 wc->wr_id = wr_id;
4306                 wc->byte_len = 0;
4307                 wc->src_qp = qp->id;
4308                 wc->qp = &qp->ibqp;
4309                 wc->wr_id = wr_id;
4310         } else {
4311                 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4312         }
4313         atomic_inc(&srq->hw_srq.wr_cons_cnt);
4314
4315         return 1;
4316 }
4317 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4318                             struct qedr_cq *cq, struct ib_wc *wc,
4319                             struct rdma_cqe_responder *resp)
4320 {
4321         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4322
4323         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4324
4325         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4326                 qed_chain_consume(&qp->rq.pbl);
4327         qedr_inc_sw_cons(&qp->rq);
4328
4329         return 1;
4330 }
4331
4332 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4333                               int num_entries, struct ib_wc *wc, u16 hw_cons)
4334 {
4335         u16 cnt = 0;
4336
4337         while (num_entries && qp->rq.wqe_cons != hw_cons) {
4338                 /* fill WC */
4339                 wc->status = IB_WC_WR_FLUSH_ERR;
4340                 wc->vendor_err = 0;
4341                 wc->wc_flags = 0;
4342                 wc->src_qp = qp->id;
4343                 wc->byte_len = 0;
4344                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4345                 wc->qp = &qp->ibqp;
4346                 num_entries--;
4347                 wc++;
4348                 cnt++;
4349                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4350                         qed_chain_consume(&qp->rq.pbl);
4351                 qedr_inc_sw_cons(&qp->rq);
4352         }
4353
4354         return cnt;
4355 }
4356
4357 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4358                                  struct rdma_cqe_responder *resp, int *update)
4359 {
4360         if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4361                 consume_cqe(cq);
4362                 *update |= 1;
4363         }
4364 }
4365
4366 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4367                                  struct qedr_cq *cq, int num_entries,
4368                                  struct ib_wc *wc,
4369                                  struct rdma_cqe_responder *resp)
4370 {
4371         int cnt;
4372
4373         cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4374         consume_cqe(cq);
4375
4376         return cnt;
4377 }
4378
4379 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4380                              struct qedr_cq *cq, int num_entries,
4381                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
4382                              int *update)
4383 {
4384         int cnt;
4385
4386         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4387                 cnt = process_resp_flush(qp, cq, num_entries, wc,
4388                                          resp->rq_cons_or_srq_id);
4389                 try_consume_resp_cqe(cq, qp, resp, update);
4390         } else {
4391                 cnt = process_resp_one(dev, qp, cq, wc, resp);
4392                 consume_cqe(cq);
4393                 *update |= 1;
4394         }
4395
4396         return cnt;
4397 }
4398
4399 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4400                                 struct rdma_cqe_requester *req, int *update)
4401 {
4402         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4403                 consume_cqe(cq);
4404                 *update |= 1;
4405         }
4406 }
4407
4408 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4409 {
4410         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4411         struct qedr_cq *cq = get_qedr_cq(ibcq);
4412         union rdma_cqe *cqe;
4413         u32 old_cons, new_cons;
4414         unsigned long flags;
4415         int update = 0;
4416         int done = 0;
4417
4418         if (cq->destroyed) {
4419                 DP_ERR(dev,
4420                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4421                        cq, cq->icid);
4422                 return 0;
4423         }
4424
4425         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4426                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4427
4428         spin_lock_irqsave(&cq->cq_lock, flags);
4429         cqe = cq->latest_cqe;
4430         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4431         while (num_entries && is_valid_cqe(cq, cqe)) {
4432                 struct qedr_qp *qp;
4433                 int cnt = 0;
4434
4435                 /* prevent speculative reads of any field of CQE */
4436                 rmb();
4437
4438                 qp = cqe_get_qp(cqe);
4439                 if (!qp) {
4440                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4441                         break;
4442                 }
4443
4444                 wc->qp = &qp->ibqp;
4445
4446                 switch (cqe_get_type(cqe)) {
4447                 case RDMA_CQE_TYPE_REQUESTER:
4448                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4449                                                &cqe->req);
4450                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
4451                         break;
4452                 case RDMA_CQE_TYPE_RESPONDER_RQ:
4453                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4454                                                 &cqe->resp, &update);
4455                         break;
4456                 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4457                         cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4458                                                     wc, &cqe->resp);
4459                         update = 1;
4460                         break;
4461                 case RDMA_CQE_TYPE_INVALID:
4462                 default:
4463                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
4464                                cqe_get_type(cqe));
4465                 }
4466                 num_entries -= cnt;
4467                 wc += cnt;
4468                 done += cnt;
4469
4470                 cqe = get_cqe(cq);
4471         }
4472         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4473
4474         cq->cq_cons += new_cons - old_cons;
4475
4476         if (update)
4477                 /* doorbell notifies abount latest VALID entry,
4478                  * but chain already point to the next INVALID one
4479                  */
4480                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4481
4482         spin_unlock_irqrestore(&cq->cq_lock, flags);
4483         return done;
4484 }
4485
4486 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4487                      u32 port_num, const struct ib_wc *in_wc,
4488                      const struct ib_grh *in_grh, const struct ib_mad *in,
4489                      struct ib_mad *out_mad, size_t *out_mad_size,
4490                      u16 *out_mad_pkey_index)
4491 {
4492         return IB_MAD_RESULT_SUCCESS;
4493 }