drivers/infiniband/hw/i40iw/i40iw_verbs.c

   1 /*******************************************************************************
   2 *
   3 * Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenFabrics.org BSD license below:
  10 *
  11 *   Redistribution and use in source and binary forms, with or
  12 *   without modification, are permitted provided that the following
  13 *   conditions are met:
  14 *
  15 *    - Redistributions of source code must retain the above
  16 *       copyright notice, this list of conditions and the following
  17 *       disclaimer.
  18 *
  19 *    - Redistributions in binary form must reproduce the above
  20 *       copyright notice, this list of conditions and the following
  21 *       disclaimer in the documentation and/or other materials
  22 *       provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 *
  33 *******************************************************************************/
  34
  35 #include <linux/module.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/random.h>
  38 #include <linux/highmem.h>
  39 #include <linux/time.h>
  40 #include <linux/hugetlb.h>
  41 #include <linux/irq.h>
  42 #include <asm/byteorder.h>
  43 #include <net/ip.h>
  44 #include <rdma/ib_verbs.h>
  45 #include <rdma/iw_cm.h>
  46 #include <rdma/ib_user_verbs.h>
  47 #include <rdma/ib_umem.h>
  48 #include <rdma/uverbs_ioctl.h>
  49 #include "i40iw.h"
  50
  51 /**
  52  * i40iw_query_device - get device attributes
  53  * @ibdev: device pointer from stack
  54  * @props: returning device attributes
  55  * @udata: user data
  56  */
  57 static int i40iw_query_device(struct ib_device *ibdev,
  58                               struct ib_device_attr *props,
  59                               struct ib_udata *udata)
  60 {
  61         struct i40iw_device *iwdev = to_iwdev(ibdev);
  62
  63         if (udata->inlen || udata->outlen)
  64                 return -EINVAL;
  65         memset(props, 0, sizeof(*props));
  66         ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr);
  67         props->fw_ver = i40iw_fw_major_ver(&iwdev->sc_dev) << 32 |
  68                         i40iw_fw_minor_ver(&iwdev->sc_dev);
  69         props->device_cap_flags = iwdev->device_cap_flags;
  70         props->vendor_id = iwdev->ldev->pcidev->vendor;
  71         props->vendor_part_id = iwdev->ldev->pcidev->device;
  72         props->hw_ver = (u32)iwdev->sc_dev.hw_rev;
  73         props->max_mr_size = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
  74         props->max_qp = iwdev->max_qp - iwdev->used_qps;
  75         props->max_qp_wr = I40IW_MAX_QP_WRS;
  76         props->max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
  77         props->max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
  78         props->max_cq = iwdev->max_cq - iwdev->used_cqs;
  79         props->max_cqe = iwdev->max_cqe;
  80         props->max_mr = iwdev->max_mr - iwdev->used_mrs;
  81         props->max_pd = iwdev->max_pd - iwdev->used_pds;
  82         props->max_sge_rd = I40IW_MAX_SGE_RD;
  83         props->max_qp_rd_atom = I40IW_MAX_IRD_SIZE;
  84         props->max_qp_init_rd_atom = props->max_qp_rd_atom;
  85         props->atomic_cap = IB_ATOMIC_NONE;
  86         props->max_fast_reg_page_list_len = I40IW_MAX_PAGES_PER_FMR;
  87         return 0;
  88 }
  89
  90 /**
  91  * i40iw_query_port - get port attrubutes
  92  * @ibdev: device pointer from stack
  93  * @port: port number for query
  94  * @props: returning device attributes
  95  */
  96 static int i40iw_query_port(struct ib_device *ibdev,
  97                             u8 port,
  98                             struct ib_port_attr *props)
  99 {
 100         props->lid = 1;
 101         props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 102                 IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
 103         props->gid_tbl_len = 1;
 104         props->active_width = IB_WIDTH_4X;
 105         props->active_speed = 1;
 106         props->max_msg_sz = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
 107         return 0;
 108 }
 109
 110 /**
 111  * i40iw_alloc_ucontext - Allocate the user context data structure
 112  * @uctx: Uverbs context pointer from stack
 113  * @udata: user data
 114  *
 115  * This keeps track of all objects associated with a particular
 116  * user-mode client.
 117  */
 118 static int i40iw_alloc_ucontext(struct ib_ucontext *uctx,
 119                                 struct ib_udata *udata)
 120 {
 121         struct ib_device *ibdev = uctx->device;
 122         struct i40iw_device *iwdev = to_iwdev(ibdev);
 123         struct i40iw_alloc_ucontext_req req;
 124         struct i40iw_alloc_ucontext_resp uresp = {};
 125         struct i40iw_ucontext *ucontext = to_ucontext(uctx);
 126
 127         if (ib_copy_from_udata(&req, udata, sizeof(req)))
 128                 return -EINVAL;
 129
 130         if (req.userspace_ver < 4 || req.userspace_ver > I40IW_ABI_VER) {
 131                 i40iw_pr_err("Unsupported provider library version %u.\n", req.userspace_ver);
 132                 return -EINVAL;
 133         }
 134
 135         uresp.max_qps = iwdev->max_qp;
 136         uresp.max_pds = iwdev->max_pd;
 137         uresp.wq_size = iwdev->max_qp_wr * 2;
 138         uresp.kernel_ver = req.userspace_ver;
 139
 140         ucontext->iwdev = iwdev;
 141         ucontext->abi_ver = req.userspace_ver;
 142
 143         if (ib_copy_to_udata(udata, &uresp, sizeof(uresp)))
 144                 return -EFAULT;
 145
 146         INIT_LIST_HEAD(&ucontext->cq_reg_mem_list);
 147         spin_lock_init(&ucontext->cq_reg_mem_list_lock);
 148         INIT_LIST_HEAD(&ucontext->qp_reg_mem_list);
 149         spin_lock_init(&ucontext->qp_reg_mem_list_lock);
 150
 151         return 0;
 152 }
 153
 154 /**
 155  * i40iw_dealloc_ucontext - deallocate the user context data structure
 156  * @context: user context created during alloc
 157  */
 158 static void i40iw_dealloc_ucontext(struct ib_ucontext *context)
 159 {
 160         return;
 161 }
 162
 163 /**
 164  * i40iw_mmap - user memory map
 165  * @context: context created during alloc
 166  * @vma: kernel info for user memory map
 167  */
 168 static int i40iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 169 {
 170         struct i40iw_ucontext *ucontext;
 171         u64 db_addr_offset, push_offset, pfn;
 172
 173         ucontext = to_ucontext(context);
 174         if (ucontext->iwdev->sc_dev.is_pf) {
 175                 db_addr_offset = I40IW_DB_ADDR_OFFSET;
 176                 push_offset = I40IW_PUSH_OFFSET;
 177                 if (vma->vm_pgoff)
 178                         vma->vm_pgoff += I40IW_PF_FIRST_PUSH_PAGE_INDEX - 1;
 179         } else {
 180                 db_addr_offset = I40IW_VF_DB_ADDR_OFFSET;
 181                 push_offset = I40IW_VF_PUSH_OFFSET;
 182                 if (vma->vm_pgoff)
 183                         vma->vm_pgoff += I40IW_VF_FIRST_PUSH_PAGE_INDEX - 1;
 184         }
 185
 186         vma->vm_pgoff += db_addr_offset >> PAGE_SHIFT;
 187
 188         if (vma->vm_pgoff == (db_addr_offset >> PAGE_SHIFT)) {
 189                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 190         } else {
 191                 if ((vma->vm_pgoff - (push_offset >> PAGE_SHIFT)) % 2)
 192                         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 193                 else
 194                         vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 195         }
 196
 197         pfn = vma->vm_pgoff +
 198               (pci_resource_start(ucontext->iwdev->ldev->pcidev, 0) >>
 199                PAGE_SHIFT);
 200
 201         return rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,
 202                                  vma->vm_page_prot, NULL);
 203 }
 204
 205 /**
 206  * i40iw_alloc_push_page - allocate a push page for qp
 207  * @iwdev: iwarp device
 208  * @qp: hardware control qp
 209  */
 210 static void i40iw_alloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp)
 211 {
 212         struct i40iw_cqp_request *cqp_request;
 213         struct cqp_commands_info *cqp_info;
 214         enum i40iw_status_code status;
 215
 216         if (qp->push_idx != I40IW_INVALID_PUSH_PAGE_INDEX)
 217                 return;
 218
 219         cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
 220         if (!cqp_request)
 221                 return;
 222
 223         atomic_inc(&cqp_request->refcount);
 224
 225         cqp_info = &cqp_request->info;
 226         cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE;
 227         cqp_info->post_sq = 1;
 228
 229         cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
 230         cqp_info->in.u.manage_push_page.info.free_page = 0;
 231         cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;
 232         cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
 233
 234         status = i40iw_handle_cqp_op(iwdev, cqp_request);
 235         if (!status)
 236                 qp->push_idx = cqp_request->compl_info.op_ret_val;
 237         else
 238                 i40iw_pr_err("CQP-OP Push page fail");
 239         i40iw_put_cqp_request(&iwdev->cqp, cqp_request);
 240 }
 241
 242 /**
 243  * i40iw_dealloc_push_page - free a push page for qp
 244  * @iwdev: iwarp device
 245  * @qp: hardware control qp
 246  */
 247 static void i40iw_dealloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp)
 248 {
 249         struct i40iw_cqp_request *cqp_request;
 250         struct cqp_commands_info *cqp_info;
 251         enum i40iw_status_code status;
 252
 253         if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX)
 254                 return;
 255
 256         cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false);
 257         if (!cqp_request)
 258                 return;
 259
 260         cqp_info = &cqp_request->info;
 261         cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE;
 262         cqp_info->post_sq = 1;
 263
 264         cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
 265         cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
 266         cqp_info->in.u.manage_push_page.info.free_page = 1;
 267         cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;
 268         cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
 269
 270         status = i40iw_handle_cqp_op(iwdev, cqp_request);
 271         if (!status)
 272                 qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;
 273         else
 274                 i40iw_pr_err("CQP-OP Push page fail");
 275 }
 276
 277 /**
 278  * i40iw_alloc_pd - allocate protection domain
 279  * @pd: PD pointer
 280  * @udata: user data
 281  */
 282 static int i40iw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 283 {
 284         struct i40iw_pd *iwpd = to_iwpd(pd);
 285         struct i40iw_device *iwdev = to_iwdev(pd->device);
 286         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 287         struct i40iw_alloc_pd_resp uresp;
 288         struct i40iw_sc_pd *sc_pd;
 289         u32 pd_id = 0;
 290         int err;
 291
 292         if (iwdev->closing)
 293                 return -ENODEV;
 294
 295         err = i40iw_alloc_resource(iwdev, iwdev->allocated_pds,
 296                                    iwdev->max_pd, &pd_id, &iwdev->next_pd);
 297         if (err) {
 298                 i40iw_pr_err("alloc resource failed\n");
 299                 return err;
 300         }
 301
 302         sc_pd = &iwpd->sc_pd;
 303
 304         if (udata) {
 305                 struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(
 306                         udata, struct i40iw_ucontext, ibucontext);
 307                 dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
 308                 memset(&uresp, 0, sizeof(uresp));
 309                 uresp.pd_id = pd_id;
 310                 if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
 311                         err = -EFAULT;
 312                         goto error;
 313                 }
 314         } else {
 315                 dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id, -1);
 316         }
 317
 318         i40iw_add_pdusecount(iwpd);
 319         return 0;
 320
 321 error:
 322         i40iw_free_resource(iwdev, iwdev->allocated_pds, pd_id);
 323         return err;
 324 }
 325
 326 /**
 327  * i40iw_dealloc_pd - deallocate pd
 328  * @ibpd: ptr of pd to be deallocated
 329  * @udata: user data or null for kernel object
 330  */
 331 static int i40iw_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 332 {
 333         struct i40iw_pd *iwpd = to_iwpd(ibpd);
 334         struct i40iw_device *iwdev = to_iwdev(ibpd->device);
 335
 336         i40iw_rem_pdusecount(iwpd, iwdev);
 337         return 0;
 338 }
 339
 340 /**
 341  * i40iw_get_pbl - Retrieve pbl from a list given a virtual
 342  * address
 343  * @va: user virtual address
 344  * @pbl_list: pbl list to search in (QP's or CQ's)
 345  */
 346 static struct i40iw_pbl *i40iw_get_pbl(unsigned long va,
 347                                        struct list_head *pbl_list)
 348 {
 349         struct i40iw_pbl *iwpbl;
 350
 351         list_for_each_entry(iwpbl, pbl_list, list) {
 352                 if (iwpbl->user_base == va) {
 353                         iwpbl->on_list = false;
 354                         list_del(&iwpbl->list);
 355                         return iwpbl;
 356                 }
 357         }
 358         return NULL;
 359 }
 360
 361 /**
 362  * i40iw_free_qp_resources - free up memory resources for qp
 363  * @iwdev: iwarp device
 364  * @iwqp: qp ptr (user or kernel)
 365  * @qp_num: qp number assigned
 366  */
 367 void i40iw_free_qp_resources(struct i40iw_qp *iwqp)
 368 {
 369         struct i40iw_pbl *iwpbl = &iwqp->iwpbl;
 370         struct i40iw_device *iwdev = iwqp->iwdev;
 371         u32 qp_num = iwqp->ibqp.qp_num;
 372
 373         i40iw_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
 374         i40iw_dealloc_push_page(iwdev, &iwqp->sc_qp);
 375         if (qp_num)
 376                 i40iw_free_resource(iwdev, iwdev->allocated_qps, qp_num);
 377         if (iwpbl->pbl_allocated)
 378                 i40iw_free_pble(iwdev->pble_rsrc, &iwpbl->pble_alloc);
 379         i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->q2_ctx_mem);
 380         i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->kqp.dma_mem);
 381         kfree(iwqp->kqp.wrid_mem);
 382         iwqp->kqp.wrid_mem = NULL;
 383         kfree(iwqp);
 384 }
 385
 386 /**
 387  * i40iw_clean_cqes - clean cq entries for qp
 388  * @iwqp: qp ptr (user or kernel)
 389  * @iwcq: cq ptr
 390  */
 391 static void i40iw_clean_cqes(struct i40iw_qp *iwqp, struct i40iw_cq *iwcq)
 392 {
 393         struct i40iw_cq_uk *ukcq = &iwcq->sc_cq.cq_uk;
 394
 395         ukcq->ops.iw_cq_clean(&iwqp->sc_qp.qp_uk, ukcq);
 396 }
 397
 398 /**
 399  * i40iw_destroy_qp - destroy qp
 400  * @ibqp: qp's ib pointer also to get to device's qp address
 401  */
 402 static int i40iw_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 403 {
 404         struct i40iw_qp *iwqp = to_iwqp(ibqp);
 405         struct ib_qp_attr attr;
 406         struct i40iw_device *iwdev = iwqp->iwdev;
 407
 408         memset(&attr, 0, sizeof(attr));
 409
 410         iwqp->destroyed = 1;
 411
 412         if (iwqp->ibqp_state >= IB_QPS_INIT && iwqp->ibqp_state < IB_QPS_RTS)
 413                 i40iw_next_iw_state(iwqp, I40IW_QP_STATE_ERROR, 0, 0, 0);
 414
 415         if (!iwqp->user_mode) {
 416                 if (iwqp->iwscq) {
 417                         i40iw_clean_cqes(iwqp, iwqp->iwscq);
 418                         if (iwqp->iwrcq != iwqp->iwscq)
 419                                 i40iw_clean_cqes(iwqp, iwqp->iwrcq);
 420                 }
 421         }
 422
 423         attr.qp_state = IB_QPS_ERR;
 424         i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
 425         i40iw_qp_rem_ref(&iwqp->ibqp);
 426         wait_for_completion(&iwqp->free_qp);
 427         i40iw_cqp_qp_destroy_cmd(&iwdev->sc_dev, &iwqp->sc_qp);
 428         i40iw_rem_pdusecount(iwqp->iwpd, iwdev);
 429         i40iw_free_qp_resources(iwqp);
 430         i40iw_rem_devusecount(iwdev);
 431
 432         return 0;
 433 }
 434
 435 /**
 436  * i40iw_setup_virt_qp - setup for allocation of virtual qp
 437  * @dev: iwarp device
 438  * @qp: qp ptr
 439  * @init_info: initialize info to return
 440  */
 441 static int i40iw_setup_virt_qp(struct i40iw_device *iwdev,
 442                                struct i40iw_qp *iwqp,
 443                                struct i40iw_qp_init_info *init_info)
 444 {
 445         struct i40iw_pbl *iwpbl = &iwqp->iwpbl;
 446         struct i40iw_qp_mr *qpmr = &iwpbl->qp_mr;
 447
 448         iwqp->page = qpmr->sq_page;
 449         init_info->shadow_area_pa = cpu_to_le64(qpmr->shadow);
 450         if (iwpbl->pbl_allocated) {
 451                 init_info->virtual_map = true;
 452                 init_info->sq_pa = qpmr->sq_pbl.idx;
 453                 init_info->rq_pa = qpmr->rq_pbl.idx;
 454         } else {
 455                 init_info->sq_pa = qpmr->sq_pbl.addr;
 456                 init_info->rq_pa = qpmr->rq_pbl.addr;
 457         }
 458         return 0;
 459 }
 460
 461 /**
 462  * i40iw_setup_kmode_qp - setup initialization for kernel mode qp
 463  * @iwdev: iwarp device
 464  * @iwqp: qp ptr (user or kernel)
 465  * @info: initialize info to return
 466  */
 467 static int i40iw_setup_kmode_qp(struct i40iw_device *iwdev,
 468                                 struct i40iw_qp *iwqp,
 469                                 struct i40iw_qp_init_info *info)
 470 {
 471         struct i40iw_dma_mem *mem = &iwqp->kqp.dma_mem;
 472         u32 sqdepth, rqdepth;
 473         u8 sqshift;
 474         u32 size;
 475         enum i40iw_status_code status;
 476         struct i40iw_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
 477
 478         i40iw_get_wqe_shift(ukinfo->max_sq_frag_cnt, ukinfo->max_inline_data, &sqshift);
 479         status = i40iw_get_sqdepth(ukinfo->sq_size, sqshift, &sqdepth);
 480         if (status)
 481                 return -ENOMEM;
 482
 483         status = i40iw_get_rqdepth(ukinfo->rq_size, I40IW_MAX_RQ_WQE_SHIFT, &rqdepth);
 484         if (status)
 485                 return -ENOMEM;
 486
 487         size = sqdepth * sizeof(struct i40iw_sq_uk_wr_trk_info) + (rqdepth << 3);
 488         iwqp->kqp.wrid_mem = kzalloc(size, GFP_KERNEL);
 489
 490         ukinfo->sq_wrtrk_array = (struct i40iw_sq_uk_wr_trk_info *)iwqp->kqp.wrid_mem;
 491         if (!ukinfo->sq_wrtrk_array)
 492                 return -ENOMEM;
 493
 494         ukinfo->rq_wrid_array = (u64 *)&ukinfo->sq_wrtrk_array[sqdepth];
 495
 496         size = (sqdepth + rqdepth) * I40IW_QP_WQE_MIN_SIZE;
 497         size += (I40IW_SHADOW_AREA_SIZE << 3);
 498
 499         status = i40iw_allocate_dma_mem(iwdev->sc_dev.hw, mem, size, 256);
 500         if (status) {
 501                 kfree(ukinfo->sq_wrtrk_array);
 502                 ukinfo->sq_wrtrk_array = NULL;
 503                 return -ENOMEM;
 504         }
 505
 506         ukinfo->sq = mem->va;
 507         info->sq_pa = mem->pa;
 508
 509         ukinfo->rq = &ukinfo->sq[sqdepth];
 510         info->rq_pa = info->sq_pa + (sqdepth * I40IW_QP_WQE_MIN_SIZE);
 511
 512         ukinfo->shadow_area = ukinfo->rq[rqdepth].elem;
 513         info->shadow_area_pa = info->rq_pa + (rqdepth * I40IW_QP_WQE_MIN_SIZE);
 514
 515         ukinfo->sq_size = sqdepth >> sqshift;
 516         ukinfo->rq_size = rqdepth >> I40IW_MAX_RQ_WQE_SHIFT;
 517         ukinfo->qp_id = iwqp->ibqp.qp_num;
 518         return 0;
 519 }
 520
 521 /**
 522  * i40iw_create_qp - create qp
 523  * @ibpd: ptr of pd
 524  * @init_attr: attributes for qp
 525  * @udata: user data for create qp
 526  */
 527 static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd,
 528                                      struct ib_qp_init_attr *init_attr,
 529                                      struct ib_udata *udata)
 530 {
 531         struct i40iw_pd *iwpd = to_iwpd(ibpd);
 532         struct i40iw_device *iwdev = to_iwdev(ibpd->device);
 533         struct i40iw_cqp *iwcqp = &iwdev->cqp;
 534         struct i40iw_qp *iwqp;
 535         struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(
 536                 udata, struct i40iw_ucontext, ibucontext);
 537         struct i40iw_create_qp_req req;
 538         struct i40iw_create_qp_resp uresp;
 539         u32 qp_num = 0;
 540         enum i40iw_status_code ret;
 541         int err_code;
 542         int sq_size;
 543         int rq_size;
 544         struct i40iw_sc_qp *qp;
 545         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 546         struct i40iw_qp_init_info init_info;
 547         struct i40iw_create_qp_info *qp_info;
 548         struct i40iw_cqp_request *cqp_request;
 549         struct cqp_commands_info *cqp_info;
 550
 551         struct i40iw_qp_host_ctx_info *ctx_info;
 552         struct i40iwarp_offload_info *iwarp_info;
 553         unsigned long flags;
 554
 555         if (iwdev->closing)
 556                 return ERR_PTR(-ENODEV);
 557
 558         if (init_attr->create_flags)
 559                 return ERR_PTR(-EINVAL);
 560         if (init_attr->cap.max_inline_data > I40IW_MAX_INLINE_DATA_SIZE)
 561                 init_attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;
 562
 563         if (init_attr->cap.max_send_sge > I40IW_MAX_WQ_FRAGMENT_COUNT)
 564                 init_attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
 565
 566         if (init_attr->cap.max_recv_sge > I40IW_MAX_WQ_FRAGMENT_COUNT)
 567                 init_attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
 568
 569         memset(&init_info, 0, sizeof(init_info));
 570
 571         sq_size = init_attr->cap.max_send_wr;
 572         rq_size = init_attr->cap.max_recv_wr;
 573
 574         init_info.vsi = &iwdev->vsi;
 575         init_info.qp_uk_init_info.sq_size = sq_size;
 576         init_info.qp_uk_init_info.rq_size = rq_size;
 577         init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge;
 578         init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge;
 579         init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data;
 580
 581         iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL);
 582         if (!iwqp)
 583                 return ERR_PTR(-ENOMEM);
 584
 585         qp = &iwqp->sc_qp;
 586         qp->back_qp = (void *)iwqp;
 587         qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;
 588
 589         iwqp->iwdev = iwdev;
 590         iwqp->ctx_info.iwarp_info = &iwqp->iwarp_info;
 591
 592         if (i40iw_allocate_dma_mem(dev->hw,
 593                                    &iwqp->q2_ctx_mem,
 594                                    I40IW_Q2_BUFFER_SIZE + I40IW_QP_CTX_SIZE,
 595                                    256)) {
 596                 i40iw_pr_err("dma_mem failed\n");
 597                 err_code = -ENOMEM;
 598                 goto error;
 599         }
 600
 601         init_info.q2 = iwqp->q2_ctx_mem.va;
 602         init_info.q2_pa = iwqp->q2_ctx_mem.pa;
 603
 604         init_info.host_ctx = (void *)init_info.q2 + I40IW_Q2_BUFFER_SIZE;
 605         init_info.host_ctx_pa = init_info.q2_pa + I40IW_Q2_BUFFER_SIZE;
 606
 607         err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_qps, iwdev->max_qp,
 608                                         &qp_num, &iwdev->next_qp);
 609         if (err_code) {
 610                 i40iw_pr_err("qp resource\n");
 611                 goto error;
 612         }
 613
 614         iwqp->iwpd = iwpd;
 615         iwqp->ibqp.qp_num = qp_num;
 616         qp = &iwqp->sc_qp;
 617         iwqp->iwscq = to_iwcq(init_attr->send_cq);
 618         iwqp->iwrcq = to_iwcq(init_attr->recv_cq);
 619
 620         iwqp->host_ctx.va = init_info.host_ctx;
 621         iwqp->host_ctx.pa = init_info.host_ctx_pa;
 622         iwqp->host_ctx.size = I40IW_QP_CTX_SIZE;
 623
 624         init_info.pd = &iwpd->sc_pd;
 625         init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num;
 626         iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;
 627
 628         if (init_attr->qp_type != IB_QPT_RC) {
 629                 err_code = -EOPNOTSUPP;
 630                 goto error;
 631         }
 632         if (iwdev->push_mode)
 633                 i40iw_alloc_push_page(iwdev, qp);
 634         if (udata) {
 635                 err_code = ib_copy_from_udata(&req, udata, sizeof(req));
 636                 if (err_code) {
 637                         i40iw_pr_err("ib_copy_from_data\n");
 638                         goto error;
 639                 }
 640                 iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;
 641                 iwqp->user_mode = 1;
 642
 643                 if (req.user_wqe_buffers) {
 644                         struct i40iw_pbl *iwpbl;
 645
 646                         spin_lock_irqsave(
 647                             &ucontext->qp_reg_mem_list_lock, flags);
 648                         iwpbl = i40iw_get_pbl(
 649                             (unsigned long)req.user_wqe_buffers,
 650                             &ucontext->qp_reg_mem_list);
 651                         spin_unlock_irqrestore(
 652                             &ucontext->qp_reg_mem_list_lock, flags);
 653
 654                         if (!iwpbl) {
 655                                 err_code = -ENODATA;
 656                                 i40iw_pr_err("no pbl info\n");
 657                                 goto error;
 658                         }
 659                         memcpy(&iwqp->iwpbl, iwpbl, sizeof(iwqp->iwpbl));
 660                 }
 661                 err_code = i40iw_setup_virt_qp(iwdev, iwqp, &init_info);
 662         } else {
 663                 err_code = i40iw_setup_kmode_qp(iwdev, iwqp, &init_info);
 664         }
 665
 666         if (err_code) {
 667                 i40iw_pr_err("setup qp failed\n");
 668                 goto error;
 669         }
 670
 671         init_info.type = I40IW_QP_TYPE_IWARP;
 672         ret = dev->iw_priv_qp_ops->qp_init(qp, &init_info);
 673         if (ret) {
 674                 err_code = -EPROTO;
 675                 i40iw_pr_err("qp_init fail\n");
 676                 goto error;
 677         }
 678         ctx_info = &iwqp->ctx_info;
 679         iwarp_info = &iwqp->iwarp_info;
 680         iwarp_info->rd_enable = true;
 681         iwarp_info->wr_rdresp_en = true;
 682         if (!iwqp->user_mode) {
 683                 iwarp_info->fast_reg_en = true;
 684                 iwarp_info->priv_mode_en = true;
 685         }
 686         iwarp_info->ddp_ver = 1;
 687         iwarp_info->rdmap_ver = 1;
 688
 689         ctx_info->iwarp_info_valid = true;
 690         ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
 691         ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
 692         if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX) {
 693                 ctx_info->push_mode_en = false;
 694         } else {
 695                 ctx_info->push_mode_en = true;
 696                 ctx_info->push_idx = qp->push_idx;
 697         }
 698
 699         ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
 700                                              (u64 *)iwqp->host_ctx.va,
 701                                              ctx_info);
 702         ctx_info->iwarp_info_valid = false;
 703         cqp_request = i40iw_get_cqp_request(iwcqp, true);
 704         if (!cqp_request) {
 705                 err_code = -ENOMEM;
 706                 goto error;
 707         }
 708         cqp_info = &cqp_request->info;
 709         qp_info = &cqp_request->info.in.u.qp_create.info;
 710
 711         memset(qp_info, 0, sizeof(*qp_info));
 712
 713         qp_info->cq_num_valid = true;
 714         qp_info->next_iwarp_state = I40IW_QP_STATE_IDLE;
 715
 716         cqp_info->cqp_cmd = OP_QP_CREATE;
 717         cqp_info->post_sq = 1;
 718         cqp_info->in.u.qp_create.qp = qp;
 719         cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
 720         ret = i40iw_handle_cqp_op(iwdev, cqp_request);
 721         if (ret) {
 722                 i40iw_pr_err("CQP-OP QP create fail");
 723                 err_code = -EACCES;
 724                 goto error;
 725         }
 726
 727         refcount_set(&iwqp->refcount, 1);
 728         spin_lock_init(&iwqp->lock);
 729         iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
 730         iwdev->qp_table[qp_num] = iwqp;
 731         i40iw_add_pdusecount(iwqp->iwpd);
 732         i40iw_add_devusecount(iwdev);
 733         if (udata) {
 734                 memset(&uresp, 0, sizeof(uresp));
 735                 uresp.actual_sq_size = sq_size;
 736                 uresp.actual_rq_size = rq_size;
 737                 uresp.qp_id = qp_num;
 738                 uresp.push_idx = qp->push_idx;
 739                 err_code = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 740                 if (err_code) {
 741                         i40iw_pr_err("copy_to_udata failed\n");
 742                         i40iw_destroy_qp(&iwqp->ibqp, udata);
 743                         /* let the completion of the qp destroy free the qp */
 744                         return ERR_PTR(err_code);
 745                 }
 746         }
 747         init_completion(&iwqp->sq_drained);
 748         init_completion(&iwqp->rq_drained);
 749         init_completion(&iwqp->free_qp);
 750
 751         return &iwqp->ibqp;
 752 error:
 753         i40iw_free_qp_resources(iwqp);
 754         return ERR_PTR(err_code);
 755 }
 756
 757 /**
 758  * i40iw_query - query qp attributes
 759  * @ibqp: qp pointer
 760  * @attr: attributes pointer
 761  * @attr_mask: Not used
 762  * @init_attr: qp attributes to return
 763  */
 764 static int i40iw_query_qp(struct ib_qp *ibqp,
 765                           struct ib_qp_attr *attr,
 766                           int attr_mask,
 767                           struct ib_qp_init_attr *init_attr)
 768 {
 769         struct i40iw_qp *iwqp = to_iwqp(ibqp);
 770         struct i40iw_sc_qp *qp = &iwqp->sc_qp;
 771
 772         attr->qp_state = iwqp->ibqp_state;
 773         attr->cur_qp_state = attr->qp_state;
 774         attr->qp_access_flags = 0;
 775         attr->cap.max_send_wr = qp->qp_uk.sq_size;
 776         attr->cap.max_recv_wr = qp->qp_uk.rq_size;
 777         attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;
 778         attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
 779         attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
 780         attr->port_num = 1;
 781         init_attr->event_handler = iwqp->ibqp.event_handler;
 782         init_attr->qp_context = iwqp->ibqp.qp_context;
 783         init_attr->send_cq = iwqp->ibqp.send_cq;
 784         init_attr->recv_cq = iwqp->ibqp.recv_cq;
 785         init_attr->srq = iwqp->ibqp.srq;
 786         init_attr->cap = attr->cap;
 787         init_attr->port_num = 1;
 788         return 0;
 789 }
 790
 791 /**
 792  * i40iw_hw_modify_qp - setup cqp for modify qp
 793  * @iwdev: iwarp device
 794  * @iwqp: qp ptr (user or kernel)
 795  * @info: info for modify qp
 796  * @wait: flag to wait or not for modify qp completion
 797  */
 798 void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp,
 799                         struct i40iw_modify_qp_info *info, bool wait)
 800 {
 801         struct i40iw_cqp_request *cqp_request;
 802         struct cqp_commands_info *cqp_info;
 803         struct i40iw_modify_qp_info *m_info;
 804         struct i40iw_gen_ae_info ae_info;
 805
 806         cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait);
 807         if (!cqp_request)
 808                 return;
 809
 810         cqp_info = &cqp_request->info;
 811         m_info = &cqp_info->in.u.qp_modify.info;
 812         memcpy(m_info, info, sizeof(*m_info));
 813         cqp_info->cqp_cmd = OP_QP_MODIFY;
 814         cqp_info->post_sq = 1;
 815         cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
 816         cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
 817         if (!i40iw_handle_cqp_op(iwdev, cqp_request))
 818                 return;
 819
 820         switch (m_info->next_iwarp_state) {
 821         case I40IW_QP_STATE_RTS:
 822                 if (iwqp->iwarp_state == I40IW_QP_STATE_IDLE)
 823                         i40iw_send_reset(iwqp->cm_node);
 824                 fallthrough;
 825         case I40IW_QP_STATE_IDLE:
 826         case I40IW_QP_STATE_TERMINATE:
 827         case I40IW_QP_STATE_CLOSING:
 828                 ae_info.ae_code = I40IW_AE_BAD_CLOSE;
 829                 ae_info.ae_source = 0;
 830                 i40iw_gen_ae(iwdev, &iwqp->sc_qp, &ae_info, false);
 831                 break;
 832         case I40IW_QP_STATE_ERROR:
 833         default:
 834                 break;
 835         }
 836 }
 837
 838 /**
 839  * i40iw_modify_qp - modify qp request
 840  * @ibqp: qp's pointer for modify
 841  * @attr: access attributes
 842  * @attr_mask: state mask
 843  * @udata: user data
 844  */
 845 int i40iw_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 846                     int attr_mask, struct ib_udata *udata)
 847 {
 848         struct i40iw_qp *iwqp = to_iwqp(ibqp);
 849         struct i40iw_device *iwdev = iwqp->iwdev;
 850         struct i40iw_qp_host_ctx_info *ctx_info;
 851         struct i40iwarp_offload_info *iwarp_info;
 852         struct i40iw_modify_qp_info info;
 853         u8 issue_modify_qp = 0;
 854         u8 dont_wait = 0;
 855         u32 err;
 856         unsigned long flags;
 857
 858         memset(&info, 0, sizeof(info));
 859         ctx_info = &iwqp->ctx_info;
 860         iwarp_info = &iwqp->iwarp_info;
 861
 862         spin_lock_irqsave(&iwqp->lock, flags);
 863
 864         if (attr_mask & IB_QP_STATE) {
 865                 if (iwdev->closing && attr->qp_state != IB_QPS_ERR) {
 866                         err = -EINVAL;
 867                         goto exit;
 868                 }
 869
 870                 switch (attr->qp_state) {
 871                 case IB_QPS_INIT:
 872                 case IB_QPS_RTR:
 873                         if (iwqp->iwarp_state > (u32)I40IW_QP_STATE_IDLE) {
 874                                 err = -EINVAL;
 875                                 goto exit;
 876                         }
 877                         if (iwqp->iwarp_state == I40IW_QP_STATE_INVALID) {
 878                                 info.next_iwarp_state = I40IW_QP_STATE_IDLE;
 879                                 issue_modify_qp = 1;
 880                         }
 881                         break;
 882                 case IB_QPS_RTS:
 883                         if ((iwqp->iwarp_state > (u32)I40IW_QP_STATE_RTS) ||
 884                             (!iwqp->cm_id)) {
 885                                 err = -EINVAL;
 886                                 goto exit;
 887                         }
 888
 889                         issue_modify_qp = 1;
 890                         iwqp->hw_tcp_state = I40IW_TCP_STATE_ESTABLISHED;
 891                         iwqp->hte_added = 1;
 892                         info.next_iwarp_state = I40IW_QP_STATE_RTS;
 893                         info.tcp_ctx_valid = true;
 894                         info.ord_valid = true;
 895                         info.arp_cache_idx_valid = true;
 896                         info.cq_num_valid = true;
 897                         break;
 898                 case IB_QPS_SQD:
 899                         if (iwqp->hw_iwarp_state > (u32)I40IW_QP_STATE_RTS) {
 900                                 err = 0;
 901                                 goto exit;
 902                         }
 903                         if ((iwqp->iwarp_state == (u32)I40IW_QP_STATE_CLOSING) ||
 904                             (iwqp->iwarp_state < (u32)I40IW_QP_STATE_RTS)) {
 905                                 err = 0;
 906                                 goto exit;
 907                         }
 908                         if (iwqp->iwarp_state > (u32)I40IW_QP_STATE_CLOSING) {
 909                                 err = -EINVAL;
 910                                 goto exit;
 911                         }
 912                         info.next_iwarp_state = I40IW_QP_STATE_CLOSING;
 913                         issue_modify_qp = 1;
 914                         break;
 915                 case IB_QPS_SQE:
 916                         if (iwqp->iwarp_state >= (u32)I40IW_QP_STATE_TERMINATE) {
 917                                 err = -EINVAL;
 918                                 goto exit;
 919                         }
 920                         info.next_iwarp_state = I40IW_QP_STATE_TERMINATE;
 921                         issue_modify_qp = 1;
 922                         break;
 923                 case IB_QPS_ERR:
 924                 case IB_QPS_RESET:
 925                         if (iwqp->iwarp_state == (u32)I40IW_QP_STATE_ERROR) {
 926                                 err = -EINVAL;
 927                                 goto exit;
 928                         }
 929                         if (iwqp->sc_qp.term_flags)
 930                                 i40iw_terminate_del_timer(&iwqp->sc_qp);
 931                         info.next_iwarp_state = I40IW_QP_STATE_ERROR;
 932                         if ((iwqp->hw_tcp_state > I40IW_TCP_STATE_CLOSED) &&
 933                             iwdev->iw_status &&
 934                             (iwqp->hw_tcp_state != I40IW_TCP_STATE_TIME_WAIT))
 935                                 info.reset_tcp_conn = true;
 936                         else
 937                                 dont_wait = 1;
 938                         issue_modify_qp = 1;
 939                         info.next_iwarp_state = I40IW_QP_STATE_ERROR;
 940                         break;
 941                 default:
 942                         err = -EINVAL;
 943                         goto exit;
 944                 }
 945
 946                 iwqp->ibqp_state = attr->qp_state;
 947
 948         }
 949         if (attr_mask & IB_QP_ACCESS_FLAGS) {
 950                 ctx_info->iwarp_info_valid = true;
 951                 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE)
 952                         iwarp_info->wr_rdresp_en = true;
 953                 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
 954                         iwarp_info->wr_rdresp_en = true;
 955                 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
 956                         iwarp_info->rd_enable = true;
 957                 if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
 958                         iwarp_info->bind_en = true;
 959
 960                 if (iwqp->user_mode) {
 961                         iwarp_info->rd_enable = true;
 962                         iwarp_info->wr_rdresp_en = true;
 963                         iwarp_info->priv_mode_en = false;
 964                 }
 965         }
 966
 967         if (ctx_info->iwarp_info_valid) {
 968                 struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 969                 int ret;
 970
 971                 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
 972                 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
 973                 ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
 974                                                      (u64 *)iwqp->host_ctx.va,
 975                                                      ctx_info);
 976                 if (ret) {
 977                         i40iw_pr_err("setting QP context\n");
 978                         err = -EINVAL;
 979                         goto exit;
 980                 }
 981         }
 982
 983         spin_unlock_irqrestore(&iwqp->lock, flags);
 984
 985         if (issue_modify_qp) {
 986                 i40iw_hw_modify_qp(iwdev, iwqp, &info, true);
 987
 988                 spin_lock_irqsave(&iwqp->lock, flags);
 989                 iwqp->iwarp_state = info.next_iwarp_state;
 990                 spin_unlock_irqrestore(&iwqp->lock, flags);
 991         }
 992
 993         if (issue_modify_qp && (iwqp->ibqp_state > IB_QPS_RTS)) {
 994                 if (dont_wait) {
 995                         if (iwqp->cm_id && iwqp->hw_tcp_state) {
 996                                 spin_lock_irqsave(&iwqp->lock, flags);
 997                                 iwqp->hw_tcp_state = I40IW_TCP_STATE_CLOSED;
 998                                 iwqp->last_aeq = I40IW_AE_RESET_SENT;
 999                                 spin_unlock_irqrestore(&iwqp->lock, flags);
1000                                 i40iw_cm_disconn(iwqp);
1001                         }
1002                 } else {
1003                         spin_lock_irqsave(&iwqp->lock, flags);
1004                         if (iwqp->cm_id) {
1005                                 if (atomic_inc_return(&iwqp->close_timer_started) == 1) {
1006                                         iwqp->cm_id->add_ref(iwqp->cm_id);
1007                                         i40iw_schedule_cm_timer(iwqp->cm_node,
1008                                                                 (struct i40iw_puda_buf *)iwqp,
1009                                                                  I40IW_TIMER_TYPE_CLOSE, 1, 0);
1010                                 }
1011                         }
1012                         spin_unlock_irqrestore(&iwqp->lock, flags);
1013                 }
1014         }
1015         return 0;
1016 exit:
1017         spin_unlock_irqrestore(&iwqp->lock, flags);
1018         return err;
1019 }
1020
1021 /**
1022  * cq_free_resources - free up recources for cq
1023  * @iwdev: iwarp device
1024  * @iwcq: cq ptr
1025  */
1026 static void cq_free_resources(struct i40iw_device *iwdev, struct i40iw_cq *iwcq)
1027 {
1028         struct i40iw_sc_cq *cq = &iwcq->sc_cq;
1029
1030         if (!iwcq->user_mode)
1031                 i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwcq->kmem);
1032         i40iw_free_resource(iwdev, iwdev->allocated_cqs, cq->cq_uk.cq_id);
1033 }
1034
1035 /**
1036  * i40iw_cq_wq_destroy - send cq destroy cqp
1037  * @iwdev: iwarp device
1038  * @cq: hardware control cq
1039  */
1040 void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq)
1041 {
1042         enum i40iw_status_code status;
1043         struct i40iw_cqp_request *cqp_request;
1044         struct cqp_commands_info *cqp_info;
1045
1046         cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
1047         if (!cqp_request)
1048                 return;
1049
1050         cqp_info = &cqp_request->info;
1051
1052         cqp_info->cqp_cmd = OP_CQ_DESTROY;
1053         cqp_info->post_sq = 1;
1054         cqp_info->in.u.cq_destroy.cq = cq;
1055         cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1056         status = i40iw_handle_cqp_op(iwdev, cqp_request);
1057         if (status)
1058                 i40iw_pr_err("CQP-OP Destroy QP fail");
1059 }
1060
1061 /**
1062  * i40iw_destroy_cq - destroy cq
1063  * @ib_cq: cq pointer
1064  * @udata: user data or NULL for kernel object
1065  */
1066 static int i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
1067 {
1068         struct i40iw_cq *iwcq;
1069         struct i40iw_device *iwdev;
1070         struct i40iw_sc_cq *cq;
1071
1072         iwcq = to_iwcq(ib_cq);
1073         iwdev = to_iwdev(ib_cq->device);
1074         cq = &iwcq->sc_cq;
1075         i40iw_cq_wq_destroy(iwdev, cq);
1076         cq_free_resources(iwdev, iwcq);
1077         i40iw_rem_devusecount(iwdev);
1078         return 0;
1079 }
1080
1081 /**
1082  * i40iw_create_cq - create cq
1083  * @ibcq: CQ allocated
1084  * @attr: attributes for cq
1085  * @udata: user data
1086  */
1087 static int i40iw_create_cq(struct ib_cq *ibcq,
1088                            const struct ib_cq_init_attr *attr,
1089                            struct ib_udata *udata)
1090 {
1091         struct ib_device *ibdev = ibcq->device;
1092         struct i40iw_device *iwdev = to_iwdev(ibdev);
1093         struct i40iw_cq *iwcq = to_iwcq(ibcq);
1094         struct i40iw_pbl *iwpbl;
1095         u32 cq_num = 0;
1096         struct i40iw_sc_cq *cq;
1097         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
1098         struct i40iw_cq_init_info info = {};
1099         enum i40iw_status_code status;
1100         struct i40iw_cqp_request *cqp_request;
1101         struct cqp_commands_info *cqp_info;
1102         struct i40iw_cq_uk_init_info *ukinfo = &info.cq_uk_init_info;
1103         unsigned long flags;
1104         int err_code;
1105         int entries = attr->cqe;
1106
1107         if (iwdev->closing)
1108                 return -ENODEV;
1109
1110         if (entries > iwdev->max_cqe)
1111                 return -EINVAL;
1112
1113         err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_cqs,
1114                                         iwdev->max_cq, &cq_num,
1115                                         &iwdev->next_cq);
1116         if (err_code)
1117                 return err_code;
1118
1119         cq = &iwcq->sc_cq;
1120         cq->back_cq = (void *)iwcq;
1121         spin_lock_init(&iwcq->lock);
1122
1123         info.dev = dev;
1124         ukinfo->cq_size = max(entries, 4);
1125         ukinfo->cq_id = cq_num;
1126         iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
1127         info.ceqe_mask = 0;
1128         if (attr->comp_vector < iwdev->ceqs_count)
1129                 info.ceq_id = attr->comp_vector;
1130         info.ceq_id_valid = true;
1131         info.ceqe_mask = 1;
1132         info.type = I40IW_CQ_TYPE_IWARP;
1133         if (udata) {
1134                 struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(
1135                         udata, struct i40iw_ucontext, ibucontext);
1136                 struct i40iw_create_cq_req req;
1137                 struct i40iw_cq_mr *cqmr;
1138
1139                 memset(&req, 0, sizeof(req));
1140                 iwcq->user_mode = true;
1141                 if (ib_copy_from_udata(&req, udata, sizeof(struct i40iw_create_cq_req))) {
1142                         err_code = -EFAULT;
1143                         goto cq_free_resources;
1144                 }
1145
1146                 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
1147                 iwpbl = i40iw_get_pbl((unsigned long)req.user_cq_buffer,
1148                                       &ucontext->cq_reg_mem_list);
1149                 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
1150                 if (!iwpbl) {
1151                         err_code = -EPROTO;
1152                         goto cq_free_resources;
1153                 }
1154
1155                 iwcq->iwpbl = iwpbl;
1156                 iwcq->cq_mem_size = 0;
1157                 cqmr = &iwpbl->cq_mr;
1158                 info.shadow_area_pa = cpu_to_le64(cqmr->shadow);
1159                 if (iwpbl->pbl_allocated) {
1160                         info.virtual_map = true;
1161                         info.pbl_chunk_size = 1;
1162                         info.first_pm_pbl_idx = cqmr->cq_pbl.idx;
1163                 } else {
1164                         info.cq_base_pa = cqmr->cq_pbl.addr;
1165                 }
1166         } else {
1167                 /* Kmode allocations */
1168                 int rsize;
1169                 int shadow;
1170
1171                 rsize = info.cq_uk_init_info.cq_size * sizeof(struct i40iw_cqe);
1172                 rsize = round_up(rsize, 256);
1173                 shadow = I40IW_SHADOW_AREA_SIZE << 3;
1174                 status = i40iw_allocate_dma_mem(dev->hw, &iwcq->kmem,
1175                                                 rsize + shadow, 256);
1176                 if (status) {
1177                         err_code = -ENOMEM;
1178                         goto cq_free_resources;
1179                 }
1180                 ukinfo->cq_base = iwcq->kmem.va;
1181                 info.cq_base_pa = iwcq->kmem.pa;
1182                 info.shadow_area_pa = info.cq_base_pa + rsize;
1183                 ukinfo->shadow_area = iwcq->kmem.va + rsize;
1184         }
1185
1186         if (dev->iw_priv_cq_ops->cq_init(cq, &info)) {
1187                 i40iw_pr_err("init cq fail\n");
1188                 err_code = -EPROTO;
1189                 goto cq_free_resources;
1190         }
1191
1192         cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
1193         if (!cqp_request) {
1194                 err_code = -ENOMEM;
1195                 goto cq_free_resources;
1196         }
1197
1198         cqp_info = &cqp_request->info;
1199         cqp_info->cqp_cmd = OP_CQ_CREATE;
1200         cqp_info->post_sq = 1;
1201         cqp_info->in.u.cq_create.cq = cq;
1202         cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1203         status = i40iw_handle_cqp_op(iwdev, cqp_request);
1204         if (status) {
1205                 i40iw_pr_err("CQP-OP Create QP fail");
1206                 err_code = -EPROTO;
1207                 goto cq_free_resources;
1208         }
1209
1210         if (udata) {
1211                 struct i40iw_create_cq_resp resp;
1212
1213                 memset(&resp, 0, sizeof(resp));
1214                 resp.cq_id = info.cq_uk_init_info.cq_id;
1215                 resp.cq_size = info.cq_uk_init_info.cq_size;
1216                 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
1217                         i40iw_pr_err("copy to user data\n");
1218                         err_code = -EPROTO;
1219                         goto cq_destroy;
1220                 }
1221         }
1222
1223         i40iw_add_devusecount(iwdev);
1224         return 0;
1225
1226 cq_destroy:
1227         i40iw_cq_wq_destroy(iwdev, cq);
1228 cq_free_resources:
1229         cq_free_resources(iwdev, iwcq);
1230         return err_code;
1231 }
1232
1233 /**
1234  * i40iw_get_user_access - get hw access from IB access
1235  * @acc: IB access to return hw access
1236  */
1237 static inline u16 i40iw_get_user_access(int acc)
1238 {
1239         u16 access = 0;
1240
1241         access |= (acc & IB_ACCESS_LOCAL_WRITE) ? I40IW_ACCESS_FLAGS_LOCALWRITE : 0;
1242         access |= (acc & IB_ACCESS_REMOTE_WRITE) ? I40IW_ACCESS_FLAGS_REMOTEWRITE : 0;
1243         access |= (acc & IB_ACCESS_REMOTE_READ) ? I40IW_ACCESS_FLAGS_REMOTEREAD : 0;
1244         access |= (acc & IB_ACCESS_MW_BIND) ? I40IW_ACCESS_FLAGS_BIND_WINDOW : 0;
1245         return access;
1246 }
1247
1248 /**
1249  * i40iw_free_stag - free stag resource
1250  * @iwdev: iwarp device
1251  * @stag: stag to free
1252  */
1253 static void i40iw_free_stag(struct i40iw_device *iwdev, u32 stag)
1254 {
1255         u32 stag_idx;
1256
1257         stag_idx = (stag & iwdev->mr_stagmask) >> I40IW_CQPSQ_STAG_IDX_SHIFT;
1258         i40iw_free_resource(iwdev, iwdev->allocated_mrs, stag_idx);
1259         i40iw_rem_devusecount(iwdev);
1260 }
1261
1262 /**
1263  * i40iw_create_stag - create random stag
1264  * @iwdev: iwarp device
1265  */
1266 static u32 i40iw_create_stag(struct i40iw_device *iwdev)
1267 {
1268         u32 stag = 0;
1269         u32 stag_index = 0;
1270         u32 next_stag_index;
1271         u32 driver_key;
1272         u32 random;
1273         u8 consumer_key;
1274         int ret;
1275
1276         get_random_bytes(&random, sizeof(random));
1277         consumer_key = (u8)random;
1278
1279         driver_key = random & ~iwdev->mr_stagmask;
1280         next_stag_index = (random & iwdev->mr_stagmask) >> 8;
1281         next_stag_index %= iwdev->max_mr;
1282
1283         ret = i40iw_alloc_resource(iwdev,
1284                                    iwdev->allocated_mrs, iwdev->max_mr,
1285                                    &stag_index, &next_stag_index);
1286         if (!ret) {
1287                 stag = stag_index << I40IW_CQPSQ_STAG_IDX_SHIFT;
1288                 stag |= driver_key;
1289                 stag += (u32)consumer_key;
1290                 i40iw_add_devusecount(iwdev);
1291         }
1292         return stag;
1293 }
1294
1295 /**
1296  * i40iw_next_pbl_addr - Get next pbl address
1297  * @pbl: pointer to a pble
1298  * @pinfo: info pointer
1299  * @idx: index
1300  */
1301 static inline u64 *i40iw_next_pbl_addr(u64 *pbl,
1302                                        struct i40iw_pble_info **pinfo,
1303                                        u32 *idx)
1304 {
1305         *idx += 1;
1306         if ((!(*pinfo)) || (*idx != (*pinfo)->cnt))
1307                 return ++pbl;
1308         *idx = 0;
1309         (*pinfo)++;
1310         return (u64 *)(*pinfo)->addr;
1311 }
1312
1313 /**
1314  * i40iw_copy_user_pgaddrs - copy user page address to pble's os locally
1315  * @iwmr: iwmr for IB's user page addresses
1316  * @pbl: ple pointer to save 1 level or 0 level pble
1317  * @level: indicated level 0, 1 or 2
1318  */
1319 static void i40iw_copy_user_pgaddrs(struct i40iw_mr *iwmr,
1320                                     u64 *pbl,
1321                                     enum i40iw_pble_level level)
1322 {
1323         struct ib_umem *region = iwmr->region;
1324         struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
1325         struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
1326         struct i40iw_pble_info *pinfo;
1327         struct ib_block_iter biter;
1328         u32 idx = 0;
1329
1330         pinfo = (level == I40IW_LEVEL_1) ? NULL : palloc->level2.leaf;
1331
1332         if (iwmr->type == IW_MEMREG_TYPE_QP)
1333                 iwpbl->qp_mr.sq_page = sg_page(region->sg_head.sgl);
1334
1335         rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) {
1336                 *pbl = rdma_block_iter_dma_address(&biter);
1337                 pbl = i40iw_next_pbl_addr(pbl, &pinfo, &idx);
1338         }
1339 }
1340
1341 /**
1342  * i40iw_check_mem_contiguous - check if pbls stored in arr are contiguous
1343  * @arr: lvl1 pbl array
1344  * @npages: page count
1345  * pg_size: page size
1346  *
1347  */
1348 static bool i40iw_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size)
1349 {
1350         u32 pg_idx;
1351
1352         for (pg_idx = 0; pg_idx < npages; pg_idx++) {
1353                 if ((*arr + (pg_size * pg_idx)) != arr[pg_idx])
1354                         return false;
1355         }
1356         return true;
1357 }
1358
1359 /**
1360  * i40iw_check_mr_contiguous - check if MR is physically contiguous
1361  * @palloc: pbl allocation struct
1362  * pg_size: page size
1363  */
1364 static bool i40iw_check_mr_contiguous(struct i40iw_pble_alloc *palloc, u32 pg_size)
1365 {
1366         struct i40iw_pble_level2 *lvl2 = &palloc->level2;
1367         struct i40iw_pble_info *leaf = lvl2->leaf;
1368         u64 *arr = NULL;
1369         u64 *start_addr = NULL;
1370         int i;
1371         bool ret;
1372
1373         if (palloc->level == I40IW_LEVEL_1) {
1374                 arr = (u64 *)palloc->level1.addr;
1375                 ret = i40iw_check_mem_contiguous(arr, palloc->total_cnt, pg_size);
1376                 return ret;
1377         }
1378
1379         start_addr = (u64 *)leaf->addr;
1380
1381         for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
1382                 arr = (u64 *)leaf->addr;
1383                 if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr)
1384                         return false;
1385                 ret = i40iw_check_mem_contiguous(arr, leaf->cnt, pg_size);
1386                 if (!ret)
1387                         return false;
1388         }
1389
1390         return true;
1391 }
1392
1393 /**
1394  * i40iw_setup_pbles - copy user pg address to pble's
1395  * @iwdev: iwarp device
1396  * @iwmr: mr pointer for this memory registration
1397  * @use_pbles: flag if to use pble's
1398  */
1399 static int i40iw_setup_pbles(struct i40iw_device *iwdev,
1400                              struct i40iw_mr *iwmr,
1401                              bool use_pbles)
1402 {
1403         struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
1404         struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
1405         struct i40iw_pble_info *pinfo;
1406         u64 *pbl;
1407         enum i40iw_status_code status;
1408         enum i40iw_pble_level level = I40IW_LEVEL_1;
1409
1410         if (use_pbles) {
1411                 mutex_lock(&iwdev->pbl_mutex);
1412                 status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);
1413                 mutex_unlock(&iwdev->pbl_mutex);
1414                 if (status)
1415                         return -ENOMEM;
1416
1417                 iwpbl->pbl_allocated = true;
1418                 level = palloc->level;
1419                 pinfo = (level == I40IW_LEVEL_1) ? &palloc->level1 : palloc->level2.leaf;
1420                 pbl = (u64 *)pinfo->addr;
1421         } else {
1422                 pbl = iwmr->pgaddrmem;
1423         }
1424
1425         i40iw_copy_user_pgaddrs(iwmr, pbl, level);
1426
1427         if (use_pbles)
1428                 iwmr->pgaddrmem[0] = *pbl;
1429
1430         return 0;
1431 }
1432
1433 /**
1434  * i40iw_handle_q_mem - handle memory for qp and cq
1435  * @iwdev: iwarp device
1436  * @req: information for q memory management
1437  * @iwpbl: pble struct
1438  * @use_pbles: flag to use pble
1439  */
1440 static int i40iw_handle_q_mem(struct i40iw_device *iwdev,
1441                               struct i40iw_mem_reg_req *req,
1442                               struct i40iw_pbl *iwpbl,
1443                               bool use_pbles)
1444 {
1445         struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
1446         struct i40iw_mr *iwmr = iwpbl->iwmr;
1447         struct i40iw_qp_mr *qpmr = &iwpbl->qp_mr;
1448         struct i40iw_cq_mr *cqmr = &iwpbl->cq_mr;
1449         struct i40iw_hmc_pble *hmc_p;
1450         u64 *arr = iwmr->pgaddrmem;
1451         u32 pg_size;
1452         int err;
1453         int total;
1454         bool ret = true;
1455
1456         total = req->sq_pages + req->rq_pages + req->cq_pages;
1457         pg_size = iwmr->page_size;
1458
1459         err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);
1460         if (err)
1461                 return err;
1462
1463         if (use_pbles && (palloc->level != I40IW_LEVEL_1)) {
1464                 i40iw_free_pble(iwdev->pble_rsrc, palloc);
1465                 iwpbl->pbl_allocated = false;
1466                 return -ENOMEM;
1467         }
1468
1469         if (use_pbles)
1470                 arr = (u64 *)palloc->level1.addr;
1471
1472         if (iwmr->type == IW_MEMREG_TYPE_QP) {
1473                 hmc_p = &qpmr->sq_pbl;
1474                 qpmr->shadow = (dma_addr_t)arr[total];
1475
1476                 if (use_pbles) {
1477                         ret = i40iw_check_mem_contiguous(arr, req->sq_pages, pg_size);
1478                         if (ret)
1479                                 ret = i40iw_check_mem_contiguous(&arr[req->sq_pages], req->rq_pages, pg_size);
1480                 }
1481
1482                 if (!ret) {
1483                         hmc_p->idx = palloc->level1.idx;
1484                         hmc_p = &qpmr->rq_pbl;
1485                         hmc_p->idx = palloc->level1.idx + req->sq_pages;
1486                 } else {
1487                         hmc_p->addr = arr[0];
1488                         hmc_p = &qpmr->rq_pbl;
1489                         hmc_p->addr = arr[req->sq_pages];
1490                 }
1491         } else {                /* CQ */
1492                 hmc_p = &cqmr->cq_pbl;
1493                 cqmr->shadow = (dma_addr_t)arr[total];
1494
1495                 if (use_pbles)
1496                         ret = i40iw_check_mem_contiguous(arr, req->cq_pages, pg_size);
1497
1498                 if (!ret)
1499                         hmc_p->idx = palloc->level1.idx;
1500                 else
1501                         hmc_p->addr = arr[0];
1502         }
1503
1504         if (use_pbles && ret) {
1505                 i40iw_free_pble(iwdev->pble_rsrc, palloc);
1506                 iwpbl->pbl_allocated = false;
1507         }
1508
1509         return err;
1510 }
1511
1512 /**
1513  * i40iw_hw_alloc_stag - cqp command to allocate stag
1514  * @iwdev: iwarp device
1515  * @iwmr: iwarp mr pointer
1516  */
1517 static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr)
1518 {
1519         struct i40iw_allocate_stag_info *info;
1520         struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
1521         enum i40iw_status_code status;
1522         int err = 0;
1523         struct i40iw_cqp_request *cqp_request;
1524         struct cqp_commands_info *cqp_info;
1525
1526         cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
1527         if (!cqp_request)
1528                 return -ENOMEM;
1529
1530         cqp_info = &cqp_request->info;
1531         info = &cqp_info->in.u.alloc_stag.info;
1532         memset(info, 0, sizeof(*info));
1533         info->page_size = PAGE_SIZE;
1534         info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT;
1535         info->pd_id = iwpd->sc_pd.pd_id;
1536         info->total_len = iwmr->length;
1537         info->remote_access = true;
1538         cqp_info->cqp_cmd = OP_ALLOC_STAG;
1539         cqp_info->post_sq = 1;
1540         cqp_info->in.u.alloc_stag.dev = &iwdev->sc_dev;
1541         cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request;
1542
1543         status = i40iw_handle_cqp_op(iwdev, cqp_request);
1544         if (status) {
1545                 err = -ENOMEM;
1546                 i40iw_pr_err("CQP-OP MR Reg fail");
1547         }
1548         return err;
1549 }
1550
1551 /**
1552  * i40iw_alloc_mr - register stag for fast memory registration
1553  * @pd: ibpd pointer
1554  * @mr_type: memory for stag registrion
1555  * @max_num_sg: man number of pages
1556  */
1557 static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1558                                     u32 max_num_sg)
1559 {
1560         struct i40iw_pd *iwpd = to_iwpd(pd);
1561         struct i40iw_device *iwdev = to_iwdev(pd->device);
1562         struct i40iw_pble_alloc *palloc;
1563         struct i40iw_pbl *iwpbl;
1564         struct i40iw_mr *iwmr;
1565         enum i40iw_status_code status;
1566         u32 stag;
1567         int err_code = -ENOMEM;
1568
1569         iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
1570         if (!iwmr)
1571                 return ERR_PTR(-ENOMEM);
1572
1573         stag = i40iw_create_stag(iwdev);
1574         if (!stag) {
1575                 err_code = -EOVERFLOW;
1576                 goto err;
1577         }
1578         stag &= ~I40IW_CQPSQ_STAG_KEY_MASK;
1579         iwmr->stag = stag;
1580         iwmr->ibmr.rkey = stag;
1581         iwmr->ibmr.lkey = stag;
1582         iwmr->ibmr.pd = pd;
1583         iwmr->ibmr.device = pd->device;
1584         iwpbl = &iwmr->iwpbl;
1585         iwpbl->iwmr = iwmr;
1586         iwmr->type = IW_MEMREG_TYPE_MEM;
1587         palloc = &iwpbl->pble_alloc;
1588         iwmr->page_cnt = max_num_sg;
1589         mutex_lock(&iwdev->pbl_mutex);
1590         status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);
1591         mutex_unlock(&iwdev->pbl_mutex);
1592         if (status)
1593                 goto err1;
1594
1595         if (palloc->level != I40IW_LEVEL_1)
1596                 goto err2;
1597         err_code = i40iw_hw_alloc_stag(iwdev, iwmr);
1598         if (err_code)
1599                 goto err2;
1600         iwpbl->pbl_allocated = true;
1601         i40iw_add_pdusecount(iwpd);
1602         return &iwmr->ibmr;
1603 err2:
1604         i40iw_free_pble(iwdev->pble_rsrc, palloc);
1605 err1:
1606         i40iw_free_stag(iwdev, stag);
1607 err:
1608         kfree(iwmr);
1609         return ERR_PTR(err_code);
1610 }
1611
1612 /**
1613  * i40iw_set_page - populate pbl list for fmr
1614  * @ibmr: ib mem to access iwarp mr pointer
1615  * @addr: page dma address fro pbl list
1616  */
1617 static int i40iw_set_page(struct ib_mr *ibmr, u64 addr)
1618 {
1619         struct i40iw_mr *iwmr = to_iwmr(ibmr);
1620         struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
1621         struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
1622         u64 *pbl;
1623
1624         if (unlikely(iwmr->npages == iwmr->page_cnt))
1625                 return -ENOMEM;
1626
1627         pbl = (u64 *)palloc->level1.addr;
1628         pbl[iwmr->npages++] = cpu_to_le64(addr);
1629         return 0;
1630 }
1631
1632 /**
1633  * i40iw_map_mr_sg - map of sg list for fmr
1634  * @ibmr: ib mem to access iwarp mr pointer
1635  * @sg: scatter gather list for fmr
1636  * @sg_nents: number of sg pages
1637  */
1638 static int i40iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
1639                            int sg_nents, unsigned int *sg_offset)
1640 {
1641         struct i40iw_mr *iwmr = to_iwmr(ibmr);
1642
1643         iwmr->npages = 0;
1644         return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, i40iw_set_page);
1645 }
1646
1647 /**
1648  * i40iw_drain_sq - drain the send queue
1649  * @ibqp: ib qp pointer
1650  */
1651 static void i40iw_drain_sq(struct ib_qp *ibqp)
1652 {
1653         struct i40iw_qp *iwqp = to_iwqp(ibqp);
1654         struct i40iw_sc_qp *qp = &iwqp->sc_qp;
1655
1656         if (I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring))
1657                 wait_for_completion(&iwqp->sq_drained);
1658 }
1659
1660 /**
1661  * i40iw_drain_rq - drain the receive queue
1662  * @ibqp: ib qp pointer
1663  */
1664 static void i40iw_drain_rq(struct ib_qp *ibqp)
1665 {
1666         struct i40iw_qp *iwqp = to_iwqp(ibqp);
1667         struct i40iw_sc_qp *qp = &iwqp->sc_qp;
1668
1669         if (I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring))
1670                 wait_for_completion(&iwqp->rq_drained);
1671 }
1672
1673 /**
1674  * i40iw_hwreg_mr - send cqp command for memory registration
1675  * @iwdev: iwarp device
1676  * @iwmr: iwarp mr pointer
1677  * @access: access for MR
1678  */
1679 static int i40iw_hwreg_mr(struct i40iw_device *iwdev,
1680                           struct i40iw_mr *iwmr,
1681                           u16 access)
1682 {
1683         struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
1684         struct i40iw_reg_ns_stag_info *stag_info;
1685         struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
1686         struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
1687         enum i40iw_status_code status;
1688         int err = 0;
1689         struct i40iw_cqp_request *cqp_request;
1690         struct cqp_commands_info *cqp_info;
1691
1692         cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
1693         if (!cqp_request)
1694                 return -ENOMEM;
1695
1696         cqp_info = &cqp_request->info;
1697         stag_info = &cqp_info->in.u.mr_reg_non_shared.info;
1698         memset(stag_info, 0, sizeof(*stag_info));
1699         stag_info->va = (void *)(unsigned long)iwpbl->user_base;
1700         stag_info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT;
1701         stag_info->stag_key = (u8)iwmr->stag;
1702         stag_info->total_len = iwmr->length;
1703         stag_info->access_rights = access;
1704         stag_info->pd_id = iwpd->sc_pd.pd_id;
1705         stag_info->addr_type = I40IW_ADDR_TYPE_VA_BASED;
1706         stag_info->page_size = iwmr->page_size;
1707
1708         if (iwpbl->pbl_allocated) {
1709                 if (palloc->level == I40IW_LEVEL_1) {
1710                         stag_info->first_pm_pbl_index = palloc->level1.idx;
1711                         stag_info->chunk_size = 1;
1712                 } else {
1713                         stag_info->first_pm_pbl_index = palloc->level2.root.idx;
1714                         stag_info->chunk_size = 3;
1715                 }
1716         } else {
1717                 stag_info->reg_addr_pa = iwmr->pgaddrmem[0];
1718         }
1719
1720         cqp_info->cqp_cmd = OP_MR_REG_NON_SHARED;
1721         cqp_info->post_sq = 1;
1722         cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->sc_dev;
1723         cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request;
1724
1725         status = i40iw_handle_cqp_op(iwdev, cqp_request);
1726         if (status) {
1727                 err = -ENOMEM;
1728                 i40iw_pr_err("CQP-OP MR Reg fail");
1729         }
1730         return err;
1731 }
1732
1733 /**
1734  * i40iw_reg_user_mr - Register a user memory region
1735  * @pd: ptr of pd
1736  * @start: virtual start address
1737  * @length: length of mr
1738  * @virt: virtual address
1739  * @acc: access of mr
1740  * @udata: user data
1741  */
1742 static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd,
1743                                        u64 start,
1744                                        u64 length,
1745                                        u64 virt,
1746                                        int acc,
1747                                        struct ib_udata *udata)
1748 {
1749         struct i40iw_pd *iwpd = to_iwpd(pd);
1750         struct i40iw_device *iwdev = to_iwdev(pd->device);
1751         struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(
1752                 udata, struct i40iw_ucontext, ibucontext);
1753         struct i40iw_pble_alloc *palloc;
1754         struct i40iw_pbl *iwpbl;
1755         struct i40iw_mr *iwmr;
1756         struct ib_umem *region;
1757         struct i40iw_mem_reg_req req;
1758         u32 stag = 0;
1759         u16 access;
1760         bool use_pbles = false;
1761         unsigned long flags;
1762         int err = -ENOSYS;
1763         int ret;
1764
1765         if (!udata)
1766                 return ERR_PTR(-EOPNOTSUPP);
1767
1768         if (iwdev->closing)
1769                 return ERR_PTR(-ENODEV);
1770
1771         if (length > I40IW_MAX_MR_SIZE)
1772                 return ERR_PTR(-EINVAL);
1773         region = ib_umem_get(pd->device, start, length, acc);
1774         if (IS_ERR(region))
1775                 return (struct ib_mr *)region;
1776
1777         if (ib_copy_from_udata(&req, udata, sizeof(req))) {
1778                 ib_umem_release(region);
1779                 return ERR_PTR(-EFAULT);
1780         }
1781
1782         iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
1783         if (!iwmr) {
1784                 ib_umem_release(region);
1785                 return ERR_PTR(-ENOMEM);
1786         }
1787
1788         iwpbl = &iwmr->iwpbl;
1789         iwpbl->iwmr = iwmr;
1790         iwmr->region = region;
1791         iwmr->ibmr.pd = pd;
1792         iwmr->ibmr.device = pd->device;
1793
1794         iwmr->page_size = PAGE_SIZE;
1795         if (req.reg_type == IW_MEMREG_TYPE_MEM)
1796                 iwmr->page_size = ib_umem_find_best_pgsz(region, SZ_4K | SZ_2M,
1797                                                          virt);
1798         iwmr->length = region->length;
1799
1800         iwpbl->user_base = virt;
1801         palloc = &iwpbl->pble_alloc;
1802
1803         iwmr->type = req.reg_type;
1804         iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size);
1805
1806         switch (req.reg_type) {
1807         case IW_MEMREG_TYPE_QP:
1808                 use_pbles = ((req.sq_pages + req.rq_pages) > 2);
1809                 err = i40iw_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
1810                 if (err)
1811                         goto error;
1812                 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
1813                 list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
1814                 iwpbl->on_list = true;
1815                 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
1816                 break;
1817         case IW_MEMREG_TYPE_CQ:
1818                 use_pbles = (req.cq_pages > 1);
1819                 err = i40iw_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
1820                 if (err)
1821                         goto error;
1822
1823                 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
1824                 list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
1825                 iwpbl->on_list = true;
1826                 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
1827                 break;
1828         case IW_MEMREG_TYPE_MEM:
1829                 use_pbles = (iwmr->page_cnt != 1);
1830                 access = I40IW_ACCESS_FLAGS_LOCALREAD;
1831
1832                 err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);
1833                 if (err)
1834                         goto error;
1835
1836                 if (use_pbles) {
1837                         ret = i40iw_check_mr_contiguous(palloc, iwmr->page_size);
1838                         if (ret) {
1839                                 i40iw_free_pble(iwdev->pble_rsrc, palloc);
1840                                 iwpbl->pbl_allocated = false;
1841                         }
1842                 }
1843
1844                 access |= i40iw_get_user_access(acc);
1845                 stag = i40iw_create_stag(iwdev);
1846                 if (!stag) {
1847                         err = -ENOMEM;
1848                         goto error;
1849                 }
1850
1851                 iwmr->stag = stag;
1852                 iwmr->ibmr.rkey = stag;
1853                 iwmr->ibmr.lkey = stag;
1854
1855                 err = i40iw_hwreg_mr(iwdev, iwmr, access);
1856                 if (err) {
1857                         i40iw_free_stag(iwdev, stag);
1858                         goto error;
1859                 }
1860
1861                 break;
1862         default:
1863                 goto error;
1864         }
1865
1866         iwmr->type = req.reg_type;
1867         if (req.reg_type == IW_MEMREG_TYPE_MEM)
1868                 i40iw_add_pdusecount(iwpd);
1869         return &iwmr->ibmr;
1870
1871 error:
1872         if (palloc->level != I40IW_LEVEL_0 && iwpbl->pbl_allocated)
1873                 i40iw_free_pble(iwdev->pble_rsrc, palloc);
1874         ib_umem_release(region);
1875         kfree(iwmr);
1876         return ERR_PTR(err);
1877 }
1878
1879 /**
1880  * i40iw_reg_phys_mr - register kernel physical memory
1881  * @pd: ibpd pointer
1882  * @addr: physical address of memory to register
1883  * @size: size of memory to register
1884  * @acc: Access rights
1885  * @iova_start: start of virtual address for physical buffers
1886  */
1887 struct ib_mr *i40iw_reg_phys_mr(struct ib_pd *pd,
1888                                 u64 addr,
1889                                 u64 size,
1890                                 int acc,
1891                                 u64 *iova_start)
1892 {
1893         struct i40iw_pd *iwpd = to_iwpd(pd);
1894         struct i40iw_device *iwdev = to_iwdev(pd->device);
1895         struct i40iw_pbl *iwpbl;
1896         struct i40iw_mr *iwmr;
1897         enum i40iw_status_code status;
1898         u32 stag;
1899         u16 access = I40IW_ACCESS_FLAGS_LOCALREAD;
1900         int ret;
1901
1902         iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
1903         if (!iwmr)
1904                 return ERR_PTR(-ENOMEM);
1905         iwmr->ibmr.pd = pd;
1906         iwmr->ibmr.device = pd->device;
1907         iwpbl = &iwmr->iwpbl;
1908         iwpbl->iwmr = iwmr;
1909         iwmr->type = IW_MEMREG_TYPE_MEM;
1910         iwpbl->user_base = *iova_start;
1911         stag = i40iw_create_stag(iwdev);
1912         if (!stag) {
1913                 ret = -EOVERFLOW;
1914                 goto err;
1915         }
1916         access |= i40iw_get_user_access(acc);
1917         iwmr->stag = stag;
1918         iwmr->ibmr.rkey = stag;
1919         iwmr->ibmr.lkey = stag;
1920         iwmr->page_cnt = 1;
1921         iwmr->pgaddrmem[0]  = addr;
1922         iwmr->length = size;
1923         status = i40iw_hwreg_mr(iwdev, iwmr, access);
1924         if (status) {
1925                 i40iw_free_stag(iwdev, stag);
1926                 ret = -ENOMEM;
1927                 goto err;
1928         }
1929
1930         i40iw_add_pdusecount(iwpd);
1931         return &iwmr->ibmr;
1932  err:
1933         kfree(iwmr);
1934         return ERR_PTR(ret);
1935 }
1936
1937 /**
1938  * i40iw_get_dma_mr - register physical mem
1939  * @pd: ptr of pd
1940  * @acc: access for memory
1941  */
1942 static struct ib_mr *i40iw_get_dma_mr(struct ib_pd *pd, int acc)
1943 {
1944         u64 kva = 0;
1945
1946         return i40iw_reg_phys_mr(pd, 0, 0, acc, &kva);
1947 }
1948
1949 /**
1950  * i40iw_del_mem_list - Deleting pbl list entries for CQ/QP
1951  * @iwmr: iwmr for IB's user page addresses
1952  * @ucontext: ptr to user context
1953  */
1954 static void i40iw_del_memlist(struct i40iw_mr *iwmr,
1955                               struct i40iw_ucontext *ucontext)
1956 {
1957         struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
1958         unsigned long flags;
1959
1960         switch (iwmr->type) {
1961         case IW_MEMREG_TYPE_CQ:
1962                 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
1963                 if (iwpbl->on_list) {
1964                         iwpbl->on_list = false;
1965                         list_del(&iwpbl->list);
1966                 }
1967                 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
1968                 break;
1969         case IW_MEMREG_TYPE_QP:
1970                 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
1971                 if (iwpbl->on_list) {
1972                         iwpbl->on_list = false;
1973                         list_del(&iwpbl->list);
1974                 }
1975                 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
1976                 break;
1977         default:
1978                 break;
1979         }
1980 }
1981
1982 /**
1983  * i40iw_dereg_mr - deregister mr
1984  * @ib_mr: mr ptr for dereg
1985  */
1986 static int i40iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
1987 {
1988         struct ib_pd *ibpd = ib_mr->pd;
1989         struct i40iw_pd *iwpd = to_iwpd(ibpd);
1990         struct i40iw_mr *iwmr = to_iwmr(ib_mr);
1991         struct i40iw_device *iwdev = to_iwdev(ib_mr->device);
1992         enum i40iw_status_code status;
1993         struct i40iw_dealloc_stag_info *info;
1994         struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
1995         struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
1996         struct i40iw_cqp_request *cqp_request;
1997         struct cqp_commands_info *cqp_info;
1998         u32 stag_idx;
1999
2000         ib_umem_release(iwmr->region);
2001
2002         if (iwmr->type != IW_MEMREG_TYPE_MEM) {
2003                 /* region is released. only test for userness. */
2004                 if (iwmr->region) {
2005                         struct i40iw_ucontext *ucontext =
2006                                 rdma_udata_to_drv_context(
2007                                         udata,
2008                                         struct i40iw_ucontext,
2009                                         ibucontext);
2010
2011                         i40iw_del_memlist(iwmr, ucontext);
2012                 }
2013                 if (iwpbl->pbl_allocated && iwmr->type != IW_MEMREG_TYPE_QP)
2014                         i40iw_free_pble(iwdev->pble_rsrc, palloc);
2015                 kfree(iwmr);
2016                 return 0;
2017         }
2018
2019         cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
2020         if (!cqp_request)
2021                 return -ENOMEM;
2022
2023         cqp_info = &cqp_request->info;
2024         info = &cqp_info->in.u.dealloc_stag.info;
2025         memset(info, 0, sizeof(*info));
2026
2027         info->pd_id = cpu_to_le32(iwpd->sc_pd.pd_id & 0x00007fff);
2028         info->stag_idx = RS_64_1(ib_mr->rkey, I40IW_CQPSQ_STAG_IDX_SHIFT);
2029         stag_idx = info->stag_idx;
2030         info->mr = true;
2031         if (iwpbl->pbl_allocated)
2032                 info->dealloc_pbl = true;
2033
2034         cqp_info->cqp_cmd = OP_DEALLOC_STAG;
2035         cqp_info->post_sq = 1;
2036         cqp_info->in.u.dealloc_stag.dev = &iwdev->sc_dev;
2037         cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
2038         status = i40iw_handle_cqp_op(iwdev, cqp_request);
2039         if (status)
2040                 i40iw_pr_err("CQP-OP dealloc failed for stag_idx = 0x%x\n", stag_idx);
2041         i40iw_rem_pdusecount(iwpd, iwdev);
2042         i40iw_free_stag(iwdev, iwmr->stag);
2043         if (iwpbl->pbl_allocated)
2044                 i40iw_free_pble(iwdev->pble_rsrc, palloc);
2045         kfree(iwmr);
2046         return 0;
2047 }
2048
2049 /**
2050  * hw_rev_show
2051  */
2052 static ssize_t hw_rev_show(struct device *dev,
2053                            struct device_attribute *attr, char *buf)
2054 {
2055         struct i40iw_ib_device *iwibdev =
2056                 rdma_device_to_drv_device(dev, struct i40iw_ib_device, ibdev);
2057         u32 hw_rev = iwibdev->iwdev->sc_dev.hw_rev;
2058
2059         return sprintf(buf, "%x\n", hw_rev);
2060 }
2061 static DEVICE_ATTR_RO(hw_rev);
2062
2063 /**
2064  * hca_type_show
2065  */
2066 static ssize_t hca_type_show(struct device *dev,
2067                              struct device_attribute *attr, char *buf)
2068 {
2069         return sprintf(buf, "I40IW\n");
2070 }
2071 static DEVICE_ATTR_RO(hca_type);
2072
2073 /**
2074  * board_id_show
2075  */
2076 static ssize_t board_id_show(struct device *dev,
2077                              struct device_attribute *attr, char *buf)
2078 {
2079         return sprintf(buf, "%.*s\n", 32, "I40IW Board ID");
2080 }
2081 static DEVICE_ATTR_RO(board_id);
2082
2083 static struct attribute *i40iw_dev_attributes[] = {
2084         &dev_attr_hw_rev.attr,
2085         &dev_attr_hca_type.attr,
2086         &dev_attr_board_id.attr,
2087         NULL
2088 };
2089
2090 static const struct attribute_group i40iw_attr_group = {
2091         .attrs = i40iw_dev_attributes,
2092 };
2093
2094 /**
2095  * i40iw_copy_sg_list - copy sg list for qp
2096  * @sg_list: copied into sg_list
2097  * @sgl: copy from sgl
2098  * @num_sges: count of sg entries
2099  */
2100 static void i40iw_copy_sg_list(struct i40iw_sge *sg_list, struct ib_sge *sgl, int num_sges)
2101 {
2102         unsigned int i;
2103
2104         for (i = 0; (i < num_sges) && (i < I40IW_MAX_WQ_FRAGMENT_COUNT); i++) {
2105                 sg_list[i].tag_off = sgl[i].addr;
2106                 sg_list[i].len = sgl[i].length;
2107                 sg_list[i].stag = sgl[i].lkey;
2108         }
2109 }
2110
2111 /**
2112  * i40iw_post_send -  kernel application wr
2113  * @ibqp: qp ptr for wr
2114  * @ib_wr: work request ptr
2115  * @bad_wr: return of bad wr if err
2116  */
2117 static int i40iw_post_send(struct ib_qp *ibqp,
2118                            const struct ib_send_wr *ib_wr,
2119                            const struct ib_send_wr **bad_wr)
2120 {
2121         struct i40iw_qp *iwqp;
2122         struct i40iw_qp_uk *ukqp;
2123         struct i40iw_post_sq_info info;
2124         enum i40iw_status_code ret;
2125         int err = 0;
2126         unsigned long flags;
2127         bool inv_stag;
2128
2129         iwqp = (struct i40iw_qp *)ibqp;
2130         ukqp = &iwqp->sc_qp.qp_uk;
2131
2132         spin_lock_irqsave(&iwqp->lock, flags);
2133
2134         if (iwqp->flush_issued) {
2135                 err = -EINVAL;
2136                 goto out;
2137         }
2138
2139         while (ib_wr) {
2140                 inv_stag = false;
2141                 memset(&info, 0, sizeof(info));
2142                 info.wr_id = (u64)(ib_wr->wr_id);
2143                 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all)
2144                         info.signaled = true;
2145                 if (ib_wr->send_flags & IB_SEND_FENCE)
2146                         info.read_fence = true;
2147
2148                 switch (ib_wr->opcode) {
2149                 case IB_WR_SEND:
2150                 case IB_WR_SEND_WITH_INV:
2151                         if (ib_wr->opcode == IB_WR_SEND) {
2152                                 if (ib_wr->send_flags & IB_SEND_SOLICITED)
2153                                         info.op_type = I40IW_OP_TYPE_SEND_SOL;
2154                                 else
2155                                         info.op_type = I40IW_OP_TYPE_SEND;
2156                         } else {
2157                                 if (ib_wr->send_flags & IB_SEND_SOLICITED)
2158                                         info.op_type = I40IW_OP_TYPE_SEND_SOL_INV;
2159                                 else
2160                                         info.op_type = I40IW_OP_TYPE_SEND_INV;
2161                         }
2162
2163                         if (ib_wr->send_flags & IB_SEND_INLINE) {
2164                                 info.op.inline_send.data = (void *)(unsigned long)ib_wr->sg_list[0].addr;
2165                                 info.op.inline_send.len = ib_wr->sg_list[0].length;
2166                                 ret = ukqp->ops.iw_inline_send(ukqp, &info, ib_wr->ex.invalidate_rkey, false);
2167                         } else {
2168                                 info.op.send.num_sges = ib_wr->num_sge;
2169                                 info.op.send.sg_list = (struct i40iw_sge *)ib_wr->sg_list;
2170                                 ret = ukqp->ops.iw_send(ukqp, &info, ib_wr->ex.invalidate_rkey, false);
2171                         }
2172
2173                         if (ret) {
2174                                 if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
2175                                         err = -ENOMEM;
2176                                 else
2177                                         err = -EINVAL;
2178                         }
2179                         break;
2180                 case IB_WR_RDMA_WRITE:
2181                         info.op_type = I40IW_OP_TYPE_RDMA_WRITE;
2182
2183                         if (ib_wr->send_flags & IB_SEND_INLINE) {
2184                                 info.op.inline_rdma_write.data = (void *)(unsigned long)ib_wr->sg_list[0].addr;
2185                                 info.op.inline_rdma_write.len = ib_wr->sg_list[0].length;
2186                                 info.op.inline_rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
2187                                 info.op.inline_rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
2188                                 ret = ukqp->ops.iw_inline_rdma_write(ukqp, &info, false);
2189                         } else {
2190                                 info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
2191                                 info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
2192                                 info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
2193                                 info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
2194                                 ret = ukqp->ops.iw_rdma_write(ukqp, &info, false);
2195                         }
2196
2197                         if (ret) {
2198                                 if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
2199                                         err = -ENOMEM;
2200                                 else
2201                                         err = -EINVAL;
2202                         }
2203                         break;
2204                 case IB_WR_RDMA_READ_WITH_INV:
2205                         inv_stag = true;
2206                         fallthrough;
2207                 case IB_WR_RDMA_READ:
2208                         if (ib_wr->num_sge > I40IW_MAX_SGE_RD) {
2209                                 err = -EINVAL;
2210                                 break;
2211                         }
2212                         info.op_type = I40IW_OP_TYPE_RDMA_READ;
2213                         info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
2214                         info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey;
2215                         info.op.rdma_read.lo_addr.tag_off = ib_wr->sg_list->addr;
2216                         info.op.rdma_read.lo_addr.stag = ib_wr->sg_list->lkey;
2217                         info.op.rdma_read.lo_addr.len = ib_wr->sg_list->length;
2218                         ret = ukqp->ops.iw_rdma_read(ukqp, &info, inv_stag, false);
2219                         if (ret) {
2220                                 if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
2221                                         err = -ENOMEM;
2222                                 else
2223                                         err = -EINVAL;
2224                         }
2225                         break;
2226                 case IB_WR_LOCAL_INV:
2227                         info.op_type = I40IW_OP_TYPE_INV_STAG;
2228                         info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey;
2229                         ret = ukqp->ops.iw_stag_local_invalidate(ukqp, &info, true);
2230                         if (ret)
2231                                 err = -ENOMEM;
2232                         break;
2233                 case IB_WR_REG_MR:
2234                 {
2235                         struct i40iw_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr);
2236                         int flags = reg_wr(ib_wr)->access;
2237                         struct i40iw_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc;
2238                         struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev;
2239                         struct i40iw_fast_reg_stag_info info;
2240
2241                         memset(&info, 0, sizeof(info));
2242                         info.access_rights = I40IW_ACCESS_FLAGS_LOCALREAD;
2243                         info.access_rights |= i40iw_get_user_access(flags);
2244                         info.stag_key = reg_wr(ib_wr)->key & 0xff;
2245                         info.stag_idx = reg_wr(ib_wr)->key >> 8;
2246                         info.page_size = reg_wr(ib_wr)->mr->page_size;
2247                         info.wr_id = ib_wr->wr_id;
2248
2249                         info.addr_type = I40IW_ADDR_TYPE_VA_BASED;
2250                         info.va = (void *)(uintptr_t)iwmr->ibmr.iova;
2251                         info.total_len = iwmr->ibmr.length;
2252                         info.reg_addr_pa = *(u64 *)palloc->level1.addr;
2253                         info.first_pm_pbl_index = palloc->level1.idx;
2254                         info.local_fence = ib_wr->send_flags & IB_SEND_FENCE;
2255                         info.signaled = ib_wr->send_flags & IB_SEND_SIGNALED;
2256
2257                         if (iwmr->npages > I40IW_MIN_PAGES_PER_FMR)
2258                                 info.chunk_size = 1;
2259
2260                         ret = dev->iw_priv_qp_ops->iw_mr_fast_register(&iwqp->sc_qp, &info, true);
2261                         if (ret)
2262                                 err = -ENOMEM;
2263                         break;
2264                 }
2265                 default:
2266                         err = -EINVAL;
2267                         i40iw_pr_err(" upost_send bad opcode = 0x%x\n",
2268                                      ib_wr->opcode);
2269                         break;
2270                 }
2271
2272                 if (err)
2273                         break;
2274                 ib_wr = ib_wr->next;
2275         }
2276
2277 out:
2278         if (err)
2279                 *bad_wr = ib_wr;
2280         else
2281                 ukqp->ops.iw_qp_post_wr(ukqp);
2282         spin_unlock_irqrestore(&iwqp->lock, flags);
2283
2284         return err;
2285 }
2286
2287 /**
2288  * i40iw_post_recv - post receive wr for kernel application
2289  * @ibqp: ib qp pointer
2290  * @ib_wr: work request for receive
2291  * @bad_wr: bad wr caused an error
2292  */
2293 static int i40iw_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *ib_wr,
2294                            const struct ib_recv_wr **bad_wr)
2295 {
2296         struct i40iw_qp *iwqp;
2297         struct i40iw_qp_uk *ukqp;
2298         struct i40iw_post_rq_info post_recv;
2299         struct i40iw_sge sg_list[I40IW_MAX_WQ_FRAGMENT_COUNT];
2300         enum i40iw_status_code ret = 0;
2301         unsigned long flags;
2302         int err = 0;
2303
2304         iwqp = (struct i40iw_qp *)ibqp;
2305         ukqp = &iwqp->sc_qp.qp_uk;
2306
2307         memset(&post_recv, 0, sizeof(post_recv));
2308         spin_lock_irqsave(&iwqp->lock, flags);
2309
2310         if (iwqp->flush_issued) {
2311                 err = -EINVAL;
2312                 goto out;
2313         }
2314
2315         while (ib_wr) {
2316                 post_recv.num_sges = ib_wr->num_sge;
2317                 post_recv.wr_id = ib_wr->wr_id;
2318                 i40iw_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
2319                 post_recv.sg_list = sg_list;
2320                 ret = ukqp->ops.iw_post_receive(ukqp, &post_recv);
2321                 if (ret) {
2322                         i40iw_pr_err(" post_recv err %d\n", ret);
2323                         if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
2324                                 err = -ENOMEM;
2325                         else
2326                                 err = -EINVAL;
2327                         *bad_wr = ib_wr;
2328                         goto out;
2329                 }
2330                 ib_wr = ib_wr->next;
2331         }
2332  out:
2333         spin_unlock_irqrestore(&iwqp->lock, flags);
2334         return err;
2335 }
2336
2337 /**
2338  * i40iw_poll_cq - poll cq for completion (kernel apps)
2339  * @ibcq: cq to poll
2340  * @num_entries: number of entries to poll
2341  * @entry: wr of entry completed
2342  */
2343 static int i40iw_poll_cq(struct ib_cq *ibcq,
2344                          int num_entries,
2345                          struct ib_wc *entry)
2346 {
2347         struct i40iw_cq *iwcq;
2348         int cqe_count = 0;
2349         struct i40iw_cq_poll_info cq_poll_info;
2350         enum i40iw_status_code ret;
2351         struct i40iw_cq_uk *ukcq;
2352         struct i40iw_sc_qp *qp;
2353         struct i40iw_qp *iwqp;
2354         unsigned long flags;
2355
2356         iwcq = (struct i40iw_cq *)ibcq;
2357         ukcq = &iwcq->sc_cq.cq_uk;
2358
2359         spin_lock_irqsave(&iwcq->lock, flags);
2360         while (cqe_count < num_entries) {
2361                 ret = ukcq->ops.iw_cq_poll_completion(ukcq, &cq_poll_info);
2362                 if (ret == I40IW_ERR_QUEUE_EMPTY) {
2363                         break;
2364                 } else if (ret == I40IW_ERR_QUEUE_DESTROYED) {
2365                         continue;
2366                 } else if (ret) {
2367                         if (!cqe_count)
2368                                 cqe_count = -1;
2369                         break;
2370                 }
2371                 entry->wc_flags = 0;
2372                 entry->wr_id = cq_poll_info.wr_id;
2373                 if (cq_poll_info.error) {
2374                         entry->status = IB_WC_WR_FLUSH_ERR;
2375                         entry->vendor_err = cq_poll_info.major_err << 16 | cq_poll_info.minor_err;
2376                 } else {
2377                         entry->status = IB_WC_SUCCESS;
2378                 }
2379
2380                 switch (cq_poll_info.op_type) {
2381                 case I40IW_OP_TYPE_RDMA_WRITE:
2382                         entry->opcode = IB_WC_RDMA_WRITE;
2383                         break;
2384                 case I40IW_OP_TYPE_RDMA_READ_INV_STAG:
2385                 case I40IW_OP_TYPE_RDMA_READ:
2386                         entry->opcode = IB_WC_RDMA_READ;
2387                         break;
2388                 case I40IW_OP_TYPE_SEND_SOL:
2389                 case I40IW_OP_TYPE_SEND_SOL_INV:
2390                 case I40IW_OP_TYPE_SEND_INV:
2391                 case I40IW_OP_TYPE_SEND:
2392                         entry->opcode = IB_WC_SEND;
2393                         break;
2394                 case I40IW_OP_TYPE_REC:
2395                         entry->opcode = IB_WC_RECV;
2396                         break;
2397                 default:
2398                         entry->opcode = IB_WC_RECV;
2399                         break;
2400                 }
2401
2402                 entry->ex.imm_data = 0;
2403                 qp = (struct i40iw_sc_qp *)cq_poll_info.qp_handle;
2404                 entry->qp = (struct ib_qp *)qp->back_qp;
2405                 entry->src_qp = cq_poll_info.qp_id;
2406                 iwqp = (struct i40iw_qp *)qp->back_qp;
2407                 if (iwqp->iwarp_state > I40IW_QP_STATE_RTS) {
2408                         if (!I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring))
2409                                 complete(&iwqp->sq_drained);
2410                         if (!I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring))
2411                                 complete(&iwqp->rq_drained);
2412                 }
2413                 entry->byte_len = cq_poll_info.bytes_xfered;
2414                 entry++;
2415                 cqe_count++;
2416         }
2417         spin_unlock_irqrestore(&iwcq->lock, flags);
2418         return cqe_count;
2419 }
2420
2421 /**
2422  * i40iw_req_notify_cq - arm cq kernel application
2423  * @ibcq: cq to arm
2424  * @notify_flags: notofication flags
2425  */
2426 static int i40iw_req_notify_cq(struct ib_cq *ibcq,
2427                                enum ib_cq_notify_flags notify_flags)
2428 {
2429         struct i40iw_cq *iwcq;
2430         struct i40iw_cq_uk *ukcq;
2431         unsigned long flags;
2432         enum i40iw_completion_notify cq_notify = IW_CQ_COMPL_EVENT;
2433
2434         iwcq = (struct i40iw_cq *)ibcq;
2435         ukcq = &iwcq->sc_cq.cq_uk;
2436         if (notify_flags == IB_CQ_SOLICITED)
2437                 cq_notify = IW_CQ_COMPL_SOLICITED;
2438         spin_lock_irqsave(&iwcq->lock, flags);
2439         ukcq->ops.iw_cq_request_notification(ukcq, cq_notify);
2440         spin_unlock_irqrestore(&iwcq->lock, flags);
2441         return 0;
2442 }
2443
2444 /**
2445  * i40iw_port_immutable - return port's immutable data
2446  * @ibdev: ib dev struct
2447  * @port_num: port number
2448  * @immutable: immutable data for the port return
2449  */
2450 static int i40iw_port_immutable(struct ib_device *ibdev, u8 port_num,
2451                                 struct ib_port_immutable *immutable)
2452 {
2453         struct ib_port_attr attr;
2454         int err;
2455
2456         immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
2457
2458         err = ib_query_port(ibdev, port_num, &attr);
2459
2460         if (err)
2461                 return err;
2462
2463         immutable->gid_tbl_len = attr.gid_tbl_len;
2464
2465         return 0;
2466 }
2467
2468 static const char * const i40iw_hw_stat_names[] = {
2469         // 32bit names
2470         [I40IW_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards",
2471         [I40IW_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts",
2472         [I40IW_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes",
2473         [I40IW_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards",
2474         [I40IW_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts",
2475         [I40IW_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes",
2476         [I40IW_HW_STAT_INDEX_TCPRTXSEG] = "tcpRetransSegs",
2477         [I40IW_HW_STAT_INDEX_TCPRXOPTERR] = "tcpInOptErrors",
2478         [I40IW_HW_STAT_INDEX_TCPRXPROTOERR] = "tcpInProtoErrors",
2479         // 64bit names
2480         [I40IW_HW_STAT_INDEX_IP4RXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =
2481                 "ip4InOctets",
2482         [I40IW_HW_STAT_INDEX_IP4RXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
2483                 "ip4InPkts",
2484         [I40IW_HW_STAT_INDEX_IP4RXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =
2485                 "ip4InReasmRqd",
2486         [I40IW_HW_STAT_INDEX_IP4RXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
2487                 "ip4InMcastPkts",
2488         [I40IW_HW_STAT_INDEX_IP4TXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =
2489                 "ip4OutOctets",
2490         [I40IW_HW_STAT_INDEX_IP4TXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
2491                 "ip4OutPkts",
2492         [I40IW_HW_STAT_INDEX_IP4TXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =
2493                 "ip4OutSegRqd",
2494         [I40IW_HW_STAT_INDEX_IP4TXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
2495                 "ip4OutMcastPkts",
2496         [I40IW_HW_STAT_INDEX_IP6RXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =
2497                 "ip6InOctets",
2498         [I40IW_HW_STAT_INDEX_IP6RXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
2499                 "ip6InPkts",
2500         [I40IW_HW_STAT_INDEX_IP6RXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =
2501                 "ip6InReasmRqd",
2502         [I40IW_HW_STAT_INDEX_IP6RXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
2503                 "ip6InMcastPkts",
2504         [I40IW_HW_STAT_INDEX_IP6TXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =
2505                 "ip6OutOctets",
2506         [I40IW_HW_STAT_INDEX_IP6TXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
2507                 "ip6OutPkts",
2508         [I40IW_HW_STAT_INDEX_IP6TXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =
2509                 "ip6OutSegRqd",
2510         [I40IW_HW_STAT_INDEX_IP6TXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
2511                 "ip6OutMcastPkts",
2512         [I40IW_HW_STAT_INDEX_TCPRXSEGS + I40IW_HW_STAT_INDEX_MAX_32] =
2513                 "tcpInSegs",
2514         [I40IW_HW_STAT_INDEX_TCPTXSEG + I40IW_HW_STAT_INDEX_MAX_32] =
2515                 "tcpOutSegs",
2516         [I40IW_HW_STAT_INDEX_RDMARXRDS + I40IW_HW_STAT_INDEX_MAX_32] =
2517                 "iwInRdmaReads",
2518         [I40IW_HW_STAT_INDEX_RDMARXSNDS + I40IW_HW_STAT_INDEX_MAX_32] =
2519                 "iwInRdmaSends",
2520         [I40IW_HW_STAT_INDEX_RDMARXWRS + I40IW_HW_STAT_INDEX_MAX_32] =
2521                 "iwInRdmaWrites",
2522         [I40IW_HW_STAT_INDEX_RDMATXRDS + I40IW_HW_STAT_INDEX_MAX_32] =
2523                 "iwOutRdmaReads",
2524         [I40IW_HW_STAT_INDEX_RDMATXSNDS + I40IW_HW_STAT_INDEX_MAX_32] =
2525                 "iwOutRdmaSends",
2526         [I40IW_HW_STAT_INDEX_RDMATXWRS + I40IW_HW_STAT_INDEX_MAX_32] =
2527                 "iwOutRdmaWrites",
2528         [I40IW_HW_STAT_INDEX_RDMAVBND + I40IW_HW_STAT_INDEX_MAX_32] =
2529                 "iwRdmaBnd",
2530         [I40IW_HW_STAT_INDEX_RDMAVINV + I40IW_HW_STAT_INDEX_MAX_32] =
2531                 "iwRdmaInv"
2532 };
2533
2534 static void i40iw_get_dev_fw_str(struct ib_device *dev, char *str)
2535 {
2536         struct i40iw_device *iwdev = to_iwdev(dev);
2537
2538         snprintf(str, IB_FW_VERSION_NAME_MAX, "%llu.%llu",
2539                  i40iw_fw_major_ver(&iwdev->sc_dev),
2540                  i40iw_fw_minor_ver(&iwdev->sc_dev));
2541 }
2542
2543 /**
2544  * i40iw_alloc_hw_stats - Allocate a hw stats structure
2545  * @ibdev: device pointer from stack
2546  * @port_num: port number
2547  */
2548 static struct rdma_hw_stats *i40iw_alloc_hw_stats(struct ib_device *ibdev,
2549                                                   u8 port_num)
2550 {
2551         struct i40iw_device *iwdev = to_iwdev(ibdev);
2552         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
2553         int num_counters = I40IW_HW_STAT_INDEX_MAX_32 +
2554                 I40IW_HW_STAT_INDEX_MAX_64;
2555         unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;
2556
2557         BUILD_BUG_ON(ARRAY_SIZE(i40iw_hw_stat_names) !=
2558                      (I40IW_HW_STAT_INDEX_MAX_32 +
2559                       I40IW_HW_STAT_INDEX_MAX_64));
2560
2561         /*
2562          * PFs get the default update lifespan, but VFs only update once
2563          * per second
2564          */
2565         if (!dev->is_pf)
2566                 lifespan = 1000;
2567         return rdma_alloc_hw_stats_struct(i40iw_hw_stat_names, num_counters,
2568                                           lifespan);
2569 }
2570
2571 /**
2572  * i40iw_get_hw_stats - Populates the rdma_hw_stats structure
2573  * @ibdev: device pointer from stack
2574  * @stats: stats pointer from stack
2575  * @port_num: port number
2576  * @index: which hw counter the stack is requesting we update
2577  */
2578 static int i40iw_get_hw_stats(struct ib_device *ibdev,
2579                               struct rdma_hw_stats *stats,
2580                               u8 port_num, int index)
2581 {
2582         struct i40iw_device *iwdev = to_iwdev(ibdev);
2583         struct i40iw_sc_dev *dev = &iwdev->sc_dev;
2584         struct i40iw_vsi_pestat *devstat = iwdev->vsi.pestat;
2585         struct i40iw_dev_hw_stats *hw_stats = &devstat->hw_stats;
2586
2587         if (dev->is_pf) {
2588                 i40iw_hw_stats_read_all(devstat, &devstat->hw_stats);
2589         } else {
2590                 if (i40iw_vchnl_vf_get_pe_stats(dev, &devstat->hw_stats))
2591                         return -ENOSYS;
2592         }
2593
2594         memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats));
2595
2596         return stats->num_counters;
2597 }
2598
2599 /**
2600  * i40iw_query_gid - Query port GID
2601  * @ibdev: device pointer from stack
2602  * @port: port number
2603  * @index: Entry index
2604  * @gid: Global ID
2605  */
2606 static int i40iw_query_gid(struct ib_device *ibdev,
2607                            u8 port,
2608                            int index,
2609                            union ib_gid *gid)
2610 {
2611         struct i40iw_device *iwdev = to_iwdev(ibdev);
2612
2613         memset(gid->raw, 0, sizeof(gid->raw));
2614         ether_addr_copy(gid->raw, iwdev->netdev->dev_addr);
2615         return 0;
2616 }
2617
2618 static const struct ib_device_ops i40iw_dev_ops = {
2619         .owner = THIS_MODULE,
2620         .driver_id = RDMA_DRIVER_I40IW,
2621         /* NOTE: Older kernels wrongly use 0 for the uverbs_abi_ver */
2622         .uverbs_abi_ver = I40IW_ABI_VER,
2623
2624         .alloc_hw_stats = i40iw_alloc_hw_stats,
2625         .alloc_mr = i40iw_alloc_mr,
2626         .alloc_pd = i40iw_alloc_pd,
2627         .alloc_ucontext = i40iw_alloc_ucontext,
2628         .create_cq = i40iw_create_cq,
2629         .create_qp = i40iw_create_qp,
2630         .dealloc_pd = i40iw_dealloc_pd,
2631         .dealloc_ucontext = i40iw_dealloc_ucontext,
2632         .dereg_mr = i40iw_dereg_mr,
2633         .destroy_cq = i40iw_destroy_cq,
2634         .destroy_qp = i40iw_destroy_qp,
2635         .drain_rq = i40iw_drain_rq,
2636         .drain_sq = i40iw_drain_sq,
2637         .get_dev_fw_str = i40iw_get_dev_fw_str,
2638         .get_dma_mr = i40iw_get_dma_mr,
2639         .get_hw_stats = i40iw_get_hw_stats,
2640         .get_port_immutable = i40iw_port_immutable,
2641         .iw_accept = i40iw_accept,
2642         .iw_add_ref = i40iw_qp_add_ref,
2643         .iw_connect = i40iw_connect,
2644         .iw_create_listen = i40iw_create_listen,
2645         .iw_destroy_listen = i40iw_destroy_listen,
2646         .iw_get_qp = i40iw_get_qp,
2647         .iw_reject = i40iw_reject,
2648         .iw_rem_ref = i40iw_qp_rem_ref,
2649         .map_mr_sg = i40iw_map_mr_sg,
2650         .mmap = i40iw_mmap,
2651         .modify_qp = i40iw_modify_qp,
2652         .poll_cq = i40iw_poll_cq,
2653         .post_recv = i40iw_post_recv,
2654         .post_send = i40iw_post_send,
2655         .query_device = i40iw_query_device,
2656         .query_gid = i40iw_query_gid,
2657         .query_port = i40iw_query_port,
2658         .query_qp = i40iw_query_qp,
2659         .reg_user_mr = i40iw_reg_user_mr,
2660         .req_notify_cq = i40iw_req_notify_cq,
2661         INIT_RDMA_OBJ_SIZE(ib_pd, i40iw_pd, ibpd),
2662         INIT_RDMA_OBJ_SIZE(ib_cq, i40iw_cq, ibcq),
2663         INIT_RDMA_OBJ_SIZE(ib_ucontext, i40iw_ucontext, ibucontext),
2664 };
2665
2666 /**
2667  * i40iw_init_rdma_device - initialization of iwarp device
2668  * @iwdev: iwarp device
2669  */
2670 static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev)
2671 {
2672         struct i40iw_ib_device *iwibdev;
2673         struct net_device *netdev = iwdev->netdev;
2674         struct pci_dev *pcidev = iwdev->hw.pcidev;
2675
2676         iwibdev = ib_alloc_device(i40iw_ib_device, ibdev);
2677         if (!iwibdev) {
2678                 i40iw_pr_err("iwdev == NULL\n");
2679                 return NULL;
2680         }
2681         iwdev->iwibdev = iwibdev;
2682         iwibdev->iwdev = iwdev;
2683
2684         iwibdev->ibdev.node_type = RDMA_NODE_RNIC;
2685         ether_addr_copy((u8 *)&iwibdev->ibdev.node_guid, netdev->dev_addr);
2686
2687         iwibdev->ibdev.uverbs_cmd_mask =
2688             (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
2689             (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
2690             (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
2691             (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
2692             (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
2693             (1ull << IB_USER_VERBS_CMD_REG_MR) |
2694             (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
2695             (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
2696             (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
2697             (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
2698             (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
2699             (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
2700             (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
2701             (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
2702             (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
2703             (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
2704             (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
2705             (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
2706             (1ull << IB_USER_VERBS_CMD_POST_RECV) |
2707             (1ull << IB_USER_VERBS_CMD_POST_SEND);
2708         iwibdev->ibdev.phys_port_cnt = 1;
2709         iwibdev->ibdev.num_comp_vectors = iwdev->ceqs_count;
2710         iwibdev->ibdev.dev.parent = &pcidev->dev;
2711         memcpy(iwibdev->ibdev.iw_ifname, netdev->name,
2712                sizeof(iwibdev->ibdev.iw_ifname));
2713         ib_set_device_ops(&iwibdev->ibdev, &i40iw_dev_ops);
2714
2715         return iwibdev;
2716 }
2717
2718 /**
2719  * i40iw_port_ibevent - indicate port event
2720  * @iwdev: iwarp device
2721  */
2722 void i40iw_port_ibevent(struct i40iw_device *iwdev)
2723 {
2724         struct i40iw_ib_device *iwibdev = iwdev->iwibdev;
2725         struct ib_event event;
2726
2727         event.device = &iwibdev->ibdev;
2728         event.element.port_num = 1;
2729         event.event = iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
2730         ib_dispatch_event(&event);
2731 }
2732
2733 /**
2734  * i40iw_destroy_rdma_device - destroy rdma device and free resources
2735  * @iwibdev: IB device ptr
2736  */
2737 void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev)
2738 {
2739         ib_unregister_device(&iwibdev->ibdev);
2740         wait_event_timeout(iwibdev->iwdev->close_wq,
2741                            !atomic64_read(&iwibdev->iwdev->use_count),
2742                            I40IW_EVENT_TIMEOUT);
2743         ib_dealloc_device(&iwibdev->ibdev);
2744 }
2745
2746 /**
2747  * i40iw_register_rdma_device - register iwarp device to IB
2748  * @iwdev: iwarp device
2749  */
2750 int i40iw_register_rdma_device(struct i40iw_device *iwdev)
2751 {
2752         int ret;
2753         struct i40iw_ib_device *iwibdev;
2754
2755         iwdev->iwibdev = i40iw_init_rdma_device(iwdev);
2756         if (!iwdev->iwibdev)
2757                 return -ENOMEM;
2758         iwibdev = iwdev->iwibdev;
2759         rdma_set_device_sysfs_group(&iwibdev->ibdev, &i40iw_attr_group);
2760         ret = ib_device_set_netdev(&iwibdev->ibdev, iwdev->netdev, 1);
2761         if (ret)
2762                 goto error;
2763
2764         dma_set_max_seg_size(&iwdev->hw.pcidev->dev, UINT_MAX);
2765         ret = ib_register_device(&iwibdev->ibdev, "i40iw%d", &iwdev->hw.pcidev->dev);
2766         if (ret)
2767                 goto error;
2768
2769         return 0;
2770 error:
2771         ib_dealloc_device(&iwdev->iwibdev->ibdev);
2772         return ret;
2773 }