drivers/infiniband/sw/siw/siw_verbs.c

   1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
   2
   3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
   4 /* Copyright (c) 2008-2019, IBM Corporation */
   5
   6 #include <linux/errno.h>
   7 #include <linux/types.h>
   8 #include <linux/uaccess.h>
   9 #include <linux/vmalloc.h>
  10 #include <linux/xarray.h>
  11
  12 #include <rdma/iw_cm.h>
  13 #include <rdma/ib_verbs.h>
  14 #include <rdma/ib_user_verbs.h>
  15 #include <rdma/uverbs_ioctl.h>
  16
  17 #include "siw.h"
  18 #include "siw_verbs.h"
  19 #include "siw_mem.h"
  20
  21 static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
  22         [IB_QPS_RESET] = SIW_QP_STATE_IDLE,
  23         [IB_QPS_INIT] = SIW_QP_STATE_IDLE,
  24         [IB_QPS_RTR] = SIW_QP_STATE_RTR,
  25         [IB_QPS_RTS] = SIW_QP_STATE_RTS,
  26         [IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
  27         [IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
  28         [IB_QPS_ERR] = SIW_QP_STATE_ERROR
  29 };
  30
  31 static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
  32         [IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
  33         [IB_QPS_RTS] = "RTS",     [IB_QPS_SQD] = "SQD",   [IB_QPS_SQE] = "SQE",
  34         [IB_QPS_ERR] = "ERR"
  35 };
  36
  37 void siw_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
  38 {
  39         struct siw_user_mmap_entry *entry = to_siw_mmap_entry(rdma_entry);
  40
  41         kfree(entry);
  42 }
  43
  44 int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
  45 {
  46         struct siw_ucontext *uctx = to_siw_ctx(ctx);
  47         size_t size = vma->vm_end - vma->vm_start;
  48         struct rdma_user_mmap_entry *rdma_entry;
  49         struct siw_user_mmap_entry *entry;
  50         int rv = -EINVAL;
  51
  52         /*
  53          * Must be page aligned
  54          */
  55         if (vma->vm_start & (PAGE_SIZE - 1)) {
  56                 pr_warn("siw: mmap not page aligned\n");
  57                 return -EINVAL;
  58         }
  59         rdma_entry = rdma_user_mmap_entry_get(&uctx->base_ucontext, vma);
  60         if (!rdma_entry) {
  61                 siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %#zx\n",
  62                         vma->vm_pgoff, size);
  63                 return -EINVAL;
  64         }
  65         entry = to_siw_mmap_entry(rdma_entry);
  66
  67         rv = remap_vmalloc_range(vma, entry->address, 0);
  68         if (rv) {
  69                 pr_warn("remap_vmalloc_range failed: %lu, %zu\n", vma->vm_pgoff,
  70                         size);
  71                 goto out;
  72         }
  73 out:
  74         rdma_user_mmap_entry_put(rdma_entry);
  75
  76         return rv;
  77 }
  78
  79 int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
  80 {
  81         struct siw_device *sdev = to_siw_dev(base_ctx->device);
  82         struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
  83         struct siw_uresp_alloc_ctx uresp = {};
  84         int rv;
  85
  86         if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
  87                 rv = -ENOMEM;
  88                 goto err_out;
  89         }
  90         ctx->sdev = sdev;
  91
  92         uresp.dev_id = sdev->vendor_part_id;
  93
  94         if (udata->outlen < sizeof(uresp)) {
  95                 rv = -EINVAL;
  96                 goto err_out;
  97         }
  98         rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
  99         if (rv)
 100                 goto err_out;
 101
 102         siw_dbg(base_ctx->device, "success. now %d context(s)\n",
 103                 atomic_read(&sdev->num_ctx));
 104
 105         return 0;
 106
 107 err_out:
 108         atomic_dec(&sdev->num_ctx);
 109         siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
 110                 atomic_read(&sdev->num_ctx));
 111
 112         return rv;
 113 }
 114
 115 void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
 116 {
 117         struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
 118
 119         atomic_dec(&uctx->sdev->num_ctx);
 120 }
 121
 122 int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
 123                      struct ib_udata *udata)
 124 {
 125         struct siw_device *sdev = to_siw_dev(base_dev);
 126
 127         if (udata->inlen || udata->outlen)
 128                 return -EINVAL;
 129
 130         memset(attr, 0, sizeof(*attr));
 131
 132         /* Revisit atomic caps if RFC 7306 gets supported */
 133         attr->atomic_cap = 0;
 134         attr->device_cap_flags =
 135                 IB_DEVICE_MEM_MGT_EXTENSIONS | IB_DEVICE_ALLOW_USER_UNREG;
 136         attr->max_cq = sdev->attrs.max_cq;
 137         attr->max_cqe = sdev->attrs.max_cqe;
 138         attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
 139         attr->max_mr = sdev->attrs.max_mr;
 140         attr->max_mw = sdev->attrs.max_mw;
 141         attr->max_mr_size = ~0ull;
 142         attr->max_pd = sdev->attrs.max_pd;
 143         attr->max_qp = sdev->attrs.max_qp;
 144         attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
 145         attr->max_qp_rd_atom = sdev->attrs.max_ord;
 146         attr->max_qp_wr = sdev->attrs.max_qp_wr;
 147         attr->max_recv_sge = sdev->attrs.max_sge;
 148         attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
 149         attr->max_send_sge = sdev->attrs.max_sge;
 150         attr->max_sge_rd = sdev->attrs.max_sge_rd;
 151         attr->max_srq = sdev->attrs.max_srq;
 152         attr->max_srq_sge = sdev->attrs.max_srq_sge;
 153         attr->max_srq_wr = sdev->attrs.max_srq_wr;
 154         attr->page_size_cap = PAGE_SIZE;
 155         attr->vendor_id = SIW_VENDOR_ID;
 156         attr->vendor_part_id = sdev->vendor_part_id;
 157
 158         memcpy(&attr->sys_image_guid, sdev->netdev->dev_addr, 6);
 159
 160         return 0;
 161 }
 162
 163 int siw_query_port(struct ib_device *base_dev, u32 port,
 164                    struct ib_port_attr *attr)
 165 {
 166         struct siw_device *sdev = to_siw_dev(base_dev);
 167         int rv;
 168
 169         memset(attr, 0, sizeof(*attr));
 170
 171         rv = ib_get_eth_speed(base_dev, port, &attr->active_speed,
 172                          &attr->active_width);
 173         attr->gid_tbl_len = 1;
 174         attr->max_msg_sz = -1;
 175         attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 176         attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 177         attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
 178                 IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
 179         attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
 180         attr->state = sdev->state;
 181         /*
 182          * All zero
 183          *
 184          * attr->lid = 0;
 185          * attr->bad_pkey_cntr = 0;
 186          * attr->qkey_viol_cntr = 0;
 187          * attr->sm_lid = 0;
 188          * attr->lmc = 0;
 189          * attr->max_vl_num = 0;
 190          * attr->sm_sl = 0;
 191          * attr->subnet_timeout = 0;
 192          * attr->init_type_repy = 0;
 193          */
 194         return rv;
 195 }
 196
 197 int siw_get_port_immutable(struct ib_device *base_dev, u32 port,
 198                            struct ib_port_immutable *port_immutable)
 199 {
 200         struct ib_port_attr attr;
 201         int rv = siw_query_port(base_dev, port, &attr);
 202
 203         if (rv)
 204                 return rv;
 205
 206         port_immutable->gid_tbl_len = attr.gid_tbl_len;
 207         port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
 208
 209         return 0;
 210 }
 211
 212 int siw_query_gid(struct ib_device *base_dev, u32 port, int idx,
 213                   union ib_gid *gid)
 214 {
 215         struct siw_device *sdev = to_siw_dev(base_dev);
 216
 217         /* subnet_prefix == interface_id == 0; */
 218         memset(gid, 0, sizeof(*gid));
 219         memcpy(&gid->raw[0], sdev->netdev->dev_addr, 6);
 220
 221         return 0;
 222 }
 223
 224 int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 225 {
 226         struct siw_device *sdev = to_siw_dev(pd->device);
 227
 228         if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
 229                 atomic_dec(&sdev->num_pd);
 230                 return -ENOMEM;
 231         }
 232         siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
 233
 234         return 0;
 235 }
 236
 237 int siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 238 {
 239         struct siw_device *sdev = to_siw_dev(pd->device);
 240
 241         siw_dbg_pd(pd, "free PD\n");
 242         atomic_dec(&sdev->num_pd);
 243         return 0;
 244 }
 245
 246 void siw_qp_get_ref(struct ib_qp *base_qp)
 247 {
 248         siw_qp_get(to_siw_qp(base_qp));
 249 }
 250
 251 void siw_qp_put_ref(struct ib_qp *base_qp)
 252 {
 253         siw_qp_put(to_siw_qp(base_qp));
 254 }
 255
 256 static struct rdma_user_mmap_entry *
 257 siw_mmap_entry_insert(struct siw_ucontext *uctx,
 258                       void *address, size_t length,
 259                       u64 *offset)
 260 {
 261         struct siw_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 262         int rv;
 263
 264         *offset = SIW_INVAL_UOBJ_KEY;
 265         if (!entry)
 266                 return NULL;
 267
 268         entry->address = address;
 269
 270         rv = rdma_user_mmap_entry_insert(&uctx->base_ucontext,
 271                                          &entry->rdma_entry,
 272                                          length);
 273         if (rv) {
 274                 kfree(entry);
 275                 return NULL;
 276         }
 277
 278         *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
 279
 280         return &entry->rdma_entry;
 281 }
 282
 283 /*
 284  * siw_create_qp()
 285  *
 286  * Create QP of requested size on given device.
 287  *
 288  * @pd:         Protection Domain
 289  * @attrs:      Initial QP attributes.
 290  * @udata:      used to provide QP ID, SQ and RQ size back to user.
 291  */
 292
 293 struct ib_qp *siw_create_qp(struct ib_pd *pd,
 294                             struct ib_qp_init_attr *attrs,
 295                             struct ib_udata *udata)
 296 {
 297         struct siw_qp *qp = NULL;
 298         struct ib_device *base_dev = pd->device;
 299         struct siw_device *sdev = to_siw_dev(base_dev);
 300         struct siw_ucontext *uctx =
 301                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
 302                                           base_ucontext);
 303         struct siw_cq *scq = NULL, *rcq = NULL;
 304         unsigned long flags;
 305         int num_sqe, num_rqe, rv = 0;
 306         size_t length;
 307
 308         siw_dbg(base_dev, "create new QP\n");
 309
 310         if (attrs->create_flags)
 311                 return ERR_PTR(-EOPNOTSUPP);
 312
 313         if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
 314                 siw_dbg(base_dev, "too many QP's\n");
 315                 rv = -ENOMEM;
 316                 goto err_out;
 317         }
 318         if (attrs->qp_type != IB_QPT_RC) {
 319                 siw_dbg(base_dev, "only RC QP's supported\n");
 320                 rv = -EOPNOTSUPP;
 321                 goto err_out;
 322         }
 323         if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
 324             (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
 325             (attrs->cap.max_send_sge > SIW_MAX_SGE) ||
 326             (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
 327                 siw_dbg(base_dev, "QP size error\n");
 328                 rv = -EINVAL;
 329                 goto err_out;
 330         }
 331         if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
 332                 siw_dbg(base_dev, "max inline send: %d > %d\n",
 333                         attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
 334                 rv = -EINVAL;
 335                 goto err_out;
 336         }
 337         /*
 338          * NOTE: we allow for zero element SQ and RQ WQE's SGL's
 339          * but not for a QP unable to hold any WQE (SQ + RQ)
 340          */
 341         if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0) {
 342                 siw_dbg(base_dev, "QP must have send or receive queue\n");
 343                 rv = -EINVAL;
 344                 goto err_out;
 345         }
 346         scq = to_siw_cq(attrs->send_cq);
 347         rcq = to_siw_cq(attrs->recv_cq);
 348
 349         if (!scq || (!rcq && !attrs->srq)) {
 350                 siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
 351                 rv = -EINVAL;
 352                 goto err_out;
 353         }
 354         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
 355         if (!qp) {
 356                 rv = -ENOMEM;
 357                 goto err_out;
 358         }
 359         init_rwsem(&qp->state_lock);
 360         spin_lock_init(&qp->sq_lock);
 361         spin_lock_init(&qp->rq_lock);
 362         spin_lock_init(&qp->orq_lock);
 363
 364         rv = siw_qp_add(sdev, qp);
 365         if (rv)
 366                 goto err_out;
 367
 368         num_sqe = attrs->cap.max_send_wr;
 369         num_rqe = attrs->cap.max_recv_wr;
 370
 371         /* All queue indices are derived from modulo operations
 372          * on a free running 'get' (consumer) and 'put' (producer)
 373          * unsigned counter. Having queue sizes at power of two
 374          * avoids handling counter wrap around.
 375          */
 376         if (num_sqe)
 377                 num_sqe = roundup_pow_of_two(num_sqe);
 378         else {
 379                 /* Zero sized SQ is not supported */
 380                 rv = -EINVAL;
 381                 goto err_out;
 382         }
 383         if (num_rqe)
 384                 num_rqe = roundup_pow_of_two(num_rqe);
 385
 386         if (udata)
 387                 qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
 388         else
 389                 qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe));
 390
 391         if (qp->sendq == NULL) {
 392                 rv = -ENOMEM;
 393                 goto err_out_xa;
 394         }
 395         if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
 396                 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
 397                         qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
 398                 else {
 399                         rv = -EINVAL;
 400                         goto err_out_xa;
 401                 }
 402         }
 403         qp->pd = pd;
 404         qp->scq = scq;
 405         qp->rcq = rcq;
 406
 407         if (attrs->srq) {
 408                 /*
 409                  * SRQ support.
 410                  * Verbs 6.3.7: ignore RQ size, if SRQ present
 411                  * Verbs 6.3.5: do not check PD of SRQ against PD of QP
 412                  */
 413                 qp->srq = to_siw_srq(attrs->srq);
 414                 qp->attrs.rq_size = 0;
 415                 siw_dbg(base_dev, "QP [%u]: SRQ attached\n",
 416                         qp->base_qp.qp_num);
 417         } else if (num_rqe) {
 418                 if (udata)
 419                         qp->recvq =
 420                                 vmalloc_user(num_rqe * sizeof(struct siw_rqe));
 421                 else
 422                         qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe));
 423
 424                 if (qp->recvq == NULL) {
 425                         rv = -ENOMEM;
 426                         goto err_out_xa;
 427                 }
 428                 qp->attrs.rq_size = num_rqe;
 429         }
 430         qp->attrs.sq_size = num_sqe;
 431         qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
 432         qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
 433
 434         /* Make those two tunables fixed for now. */
 435         qp->tx_ctx.gso_seg_limit = 1;
 436         qp->tx_ctx.zcopy_tx = zcopy_tx;
 437
 438         qp->attrs.state = SIW_QP_STATE_IDLE;
 439
 440         if (udata) {
 441                 struct siw_uresp_create_qp uresp = {};
 442
 443                 uresp.num_sqe = num_sqe;
 444                 uresp.num_rqe = num_rqe;
 445                 uresp.qp_id = qp_id(qp);
 446
 447                 if (qp->sendq) {
 448                         length = num_sqe * sizeof(struct siw_sqe);
 449                         qp->sq_entry =
 450                                 siw_mmap_entry_insert(uctx, qp->sendq,
 451                                                       length, &uresp.sq_key);
 452                         if (!qp->sq_entry) {
 453                                 rv = -ENOMEM;
 454                                 goto err_out_xa;
 455                         }
 456                 }
 457
 458                 if (qp->recvq) {
 459                         length = num_rqe * sizeof(struct siw_rqe);
 460                         qp->rq_entry =
 461                                 siw_mmap_entry_insert(uctx, qp->recvq,
 462                                                       length, &uresp.rq_key);
 463                         if (!qp->rq_entry) {
 464                                 uresp.sq_key = SIW_INVAL_UOBJ_KEY;
 465                                 rv = -ENOMEM;
 466                                 goto err_out_xa;
 467                         }
 468                 }
 469
 470                 if (udata->outlen < sizeof(uresp)) {
 471                         rv = -EINVAL;
 472                         goto err_out_xa;
 473                 }
 474                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 475                 if (rv)
 476                         goto err_out_xa;
 477         }
 478         qp->tx_cpu = siw_get_tx_cpu(sdev);
 479         if (qp->tx_cpu < 0) {
 480                 rv = -EINVAL;
 481                 goto err_out_xa;
 482         }
 483         INIT_LIST_HEAD(&qp->devq);
 484         spin_lock_irqsave(&sdev->lock, flags);
 485         list_add_tail(&qp->devq, &sdev->qp_list);
 486         spin_unlock_irqrestore(&sdev->lock, flags);
 487
 488         return &qp->base_qp;
 489
 490 err_out_xa:
 491         xa_erase(&sdev->qp_xa, qp_id(qp));
 492 err_out:
 493         if (qp) {
 494                 if (uctx) {
 495                         rdma_user_mmap_entry_remove(qp->sq_entry);
 496                         rdma_user_mmap_entry_remove(qp->rq_entry);
 497                 }
 498                 vfree(qp->sendq);
 499                 vfree(qp->recvq);
 500                 kfree(qp);
 501         }
 502         atomic_dec(&sdev->num_qp);
 503
 504         return ERR_PTR(rv);
 505 }
 506
 507 /*
 508  * Minimum siw_query_qp() verb interface.
 509  *
 510  * @qp_attr_mask is not used but all available information is provided
 511  */
 512 int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
 513                  int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
 514 {
 515         struct siw_qp *qp;
 516         struct siw_device *sdev;
 517
 518         if (base_qp && qp_attr && qp_init_attr) {
 519                 qp = to_siw_qp(base_qp);
 520                 sdev = to_siw_dev(base_qp->device);
 521         } else {
 522                 return -EINVAL;
 523         }
 524         qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
 525         qp_attr->cap.max_send_wr = qp->attrs.sq_size;
 526         qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
 527         qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
 528         qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
 529         qp_attr->path_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 530         qp_attr->max_rd_atomic = qp->attrs.irq_size;
 531         qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
 532
 533         qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
 534                                    IB_ACCESS_REMOTE_WRITE |
 535                                    IB_ACCESS_REMOTE_READ;
 536
 537         qp_init_attr->qp_type = base_qp->qp_type;
 538         qp_init_attr->send_cq = base_qp->send_cq;
 539         qp_init_attr->recv_cq = base_qp->recv_cq;
 540         qp_init_attr->srq = base_qp->srq;
 541
 542         qp_init_attr->cap = qp_attr->cap;
 543
 544         return 0;
 545 }
 546
 547 int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
 548                         int attr_mask, struct ib_udata *udata)
 549 {
 550         struct siw_qp_attrs new_attrs;
 551         enum siw_qp_attr_mask siw_attr_mask = 0;
 552         struct siw_qp *qp = to_siw_qp(base_qp);
 553         int rv = 0;
 554
 555         if (!attr_mask)
 556                 return 0;
 557
 558         if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
 559                 return -EOPNOTSUPP;
 560
 561         memset(&new_attrs, 0, sizeof(new_attrs));
 562
 563         if (attr_mask & IB_QP_ACCESS_FLAGS) {
 564                 siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
 565
 566                 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
 567                         new_attrs.flags |= SIW_RDMA_READ_ENABLED;
 568                 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
 569                         new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
 570                 if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
 571                         new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
 572         }
 573         if (attr_mask & IB_QP_STATE) {
 574                 siw_dbg_qp(qp, "desired IB QP state: %s\n",
 575                            ib_qp_state_to_string[attr->qp_state]);
 576
 577                 new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
 578
 579                 if (new_attrs.state > SIW_QP_STATE_RTS)
 580                         qp->tx_ctx.tx_suspend = 1;
 581
 582                 siw_attr_mask |= SIW_QP_ATTR_STATE;
 583         }
 584         if (!siw_attr_mask)
 585                 goto out;
 586
 587         down_write(&qp->state_lock);
 588
 589         rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
 590
 591         up_write(&qp->state_lock);
 592 out:
 593         return rv;
 594 }
 595
 596 int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
 597 {
 598         struct siw_qp *qp = to_siw_qp(base_qp);
 599         struct siw_ucontext *uctx =
 600                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
 601                                           base_ucontext);
 602         struct siw_qp_attrs qp_attrs;
 603
 604         siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
 605
 606         /*
 607          * Mark QP as in process of destruction to prevent from
 608          * any async callbacks to RDMA core
 609          */
 610         qp->attrs.flags |= SIW_QP_IN_DESTROY;
 611         qp->rx_stream.rx_suspend = 1;
 612
 613         if (uctx) {
 614                 rdma_user_mmap_entry_remove(qp->sq_entry);
 615                 rdma_user_mmap_entry_remove(qp->rq_entry);
 616         }
 617
 618         down_write(&qp->state_lock);
 619
 620         qp_attrs.state = SIW_QP_STATE_ERROR;
 621         siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
 622
 623         if (qp->cep) {
 624                 siw_cep_put(qp->cep);
 625                 qp->cep = NULL;
 626         }
 627         up_write(&qp->state_lock);
 628
 629         kfree(qp->tx_ctx.mpa_crc_hd);
 630         kfree(qp->rx_stream.mpa_crc_hd);
 631
 632         qp->scq = qp->rcq = NULL;
 633
 634         siw_qp_put(qp);
 635
 636         return 0;
 637 }
 638
 639 /*
 640  * siw_copy_inline_sgl()
 641  *
 642  * Prepare sgl of inlined data for sending. For userland callers
 643  * function checks if given buffer addresses and len's are within
 644  * process context bounds.
 645  * Data from all provided sge's are copied together into the wqe,
 646  * referenced by a single sge.
 647  */
 648 static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
 649                                struct siw_sqe *sqe)
 650 {
 651         struct ib_sge *core_sge = core_wr->sg_list;
 652         void *kbuf = &sqe->sge[1];
 653         int num_sge = core_wr->num_sge, bytes = 0;
 654
 655         sqe->sge[0].laddr = (uintptr_t)kbuf;
 656         sqe->sge[0].lkey = 0;
 657
 658         while (num_sge--) {
 659                 if (!core_sge->length) {
 660                         core_sge++;
 661                         continue;
 662                 }
 663                 bytes += core_sge->length;
 664                 if (bytes > SIW_MAX_INLINE) {
 665                         bytes = -EINVAL;
 666                         break;
 667                 }
 668                 memcpy(kbuf, (void *)(uintptr_t)core_sge->addr,
 669                        core_sge->length);
 670
 671                 kbuf += core_sge->length;
 672                 core_sge++;
 673         }
 674         sqe->sge[0].length = bytes > 0 ? bytes : 0;
 675         sqe->num_sge = bytes > 0 ? 1 : 0;
 676
 677         return bytes;
 678 }
 679
 680 /* Complete SQ WR's without processing */
 681 static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
 682                            const struct ib_send_wr **bad_wr)
 683 {
 684         struct siw_sqe sqe = {};
 685         int rv = 0;
 686
 687         while (wr) {
 688                 sqe.id = wr->wr_id;
 689                 sqe.opcode = wr->opcode;
 690                 rv = siw_sqe_complete(qp, &sqe, 0, SIW_WC_WR_FLUSH_ERR);
 691                 if (rv) {
 692                         if (bad_wr)
 693                                 *bad_wr = wr;
 694                         break;
 695                 }
 696                 wr = wr->next;
 697         }
 698         return rv;
 699 }
 700
 701 /* Complete RQ WR's without processing */
 702 static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
 703                            const struct ib_recv_wr **bad_wr)
 704 {
 705         struct siw_rqe rqe = {};
 706         int rv = 0;
 707
 708         while (wr) {
 709                 rqe.id = wr->wr_id;
 710                 rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR);
 711                 if (rv) {
 712                         if (bad_wr)
 713                                 *bad_wr = wr;
 714                         break;
 715                 }
 716                 wr = wr->next;
 717         }
 718         return rv;
 719 }
 720
 721 /*
 722  * siw_post_send()
 723  *
 724  * Post a list of S-WR's to a SQ.
 725  *
 726  * @base_qp:    Base QP contained in siw QP
 727  * @wr:         Null terminated list of user WR's
 728  * @bad_wr:     Points to failing WR in case of synchronous failure.
 729  */
 730 int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
 731                   const struct ib_send_wr **bad_wr)
 732 {
 733         struct siw_qp *qp = to_siw_qp(base_qp);
 734         struct siw_wqe *wqe = tx_wqe(qp);
 735
 736         unsigned long flags;
 737         int rv = 0;
 738
 739         if (wr && !rdma_is_kernel_res(&qp->base_qp.res)) {
 740                 siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
 741                 *bad_wr = wr;
 742                 return -EINVAL;
 743         }
 744
 745         /*
 746          * Try to acquire QP state lock. Must be non-blocking
 747          * to accommodate kernel clients needs.
 748          */
 749         if (!down_read_trylock(&qp->state_lock)) {
 750                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 751                         /*
 752                          * ERROR state is final, so we can be sure
 753                          * this state will not change as long as the QP
 754                          * exists.
 755                          *
 756                          * This handles an ib_drain_sq() call with
 757                          * a concurrent request to set the QP state
 758                          * to ERROR.
 759                          */
 760                         rv = siw_sq_flush_wr(qp, wr, bad_wr);
 761                 } else {
 762                         siw_dbg_qp(qp, "QP locked, state %d\n",
 763                                    qp->attrs.state);
 764                         *bad_wr = wr;
 765                         rv = -ENOTCONN;
 766                 }
 767                 return rv;
 768         }
 769         if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
 770                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 771                         /*
 772                          * Immediately flush this WR to CQ, if QP
 773                          * is in ERROR state. SQ is guaranteed to
 774                          * be empty, so WR complets in-order.
 775                          *
 776                          * Typically triggered by ib_drain_sq().
 777                          */
 778                         rv = siw_sq_flush_wr(qp, wr, bad_wr);
 779                 } else {
 780                         siw_dbg_qp(qp, "QP out of state %d\n",
 781                                    qp->attrs.state);
 782                         *bad_wr = wr;
 783                         rv = -ENOTCONN;
 784                 }
 785                 up_read(&qp->state_lock);
 786                 return rv;
 787         }
 788         spin_lock_irqsave(&qp->sq_lock, flags);
 789
 790         while (wr) {
 791                 u32 idx = qp->sq_put % qp->attrs.sq_size;
 792                 struct siw_sqe *sqe = &qp->sendq[idx];
 793
 794                 if (sqe->flags) {
 795                         siw_dbg_qp(qp, "sq full\n");
 796                         rv = -ENOMEM;
 797                         break;
 798                 }
 799                 if (wr->num_sge > qp->attrs.sq_max_sges) {
 800                         siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
 801                         rv = -EINVAL;
 802                         break;
 803                 }
 804                 sqe->id = wr->wr_id;
 805
 806                 if ((wr->send_flags & IB_SEND_SIGNALED) ||
 807                     (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
 808                         sqe->flags |= SIW_WQE_SIGNALLED;
 809
 810                 if (wr->send_flags & IB_SEND_FENCE)
 811                         sqe->flags |= SIW_WQE_READ_FENCE;
 812
 813                 switch (wr->opcode) {
 814                 case IB_WR_SEND:
 815                 case IB_WR_SEND_WITH_INV:
 816                         if (wr->send_flags & IB_SEND_SOLICITED)
 817                                 sqe->flags |= SIW_WQE_SOLICITED;
 818
 819                         if (!(wr->send_flags & IB_SEND_INLINE)) {
 820                                 siw_copy_sgl(wr->sg_list, sqe->sge,
 821                                              wr->num_sge);
 822                                 sqe->num_sge = wr->num_sge;
 823                         } else {
 824                                 rv = siw_copy_inline_sgl(wr, sqe);
 825                                 if (rv <= 0) {
 826                                         rv = -EINVAL;
 827                                         break;
 828                                 }
 829                                 sqe->flags |= SIW_WQE_INLINE;
 830                                 sqe->num_sge = 1;
 831                         }
 832                         if (wr->opcode == IB_WR_SEND)
 833                                 sqe->opcode = SIW_OP_SEND;
 834                         else {
 835                                 sqe->opcode = SIW_OP_SEND_REMOTE_INV;
 836                                 sqe->rkey = wr->ex.invalidate_rkey;
 837                         }
 838                         break;
 839
 840                 case IB_WR_RDMA_READ_WITH_INV:
 841                 case IB_WR_RDMA_READ:
 842                         /*
 843                          * iWarp restricts RREAD sink to SGL containing
 844                          * 1 SGE only. we could relax to SGL with multiple
 845                          * elements referring the SAME ltag or even sending
 846                          * a private per-rreq tag referring to a checked
 847                          * local sgl with MULTIPLE ltag's.
 848                          */
 849                         if (unlikely(wr->num_sge != 1)) {
 850                                 rv = -EINVAL;
 851                                 break;
 852                         }
 853                         siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
 854                         /*
 855                          * NOTE: zero length RREAD is allowed!
 856                          */
 857                         sqe->raddr = rdma_wr(wr)->remote_addr;
 858                         sqe->rkey = rdma_wr(wr)->rkey;
 859                         sqe->num_sge = 1;
 860
 861                         if (wr->opcode == IB_WR_RDMA_READ)
 862                                 sqe->opcode = SIW_OP_READ;
 863                         else
 864                                 sqe->opcode = SIW_OP_READ_LOCAL_INV;
 865                         break;
 866
 867                 case IB_WR_RDMA_WRITE:
 868                         if (!(wr->send_flags & IB_SEND_INLINE)) {
 869                                 siw_copy_sgl(wr->sg_list, &sqe->sge[0],
 870                                              wr->num_sge);
 871                                 sqe->num_sge = wr->num_sge;
 872                         } else {
 873                                 rv = siw_copy_inline_sgl(wr, sqe);
 874                                 if (unlikely(rv < 0)) {
 875                                         rv = -EINVAL;
 876                                         break;
 877                                 }
 878                                 sqe->flags |= SIW_WQE_INLINE;
 879                                 sqe->num_sge = 1;
 880                         }
 881                         sqe->raddr = rdma_wr(wr)->remote_addr;
 882                         sqe->rkey = rdma_wr(wr)->rkey;
 883                         sqe->opcode = SIW_OP_WRITE;
 884                         break;
 885
 886                 case IB_WR_REG_MR:
 887                         sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
 888                         sqe->rkey = reg_wr(wr)->key;
 889                         sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
 890                         sqe->opcode = SIW_OP_REG_MR;
 891                         break;
 892
 893                 case IB_WR_LOCAL_INV:
 894                         sqe->rkey = wr->ex.invalidate_rkey;
 895                         sqe->opcode = SIW_OP_INVAL_STAG;
 896                         break;
 897
 898                 default:
 899                         siw_dbg_qp(qp, "ib wr type %d unsupported\n",
 900                                    wr->opcode);
 901                         rv = -EINVAL;
 902                         break;
 903                 }
 904                 siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n",
 905                            sqe->opcode, sqe->flags,
 906                            (void *)(uintptr_t)sqe->id);
 907
 908                 if (unlikely(rv < 0))
 909                         break;
 910
 911                 /* make SQE only valid after completely written */
 912                 smp_wmb();
 913                 sqe->flags |= SIW_WQE_VALID;
 914
 915                 qp->sq_put++;
 916                 wr = wr->next;
 917         }
 918
 919         /*
 920          * Send directly if SQ processing is not in progress.
 921          * Eventual immediate errors (rv < 0) do not affect the involved
 922          * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
 923          * processing, if new work is already pending. But rv must be passed
 924          * to caller.
 925          */
 926         if (wqe->wr_status != SIW_WR_IDLE) {
 927                 spin_unlock_irqrestore(&qp->sq_lock, flags);
 928                 goto skip_direct_sending;
 929         }
 930         rv = siw_activate_tx(qp);
 931         spin_unlock_irqrestore(&qp->sq_lock, flags);
 932
 933         if (rv <= 0)
 934                 goto skip_direct_sending;
 935
 936         if (rdma_is_kernel_res(&qp->base_qp.res)) {
 937                 rv = siw_sq_start(qp);
 938         } else {
 939                 qp->tx_ctx.in_syscall = 1;
 940
 941                 if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
 942                         siw_qp_cm_drop(qp, 0);
 943
 944                 qp->tx_ctx.in_syscall = 0;
 945         }
 946 skip_direct_sending:
 947
 948         up_read(&qp->state_lock);
 949
 950         if (rv >= 0)
 951                 return 0;
 952         /*
 953          * Immediate error
 954          */
 955         siw_dbg_qp(qp, "error %d\n", rv);
 956
 957         *bad_wr = wr;
 958         return rv;
 959 }
 960
 961 /*
 962  * siw_post_receive()
 963  *
 964  * Post a list of R-WR's to a RQ.
 965  *
 966  * @base_qp:    Base QP contained in siw QP
 967  * @wr:         Null terminated list of user WR's
 968  * @bad_wr:     Points to failing WR in case of synchronous failure.
 969  */
 970 int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
 971                      const struct ib_recv_wr **bad_wr)
 972 {
 973         struct siw_qp *qp = to_siw_qp(base_qp);
 974         unsigned long flags;
 975         int rv = 0;
 976
 977         if (qp->srq || qp->attrs.rq_size == 0) {
 978                 *bad_wr = wr;
 979                 return -EINVAL;
 980         }
 981         if (!rdma_is_kernel_res(&qp->base_qp.res)) {
 982                 siw_dbg_qp(qp, "no kernel post_recv for user mapped rq\n");
 983                 *bad_wr = wr;
 984                 return -EINVAL;
 985         }
 986
 987         /*
 988          * Try to acquire QP state lock. Must be non-blocking
 989          * to accommodate kernel clients needs.
 990          */
 991         if (!down_read_trylock(&qp->state_lock)) {
 992                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 993                         /*
 994                          * ERROR state is final, so we can be sure
 995                          * this state will not change as long as the QP
 996                          * exists.
 997                          *
 998                          * This handles an ib_drain_rq() call with
 999                          * a concurrent request to set the QP state
1000                          * to ERROR.
1001                          */
1002                         rv = siw_rq_flush_wr(qp, wr, bad_wr);
1003                 } else {
1004                         siw_dbg_qp(qp, "QP locked, state %d\n",
1005                                    qp->attrs.state);
1006                         *bad_wr = wr;
1007                         rv = -ENOTCONN;
1008                 }
1009                 return rv;
1010         }
1011         if (qp->attrs.state > SIW_QP_STATE_RTS) {
1012                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1013                         /*
1014                          * Immediately flush this WR to CQ, if QP
1015                          * is in ERROR state. RQ is guaranteed to
1016                          * be empty, so WR complets in-order.
1017                          *
1018                          * Typically triggered by ib_drain_rq().
1019                          */
1020                         rv = siw_rq_flush_wr(qp, wr, bad_wr);
1021                 } else {
1022                         siw_dbg_qp(qp, "QP out of state %d\n",
1023                                    qp->attrs.state);
1024                         *bad_wr = wr;
1025                         rv = -ENOTCONN;
1026                 }
1027                 up_read(&qp->state_lock);
1028                 return rv;
1029         }
1030         /*
1031          * Serialize potentially multiple producers.
1032          * Not needed for single threaded consumer side.
1033          */
1034         spin_lock_irqsave(&qp->rq_lock, flags);
1035
1036         while (wr) {
1037                 u32 idx = qp->rq_put % qp->attrs.rq_size;
1038                 struct siw_rqe *rqe = &qp->recvq[idx];
1039
1040                 if (rqe->flags) {
1041                         siw_dbg_qp(qp, "RQ full\n");
1042                         rv = -ENOMEM;
1043                         break;
1044                 }
1045                 if (wr->num_sge > qp->attrs.rq_max_sges) {
1046                         siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1047                         rv = -EINVAL;
1048                         break;
1049                 }
1050                 rqe->id = wr->wr_id;
1051                 rqe->num_sge = wr->num_sge;
1052                 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1053
1054                 /* make sure RQE is completely written before valid */
1055                 smp_wmb();
1056
1057                 rqe->flags = SIW_WQE_VALID;
1058
1059                 qp->rq_put++;
1060                 wr = wr->next;
1061         }
1062         spin_unlock_irqrestore(&qp->rq_lock, flags);
1063
1064         up_read(&qp->state_lock);
1065
1066         if (rv < 0) {
1067                 siw_dbg_qp(qp, "error %d\n", rv);
1068                 *bad_wr = wr;
1069         }
1070         return rv > 0 ? 0 : rv;
1071 }
1072
1073 int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1074 {
1075         struct siw_cq *cq = to_siw_cq(base_cq);
1076         struct siw_device *sdev = to_siw_dev(base_cq->device);
1077         struct siw_ucontext *ctx =
1078                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1079                                           base_ucontext);
1080
1081         siw_dbg_cq(cq, "free CQ resources\n");
1082
1083         siw_cq_flush(cq);
1084
1085         if (ctx)
1086                 rdma_user_mmap_entry_remove(cq->cq_entry);
1087
1088         atomic_dec(&sdev->num_cq);
1089
1090         vfree(cq->queue);
1091         return 0;
1092 }
1093
1094 /*
1095  * siw_create_cq()
1096  *
1097  * Populate CQ of requested size
1098  *
1099  * @base_cq: CQ as allocated by RDMA midlayer
1100  * @attr: Initial CQ attributes
1101  * @udata: relates to user context
1102  */
1103
1104 int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1105                   struct ib_udata *udata)
1106 {
1107         struct siw_device *sdev = to_siw_dev(base_cq->device);
1108         struct siw_cq *cq = to_siw_cq(base_cq);
1109         int rv, size = attr->cqe;
1110
1111         if (attr->flags)
1112                 return -EOPNOTSUPP;
1113
1114         if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
1115                 siw_dbg(base_cq->device, "too many CQ's\n");
1116                 rv = -ENOMEM;
1117                 goto err_out;
1118         }
1119         if (size < 1 || size > sdev->attrs.max_cqe) {
1120                 siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1121                 rv = -EINVAL;
1122                 goto err_out;
1123         }
1124         size = roundup_pow_of_two(size);
1125         cq->base_cq.cqe = size;
1126         cq->num_cqe = size;
1127
1128         if (udata)
1129                 cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1130                                          sizeof(struct siw_cq_ctrl));
1131         else
1132                 cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1133                                     sizeof(struct siw_cq_ctrl));
1134
1135         if (cq->queue == NULL) {
1136                 rv = -ENOMEM;
1137                 goto err_out;
1138         }
1139         get_random_bytes(&cq->id, 4);
1140         siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1141
1142         spin_lock_init(&cq->lock);
1143
1144         cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1145
1146         if (udata) {
1147                 struct siw_uresp_create_cq uresp = {};
1148                 struct siw_ucontext *ctx =
1149                         rdma_udata_to_drv_context(udata, struct siw_ucontext,
1150                                                   base_ucontext);
1151                 size_t length = size * sizeof(struct siw_cqe) +
1152                         sizeof(struct siw_cq_ctrl);
1153
1154                 cq->cq_entry =
1155                         siw_mmap_entry_insert(ctx, cq->queue,
1156                                               length, &uresp.cq_key);
1157                 if (!cq->cq_entry) {
1158                         rv = -ENOMEM;
1159                         goto err_out;
1160                 }
1161
1162                 uresp.cq_id = cq->id;
1163                 uresp.num_cqe = size;
1164
1165                 if (udata->outlen < sizeof(uresp)) {
1166                         rv = -EINVAL;
1167                         goto err_out;
1168                 }
1169                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1170                 if (rv)
1171                         goto err_out;
1172         }
1173         return 0;
1174
1175 err_out:
1176         siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1177
1178         if (cq && cq->queue) {
1179                 struct siw_ucontext *ctx =
1180                         rdma_udata_to_drv_context(udata, struct siw_ucontext,
1181                                                   base_ucontext);
1182                 if (ctx)
1183                         rdma_user_mmap_entry_remove(cq->cq_entry);
1184                 vfree(cq->queue);
1185         }
1186         atomic_dec(&sdev->num_cq);
1187
1188         return rv;
1189 }
1190
1191 /*
1192  * siw_poll_cq()
1193  *
1194  * Reap CQ entries if available and copy work completion status into
1195  * array of WC's provided by caller. Returns number of reaped CQE's.
1196  *
1197  * @base_cq:    Base CQ contained in siw CQ.
1198  * @num_cqe:    Maximum number of CQE's to reap.
1199  * @wc:         Array of work completions to be filled by siw.
1200  */
1201 int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1202 {
1203         struct siw_cq *cq = to_siw_cq(base_cq);
1204         int i;
1205
1206         for (i = 0; i < num_cqe; i++) {
1207                 if (!siw_reap_cqe(cq, wc))
1208                         break;
1209                 wc++;
1210         }
1211         return i;
1212 }
1213
1214 /*
1215  * siw_req_notify_cq()
1216  *
1217  * Request notification for new CQE's added to that CQ.
1218  * Defined flags:
1219  * o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
1220  *   event if a WQE with notification flag set enters the CQ
1221  * o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
1222  *   event if a WQE enters the CQ.
1223  * o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
1224  *   number of not reaped CQE's regardless of its notification
1225  *   type and current or new CQ notification settings.
1226  *
1227  * @base_cq:    Base CQ contained in siw CQ.
1228  * @flags:      Requested notification flags.
1229  */
1230 int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1231 {
1232         struct siw_cq *cq = to_siw_cq(base_cq);
1233
1234         siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1235
1236         if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1237                 /*
1238                  * Enable CQ event for next solicited completion.
1239                  * and make it visible to all associated producers.
1240                  */
1241                 smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1242         else
1243                 /*
1244                  * Enable CQ event for any signalled completion.
1245                  * and make it visible to all associated producers.
1246                  */
1247                 smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1248
1249         if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1250                 return cq->cq_put - cq->cq_get;
1251
1252         return 0;
1253 }
1254
1255 /*
1256  * siw_dereg_mr()
1257  *
1258  * Release Memory Region.
1259  *
1260  * @base_mr: Base MR contained in siw MR.
1261  * @udata: points to user context, unused.
1262  */
1263 int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1264 {
1265         struct siw_mr *mr = to_siw_mr(base_mr);
1266         struct siw_device *sdev = to_siw_dev(base_mr->device);
1267
1268         siw_dbg_mem(mr->mem, "deregister MR\n");
1269
1270         atomic_dec(&sdev->num_mr);
1271
1272         siw_mr_drop_mem(mr);
1273         kfree_rcu(mr, rcu);
1274
1275         return 0;
1276 }
1277
1278 /*
1279  * siw_reg_user_mr()
1280  *
1281  * Register Memory Region.
1282  *
1283  * @pd:         Protection Domain
1284  * @start:      starting address of MR (virtual address)
1285  * @len:        len of MR
1286  * @rnic_va:    not used by siw
1287  * @rights:     MR access rights
1288  * @udata:      user buffer to communicate STag and Key.
1289  */
1290 struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1291                               u64 rnic_va, int rights, struct ib_udata *udata)
1292 {
1293         struct siw_mr *mr = NULL;
1294         struct siw_umem *umem = NULL;
1295         struct siw_ureq_reg_mr ureq;
1296         struct siw_device *sdev = to_siw_dev(pd->device);
1297
1298         unsigned long mem_limit = rlimit(RLIMIT_MEMLOCK);
1299         int rv;
1300
1301         siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n",
1302                    (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1303                    (unsigned long long)len);
1304
1305         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1306                 siw_dbg_pd(pd, "too many mr's\n");
1307                 rv = -ENOMEM;
1308                 goto err_out;
1309         }
1310         if (!len) {
1311                 rv = -EINVAL;
1312                 goto err_out;
1313         }
1314         if (mem_limit != RLIM_INFINITY) {
1315                 unsigned long num_pages =
1316                         (PAGE_ALIGN(len + (start & ~PAGE_MASK))) >> PAGE_SHIFT;
1317                 mem_limit >>= PAGE_SHIFT;
1318
1319                 if (num_pages > mem_limit - current->mm->locked_vm) {
1320                         siw_dbg_pd(pd, "pages req %lu, max %lu, lock %lu\n",
1321                                    num_pages, mem_limit,
1322                                    current->mm->locked_vm);
1323                         rv = -ENOMEM;
1324                         goto err_out;
1325                 }
1326         }
1327         umem = siw_umem_get(start, len, ib_access_writable(rights));
1328         if (IS_ERR(umem)) {
1329                 rv = PTR_ERR(umem);
1330                 siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1331                 umem = NULL;
1332                 goto err_out;
1333         }
1334         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1335         if (!mr) {
1336                 rv = -ENOMEM;
1337                 goto err_out;
1338         }
1339         rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
1340         if (rv)
1341                 goto err_out;
1342
1343         if (udata) {
1344                 struct siw_uresp_reg_mr uresp = {};
1345                 struct siw_mem *mem = mr->mem;
1346
1347                 if (udata->inlen < sizeof(ureq)) {
1348                         rv = -EINVAL;
1349                         goto err_out;
1350                 }
1351                 rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1352                 if (rv)
1353                         goto err_out;
1354
1355                 mr->base_mr.lkey |= ureq.stag_key;
1356                 mr->base_mr.rkey |= ureq.stag_key;
1357                 mem->stag |= ureq.stag_key;
1358                 uresp.stag = mem->stag;
1359
1360                 if (udata->outlen < sizeof(uresp)) {
1361                         rv = -EINVAL;
1362                         goto err_out;
1363                 }
1364                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1365                 if (rv)
1366                         goto err_out;
1367         }
1368         mr->mem->stag_valid = 1;
1369
1370         return &mr->base_mr;
1371
1372 err_out:
1373         atomic_dec(&sdev->num_mr);
1374         if (mr) {
1375                 if (mr->mem)
1376                         siw_mr_drop_mem(mr);
1377                 kfree_rcu(mr, rcu);
1378         } else {
1379                 if (umem)
1380                         siw_umem_release(umem, false);
1381         }
1382         return ERR_PTR(rv);
1383 }
1384
1385 struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1386                            u32 max_sge)
1387 {
1388         struct siw_device *sdev = to_siw_dev(pd->device);
1389         struct siw_mr *mr = NULL;
1390         struct siw_pbl *pbl = NULL;
1391         int rv;
1392
1393         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1394                 siw_dbg_pd(pd, "too many mr's\n");
1395                 rv = -ENOMEM;
1396                 goto err_out;
1397         }
1398         if (mr_type != IB_MR_TYPE_MEM_REG) {
1399                 siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1400                 rv = -EOPNOTSUPP;
1401                 goto err_out;
1402         }
1403         if (max_sge > SIW_MAX_SGE_PBL) {
1404                 siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1405                 rv = -ENOMEM;
1406                 goto err_out;
1407         }
1408         pbl = siw_pbl_alloc(max_sge);
1409         if (IS_ERR(pbl)) {
1410                 rv = PTR_ERR(pbl);
1411                 siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1412                 pbl = NULL;
1413                 goto err_out;
1414         }
1415         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1416         if (!mr) {
1417                 rv = -ENOMEM;
1418                 goto err_out;
1419         }
1420         rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
1421         if (rv)
1422                 goto err_out;
1423
1424         mr->mem->is_pbl = 1;
1425
1426         siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1427
1428         return &mr->base_mr;
1429
1430 err_out:
1431         atomic_dec(&sdev->num_mr);
1432
1433         if (!mr) {
1434                 kfree(pbl);
1435         } else {
1436                 if (mr->mem)
1437                         siw_mr_drop_mem(mr);
1438                 kfree_rcu(mr, rcu);
1439         }
1440         siw_dbg_pd(pd, "failed: %d\n", rv);
1441
1442         return ERR_PTR(rv);
1443 }
1444
1445 /* Just used to count number of pages being mapped */
1446 static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1447 {
1448         return 0;
1449 }
1450
1451 int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1452                   unsigned int *sg_off)
1453 {
1454         struct scatterlist *slp;
1455         struct siw_mr *mr = to_siw_mr(base_mr);
1456         struct siw_mem *mem = mr->mem;
1457         struct siw_pbl *pbl = mem->pbl;
1458         struct siw_pble *pble;
1459         unsigned long pbl_size;
1460         int i, rv;
1461
1462         if (!pbl) {
1463                 siw_dbg_mem(mem, "no PBL allocated\n");
1464                 return -EINVAL;
1465         }
1466         pble = pbl->pbe;
1467
1468         if (pbl->max_buf < num_sle) {
1469                 siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1470                             mem->pbl->max_buf, num_sle);
1471                 return -ENOMEM;
1472         }
1473         for_each_sg(sl, slp, num_sle, i) {
1474                 if (sg_dma_len(slp) == 0) {
1475                         siw_dbg_mem(mem, "empty SGE\n");
1476                         return -EINVAL;
1477                 }
1478                 if (i == 0) {
1479                         pble->addr = sg_dma_address(slp);
1480                         pble->size = sg_dma_len(slp);
1481                         pble->pbl_off = 0;
1482                         pbl_size = pble->size;
1483                         pbl->num_buf = 1;
1484                 } else {
1485                         /* Merge PBL entries if adjacent */
1486                         if (pble->addr + pble->size == sg_dma_address(slp)) {
1487                                 pble->size += sg_dma_len(slp);
1488                         } else {
1489                                 pble++;
1490                                 pbl->num_buf++;
1491                                 pble->addr = sg_dma_address(slp);
1492                                 pble->size = sg_dma_len(slp);
1493                                 pble->pbl_off = pbl_size;
1494                         }
1495                         pbl_size += sg_dma_len(slp);
1496                 }
1497                 siw_dbg_mem(mem,
1498                         "sge[%d], size %u, addr 0x%p, total %lu\n",
1499                         i, pble->size, (void *)(uintptr_t)pble->addr,
1500                         pbl_size);
1501         }
1502         rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
1503         if (rv > 0) {
1504                 mem->len = base_mr->length;
1505                 mem->va = base_mr->iova;
1506                 siw_dbg_mem(mem,
1507                         "%llu bytes, start 0x%pK, %u SLE to %u entries\n",
1508                         mem->len, (void *)(uintptr_t)mem->va, num_sle,
1509                         pbl->num_buf);
1510         }
1511         return rv;
1512 }
1513
1514 /*
1515  * siw_get_dma_mr()
1516  *
1517  * Create a (empty) DMA memory region, where no umem is attached.
1518  */
1519 struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1520 {
1521         struct siw_device *sdev = to_siw_dev(pd->device);
1522         struct siw_mr *mr = NULL;
1523         int rv;
1524
1525         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1526                 siw_dbg_pd(pd, "too many mr's\n");
1527                 rv = -ENOMEM;
1528                 goto err_out;
1529         }
1530         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1531         if (!mr) {
1532                 rv = -ENOMEM;
1533                 goto err_out;
1534         }
1535         rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
1536         if (rv)
1537                 goto err_out;
1538
1539         mr->mem->stag_valid = 1;
1540
1541         siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1542
1543         return &mr->base_mr;
1544
1545 err_out:
1546         if (rv)
1547                 kfree(mr);
1548
1549         atomic_dec(&sdev->num_mr);
1550
1551         return ERR_PTR(rv);
1552 }
1553
1554 /*
1555  * siw_create_srq()
1556  *
1557  * Create Shared Receive Queue of attributes @init_attrs
1558  * within protection domain given by @pd.
1559  *
1560  * @base_srq:   Base SRQ contained in siw SRQ.
1561  * @init_attrs: SRQ init attributes.
1562  * @udata:      points to user context
1563  */
1564 int siw_create_srq(struct ib_srq *base_srq,
1565                    struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1566 {
1567         struct siw_srq *srq = to_siw_srq(base_srq);
1568         struct ib_srq_attr *attrs = &init_attrs->attr;
1569         struct siw_device *sdev = to_siw_dev(base_srq->device);
1570         struct siw_ucontext *ctx =
1571                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1572                                           base_ucontext);
1573         int rv;
1574
1575         if (init_attrs->srq_type != IB_SRQT_BASIC)
1576                 return -EOPNOTSUPP;
1577
1578         if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
1579                 siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1580                 rv = -ENOMEM;
1581                 goto err_out;
1582         }
1583         if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1584             attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1585                 rv = -EINVAL;
1586                 goto err_out;
1587         }
1588         srq->max_sge = attrs->max_sge;
1589         srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1590         srq->limit = attrs->srq_limit;
1591         if (srq->limit)
1592                 srq->armed = true;
1593
1594         srq->is_kernel_res = !udata;
1595
1596         if (udata)
1597                 srq->recvq =
1598                         vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1599         else
1600                 srq->recvq = vzalloc(srq->num_rqe * sizeof(struct siw_rqe));
1601
1602         if (srq->recvq == NULL) {
1603                 rv = -ENOMEM;
1604                 goto err_out;
1605         }
1606         if (udata) {
1607                 struct siw_uresp_create_srq uresp = {};
1608                 size_t length = srq->num_rqe * sizeof(struct siw_rqe);
1609
1610                 srq->srq_entry =
1611                         siw_mmap_entry_insert(ctx, srq->recvq,
1612                                               length, &uresp.srq_key);
1613                 if (!srq->srq_entry) {
1614                         rv = -ENOMEM;
1615                         goto err_out;
1616                 }
1617
1618                 uresp.num_rqe = srq->num_rqe;
1619
1620                 if (udata->outlen < sizeof(uresp)) {
1621                         rv = -EINVAL;
1622                         goto err_out;
1623                 }
1624                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1625                 if (rv)
1626                         goto err_out;
1627         }
1628         spin_lock_init(&srq->lock);
1629
1630         siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1631
1632         return 0;
1633
1634 err_out:
1635         if (srq->recvq) {
1636                 if (ctx)
1637                         rdma_user_mmap_entry_remove(srq->srq_entry);
1638                 vfree(srq->recvq);
1639         }
1640         atomic_dec(&sdev->num_srq);
1641
1642         return rv;
1643 }
1644
1645 /*
1646  * siw_modify_srq()
1647  *
1648  * Modify SRQ. The caller may resize SRQ and/or set/reset notification
1649  * limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
1650  *
1651  * NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
1652  * parameter. siw_modify_srq() does not check the attrs->max_sge param.
1653  */
1654 int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1655                    enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1656 {
1657         struct siw_srq *srq = to_siw_srq(base_srq);
1658         unsigned long flags;
1659         int rv = 0;
1660
1661         spin_lock_irqsave(&srq->lock, flags);
1662
1663         if (attr_mask & IB_SRQ_MAX_WR) {
1664                 /* resize request not yet supported */
1665                 rv = -EOPNOTSUPP;
1666                 goto out;
1667         }
1668         if (attr_mask & IB_SRQ_LIMIT) {
1669                 if (attrs->srq_limit) {
1670                         if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1671                                 rv = -EINVAL;
1672                                 goto out;
1673                         }
1674                         srq->armed = true;
1675                 } else {
1676                         srq->armed = false;
1677                 }
1678                 srq->limit = attrs->srq_limit;
1679         }
1680 out:
1681         spin_unlock_irqrestore(&srq->lock, flags);
1682
1683         return rv;
1684 }
1685
1686 /*
1687  * siw_query_srq()
1688  *
1689  * Query SRQ attributes.
1690  */
1691 int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1692 {
1693         struct siw_srq *srq = to_siw_srq(base_srq);
1694         unsigned long flags;
1695
1696         spin_lock_irqsave(&srq->lock, flags);
1697
1698         attrs->max_wr = srq->num_rqe;
1699         attrs->max_sge = srq->max_sge;
1700         attrs->srq_limit = srq->limit;
1701
1702         spin_unlock_irqrestore(&srq->lock, flags);
1703
1704         return 0;
1705 }
1706
1707 /*
1708  * siw_destroy_srq()
1709  *
1710  * Destroy SRQ.
1711  * It is assumed that the SRQ is not referenced by any
1712  * QP anymore - the code trusts the RDMA core environment to keep track
1713  * of QP references.
1714  */
1715 int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1716 {
1717         struct siw_srq *srq = to_siw_srq(base_srq);
1718         struct siw_device *sdev = to_siw_dev(base_srq->device);
1719         struct siw_ucontext *ctx =
1720                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1721                                           base_ucontext);
1722
1723         if (ctx)
1724                 rdma_user_mmap_entry_remove(srq->srq_entry);
1725         vfree(srq->recvq);
1726         atomic_dec(&sdev->num_srq);
1727         return 0;
1728 }
1729
1730 /*
1731  * siw_post_srq_recv()
1732  *
1733  * Post a list of receive queue elements to SRQ.
1734  * NOTE: The function does not check or lock a certain SRQ state
1735  *       during the post operation. The code simply trusts the
1736  *       RDMA core environment.
1737  *
1738  * @base_srq:   Base SRQ contained in siw SRQ
1739  * @wr:         List of R-WR's
1740  * @bad_wr:     Updated to failing WR if posting fails.
1741  */
1742 int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1743                       const struct ib_recv_wr **bad_wr)
1744 {
1745         struct siw_srq *srq = to_siw_srq(base_srq);
1746         unsigned long flags;
1747         int rv = 0;
1748
1749         if (unlikely(!srq->is_kernel_res)) {
1750                 siw_dbg_pd(base_srq->pd,
1751                            "[SRQ]: no kernel post_recv for mapped srq\n");
1752                 rv = -EINVAL;
1753                 goto out;
1754         }
1755         /*
1756          * Serialize potentially multiple producers.
1757          * Also needed to serialize potentially multiple
1758          * consumers.
1759          */
1760         spin_lock_irqsave(&srq->lock, flags);
1761
1762         while (wr) {
1763                 u32 idx = srq->rq_put % srq->num_rqe;
1764                 struct siw_rqe *rqe = &srq->recvq[idx];
1765
1766                 if (rqe->flags) {
1767                         siw_dbg_pd(base_srq->pd, "SRQ full\n");
1768                         rv = -ENOMEM;
1769                         break;
1770                 }
1771                 if (unlikely(wr->num_sge > srq->max_sge)) {
1772                         siw_dbg_pd(base_srq->pd,
1773                                    "[SRQ]: too many sge's: %d\n", wr->num_sge);
1774                         rv = -EINVAL;
1775                         break;
1776                 }
1777                 rqe->id = wr->wr_id;
1778                 rqe->num_sge = wr->num_sge;
1779                 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1780
1781                 /* Make sure S-RQE is completely written before valid */
1782                 smp_wmb();
1783
1784                 rqe->flags = SIW_WQE_VALID;
1785
1786                 srq->rq_put++;
1787                 wr = wr->next;
1788         }
1789         spin_unlock_irqrestore(&srq->lock, flags);
1790 out:
1791         if (unlikely(rv < 0)) {
1792                 siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1793                 *bad_wr = wr;
1794         }
1795         return rv;
1796 }
1797
1798 void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1799 {
1800         struct ib_event event;
1801         struct ib_qp *base_qp = &qp->base_qp;
1802
1803         /*
1804          * Do not report asynchronous errors on QP which gets
1805          * destroyed via verbs interface (siw_destroy_qp())
1806          */
1807         if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1808                 return;
1809
1810         event.event = etype;
1811         event.device = base_qp->device;
1812         event.element.qp = base_qp;
1813
1814         if (base_qp->event_handler) {
1815                 siw_dbg_qp(qp, "reporting event %d\n", etype);
1816                 base_qp->event_handler(&event, base_qp->qp_context);
1817         }
1818 }
1819
1820 void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1821 {
1822         struct ib_event event;
1823         struct ib_cq *base_cq = &cq->base_cq;
1824
1825         event.event = etype;
1826         event.device = base_cq->device;
1827         event.element.cq = base_cq;
1828
1829         if (base_cq->event_handler) {
1830                 siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1831                 base_cq->event_handler(&event, base_cq->cq_context);
1832         }
1833 }
1834
1835 void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1836 {
1837         struct ib_event event;
1838         struct ib_srq *base_srq = &srq->base_srq;
1839
1840         event.event = etype;
1841         event.device = base_srq->device;
1842         event.element.srq = base_srq;
1843
1844         if (base_srq->event_handler) {
1845                 siw_dbg_pd(srq->base_srq.pd,
1846                            "reporting SRQ event %d\n", etype);
1847                 base_srq->event_handler(&event, base_srq->srq_context);
1848         }
1849 }
1850
1851 void siw_port_event(struct siw_device *sdev, u32 port, enum ib_event_type etype)
1852 {
1853         struct ib_event event;
1854
1855         event.event = etype;
1856         event.device = &sdev->base_dev;
1857         event.element.port_num = port;
1858
1859         siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1860
1861         ib_dispatch_event(&event);
1862 }