drivers/infiniband/hw/cxgb3/iwch_provider.c

   1 /*
   2  * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/device.h>
  35 #include <linux/netdevice.h>
  36 #include <linux/etherdevice.h>
  37 #include <linux/delay.h>
  38 #include <linux/errno.h>
  39 #include <linux/list.h>
  40 #include <linux/sched/mm.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/ethtool.h>
  43 #include <linux/rtnetlink.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/slab.h>
  46
  47 #include <asm/io.h>
  48 #include <asm/irq.h>
  49 #include <asm/byteorder.h>
  50
  51 #include <rdma/iw_cm.h>
  52 #include <rdma/ib_verbs.h>
  53 #include <rdma/ib_smi.h>
  54 #include <rdma/ib_umem.h>
  55 #include <rdma/ib_user_verbs.h>
  56 #include <rdma/uverbs_ioctl.h>
  57
  58 #include "cxio_hal.h"
  59 #include "iwch.h"
  60 #include "iwch_provider.h"
  61 #include "iwch_cm.h"
  62 #include <rdma/cxgb3-abi.h>
  63 #include "common.h"
  64
  65 static void iwch_dealloc_ucontext(struct ib_ucontext *context)
  66 {
  67         struct iwch_dev *rhp = to_iwch_dev(context->device);
  68         struct iwch_ucontext *ucontext = to_iwch_ucontext(context);
  69         struct iwch_mm_entry *mm, *tmp;
  70
  71         pr_debug("%s context %p\n", __func__, context);
  72         list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)
  73                 kfree(mm);
  74         cxio_release_ucontext(&rhp->rdev, &ucontext->uctx);
  75 }
  76
  77 static int iwch_alloc_ucontext(struct ib_ucontext *ucontext,
  78                                struct ib_udata *udata)
  79 {
  80         struct ib_device *ibdev = ucontext->device;
  81         struct iwch_ucontext *context = to_iwch_ucontext(ucontext);
  82         struct iwch_dev *rhp = to_iwch_dev(ibdev);
  83
  84         pr_debug("%s ibdev %p\n", __func__, ibdev);
  85         cxio_init_ucontext(&rhp->rdev, &context->uctx);
  86         INIT_LIST_HEAD(&context->mmaps);
  87         spin_lock_init(&context->mmap_lock);
  88         return 0;
  89 }
  90
  91 static int iwch_destroy_cq(struct ib_cq *ib_cq)
  92 {
  93         struct iwch_cq *chp;
  94
  95         pr_debug("%s ib_cq %p\n", __func__, ib_cq);
  96         chp = to_iwch_cq(ib_cq);
  97
  98         remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
  99         atomic_dec(&chp->refcnt);
 100         wait_event(chp->wait, !atomic_read(&chp->refcnt));
 101
 102         cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
 103         kfree(chp);
 104         return 0;
 105 }
 106
 107 static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
 108                                     const struct ib_cq_init_attr *attr,
 109                                     struct ib_ucontext *ib_context,
 110                                     struct ib_udata *udata)
 111 {
 112         int entries = attr->cqe;
 113         struct iwch_dev *rhp;
 114         struct iwch_cq *chp;
 115         struct iwch_create_cq_resp uresp;
 116         struct iwch_create_cq_req ureq;
 117         struct iwch_ucontext *ucontext = NULL;
 118         static int warned;
 119         size_t resplen;
 120
 121         pr_debug("%s ib_dev %p entries %d\n", __func__, ibdev, entries);
 122         if (attr->flags)
 123                 return ERR_PTR(-EINVAL);
 124
 125         rhp = to_iwch_dev(ibdev);
 126         chp = kzalloc(sizeof(*chp), GFP_KERNEL);
 127         if (!chp)
 128                 return ERR_PTR(-ENOMEM);
 129
 130         if (ib_context) {
 131                 ucontext = to_iwch_ucontext(ib_context);
 132                 if (!t3a_device(rhp)) {
 133                         if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
 134                                 kfree(chp);
 135                                 return ERR_PTR(-EFAULT);
 136                         }
 137                         chp->user_rptr_addr = (u32 __user *)(unsigned long)ureq.user_rptr_addr;
 138                 }
 139         }
 140
 141         if (t3a_device(rhp)) {
 142
 143                 /*
 144                  * T3A: Add some fluff to handle extra CQEs inserted
 145                  * for various errors.
 146                  * Additional CQE possibilities:
 147                  *      TERMINATE,
 148                  *      incoming RDMA WRITE Failures
 149                  *      incoming RDMA READ REQUEST FAILUREs
 150                  * NOTE: We cannot ensure the CQ won't overflow.
 151                  */
 152                 entries += 16;
 153         }
 154         entries = roundup_pow_of_two(entries);
 155         chp->cq.size_log2 = ilog2(entries);
 156
 157         if (cxio_create_cq(&rhp->rdev, &chp->cq, !ucontext)) {
 158                 kfree(chp);
 159                 return ERR_PTR(-ENOMEM);
 160         }
 161         chp->rhp = rhp;
 162         chp->ibcq.cqe = 1 << chp->cq.size_log2;
 163         spin_lock_init(&chp->lock);
 164         spin_lock_init(&chp->comp_handler_lock);
 165         atomic_set(&chp->refcnt, 1);
 166         init_waitqueue_head(&chp->wait);
 167         if (insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid)) {
 168                 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
 169                 kfree(chp);
 170                 return ERR_PTR(-ENOMEM);
 171         }
 172
 173         if (ucontext) {
 174                 struct iwch_mm_entry *mm;
 175
 176                 mm = kmalloc(sizeof *mm, GFP_KERNEL);
 177                 if (!mm) {
 178                         iwch_destroy_cq(&chp->ibcq);
 179                         return ERR_PTR(-ENOMEM);
 180                 }
 181                 uresp.cqid = chp->cq.cqid;
 182                 uresp.size_log2 = chp->cq.size_log2;
 183                 spin_lock(&ucontext->mmap_lock);
 184                 uresp.key = ucontext->key;
 185                 ucontext->key += PAGE_SIZE;
 186                 spin_unlock(&ucontext->mmap_lock);
 187                 mm->key = uresp.key;
 188                 mm->addr = virt_to_phys(chp->cq.queue);
 189                 if (udata->outlen < sizeof uresp) {
 190                         if (!warned++)
 191                                 pr_warn("Warning - downlevel libcxgb3 (non-fatal)\n");
 192                         mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
 193                                              sizeof(struct t3_cqe));
 194                         resplen = sizeof(struct iwch_create_cq_resp_v0);
 195                 } else {
 196                         mm->len = PAGE_ALIGN(((1UL << uresp.size_log2) + 1) *
 197                                              sizeof(struct t3_cqe));
 198                         uresp.memsize = mm->len;
 199                         uresp.reserved = 0;
 200                         resplen = sizeof uresp;
 201                 }
 202                 if (ib_copy_to_udata(udata, &uresp, resplen)) {
 203                         kfree(mm);
 204                         iwch_destroy_cq(&chp->ibcq);
 205                         return ERR_PTR(-EFAULT);
 206                 }
 207                 insert_mmap(ucontext, mm);
 208         }
 209         pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n",
 210                  chp->cq.cqid, chp, (1 << chp->cq.size_log2),
 211                  (unsigned long long)chp->cq.dma_addr);
 212         return &chp->ibcq;
 213 }
 214
 215 static int iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
 216 {
 217 #ifdef notyet
 218         struct iwch_cq *chp = to_iwch_cq(cq);
 219         struct t3_cq oldcq, newcq;
 220         int ret;
 221
 222         pr_debug("%s ib_cq %p cqe %d\n", __func__, cq, cqe);
 223
 224         /* We don't downsize... */
 225         if (cqe <= cq->cqe)
 226                 return 0;
 227
 228         /* create new t3_cq with new size */
 229         cqe = roundup_pow_of_two(cqe+1);
 230         newcq.size_log2 = ilog2(cqe);
 231
 232         /* Dont allow resize to less than the current wce count */
 233         if (cqe < Q_COUNT(chp->cq.rptr, chp->cq.wptr)) {
 234                 return -ENOMEM;
 235         }
 236
 237         /* Quiesce all QPs using this CQ */
 238         ret = iwch_quiesce_qps(chp);
 239         if (ret) {
 240                 return ret;
 241         }
 242
 243         ret = cxio_create_cq(&chp->rhp->rdev, &newcq);
 244         if (ret) {
 245                 return ret;
 246         }
 247
 248         /* copy CQEs */
 249         memcpy(newcq.queue, chp->cq.queue, (1 << chp->cq.size_log2) *
 250                                         sizeof(struct t3_cqe));
 251
 252         /* old iwch_qp gets new t3_cq but keeps old cqid */
 253         oldcq = chp->cq;
 254         chp->cq = newcq;
 255         chp->cq.cqid = oldcq.cqid;
 256
 257         /* resize new t3_cq to update the HW context */
 258         ret = cxio_resize_cq(&chp->rhp->rdev, &chp->cq);
 259         if (ret) {
 260                 chp->cq = oldcq;
 261                 return ret;
 262         }
 263         chp->ibcq.cqe = (1<<chp->cq.size_log2) - 1;
 264
 265         /* destroy old t3_cq */
 266         oldcq.cqid = newcq.cqid;
 267         ret = cxio_destroy_cq(&chp->rhp->rdev, &oldcq);
 268         if (ret) {
 269                 pr_err("%s - cxio_destroy_cq failed %d\n", __func__, ret);
 270         }
 271
 272         /* add user hooks here */
 273
 274         /* resume qps */
 275         ret = iwch_resume_qps(chp);
 276         return ret;
 277 #else
 278         return -ENOSYS;
 279 #endif
 280 }
 281
 282 static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 283 {
 284         struct iwch_dev *rhp;
 285         struct iwch_cq *chp;
 286         enum t3_cq_opcode cq_op;
 287         int err;
 288         unsigned long flag;
 289         u32 rptr;
 290
 291         chp = to_iwch_cq(ibcq);
 292         rhp = chp->rhp;
 293         if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
 294                 cq_op = CQ_ARM_SE;
 295         else
 296                 cq_op = CQ_ARM_AN;
 297         if (chp->user_rptr_addr) {
 298                 if (get_user(rptr, chp->user_rptr_addr))
 299                         return -EFAULT;
 300                 spin_lock_irqsave(&chp->lock, flag);
 301                 chp->cq.rptr = rptr;
 302         } else
 303                 spin_lock_irqsave(&chp->lock, flag);
 304         pr_debug("%s rptr 0x%x\n", __func__, chp->cq.rptr);
 305         err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0);
 306         spin_unlock_irqrestore(&chp->lock, flag);
 307         if (err < 0)
 308                 pr_err("Error %d rearming CQID 0x%x\n", err, chp->cq.cqid);
 309         if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
 310                 err = 0;
 311         return err;
 312 }
 313
 314 static int iwch_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 315 {
 316         int len = vma->vm_end - vma->vm_start;
 317         u32 key = vma->vm_pgoff << PAGE_SHIFT;
 318         struct cxio_rdev *rdev_p;
 319         int ret = 0;
 320         struct iwch_mm_entry *mm;
 321         struct iwch_ucontext *ucontext;
 322         u64 addr;
 323
 324         pr_debug("%s pgoff 0x%lx key 0x%x len %d\n", __func__, vma->vm_pgoff,
 325                  key, len);
 326
 327         if (vma->vm_start & (PAGE_SIZE-1)) {
 328                 return -EINVAL;
 329         }
 330
 331         rdev_p = &(to_iwch_dev(context->device)->rdev);
 332         ucontext = to_iwch_ucontext(context);
 333
 334         mm = remove_mmap(ucontext, key, len);
 335         if (!mm)
 336                 return -EINVAL;
 337         addr = mm->addr;
 338         kfree(mm);
 339
 340         if ((addr >= rdev_p->rnic_info.udbell_physbase) &&
 341             (addr < (rdev_p->rnic_info.udbell_physbase +
 342                        rdev_p->rnic_info.udbell_len))) {
 343
 344                 /*
 345                  * Map T3 DB register.
 346                  */
 347                 if (vma->vm_flags & VM_READ) {
 348                         return -EPERM;
 349                 }
 350
 351                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 352                 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
 353                 vma->vm_flags &= ~VM_MAYREAD;
 354                 ret = io_remap_pfn_range(vma, vma->vm_start,
 355                                          addr >> PAGE_SHIFT,
 356                                          len, vma->vm_page_prot);
 357         } else {
 358
 359                 /*
 360                  * Map WQ or CQ contig dma memory...
 361                  */
 362                 ret = remap_pfn_range(vma, vma->vm_start,
 363                                       addr >> PAGE_SHIFT,
 364                                       len, vma->vm_page_prot);
 365         }
 366
 367         return ret;
 368 }
 369
 370 static void iwch_deallocate_pd(struct ib_pd *pd)
 371 {
 372         struct iwch_dev *rhp;
 373         struct iwch_pd *php;
 374
 375         php = to_iwch_pd(pd);
 376         rhp = php->rhp;
 377         pr_debug("%s ibpd %p pdid 0x%x\n", __func__, pd, php->pdid);
 378         cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
 379 }
 380
 381 static int iwch_allocate_pd(struct ib_pd *pd, struct ib_ucontext *context,
 382                             struct ib_udata *udata)
 383 {
 384         struct iwch_pd *php = to_iwch_pd(pd);
 385         struct ib_device *ibdev = pd->device;
 386         u32 pdid;
 387         struct iwch_dev *rhp;
 388
 389         pr_debug("%s ibdev %p\n", __func__, ibdev);
 390         rhp = (struct iwch_dev *) ibdev;
 391         pdid = cxio_hal_get_pdid(rhp->rdev.rscp);
 392         if (!pdid)
 393                 return -EINVAL;
 394
 395         php->pdid = pdid;
 396         php->rhp = rhp;
 397         if (context) {
 398                 struct iwch_alloc_pd_resp resp = {.pdid = php->pdid};
 399
 400                 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
 401                         iwch_deallocate_pd(&php->ibpd);
 402                         return -EFAULT;
 403                 }
 404         }
 405         pr_debug("%s pdid 0x%0x ptr 0x%p\n", __func__, pdid, php);
 406         return 0;
 407 }
 408
 409 static int iwch_dereg_mr(struct ib_mr *ib_mr)
 410 {
 411         struct iwch_dev *rhp;
 412         struct iwch_mr *mhp;
 413         u32 mmid;
 414
 415         pr_debug("%s ib_mr %p\n", __func__, ib_mr);
 416
 417         mhp = to_iwch_mr(ib_mr);
 418         kfree(mhp->pages);
 419         rhp = mhp->rhp;
 420         mmid = mhp->attr.stag >> 8;
 421         cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
 422                        mhp->attr.pbl_addr);
 423         iwch_free_pbl(mhp);
 424         remove_handle(rhp, &rhp->mmidr, mmid);
 425         if (mhp->kva)
 426                 kfree((void *) (unsigned long) mhp->kva);
 427         if (mhp->umem)
 428                 ib_umem_release(mhp->umem);
 429         pr_debug("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp);
 430         kfree(mhp);
 431         return 0;
 432 }
 433
 434 static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
 435 {
 436         const u64 total_size = 0xffffffff;
 437         const u64 mask = (total_size + PAGE_SIZE - 1) & PAGE_MASK;
 438         struct iwch_pd *php = to_iwch_pd(pd);
 439         struct iwch_dev *rhp = php->rhp;
 440         struct iwch_mr *mhp;
 441         __be64 *page_list;
 442         int shift = 26, npages, ret, i;
 443
 444         pr_debug("%s ib_pd %p\n", __func__, pd);
 445
 446         /*
 447          * T3 only supports 32 bits of size.
 448          */
 449         if (sizeof(phys_addr_t) > 4) {
 450                 pr_warn_once("Cannot support dma_mrs on this platform\n");
 451                 return ERR_PTR(-ENOTSUPP);
 452         }
 453
 454         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 455         if (!mhp)
 456                 return ERR_PTR(-ENOMEM);
 457
 458         mhp->rhp = rhp;
 459
 460         npages = (total_size + (1ULL << shift) - 1) >> shift;
 461         if (!npages) {
 462                 ret = -EINVAL;
 463                 goto err;
 464         }
 465
 466         page_list = kmalloc_array(npages, sizeof(u64), GFP_KERNEL);
 467         if (!page_list) {
 468                 ret = -ENOMEM;
 469                 goto err;
 470         }
 471
 472         for (i = 0; i < npages; i++)
 473                 page_list[i] = cpu_to_be64((u64)i << shift);
 474
 475         pr_debug("%s mask 0x%llx shift %d len %lld pbl_size %d\n",
 476                  __func__, mask, shift, total_size, npages);
 477
 478         ret = iwch_alloc_pbl(mhp, npages);
 479         if (ret) {
 480                 kfree(page_list);
 481                 goto err_pbl;
 482         }
 483
 484         ret = iwch_write_pbl(mhp, page_list, npages, 0);
 485         kfree(page_list);
 486         if (ret)
 487                 goto err_pbl;
 488
 489         mhp->attr.pdid = php->pdid;
 490         mhp->attr.zbva = 0;
 491
 492         mhp->attr.perms = iwch_ib_to_tpt_access(acc);
 493         mhp->attr.va_fbo = 0;
 494         mhp->attr.page_size = shift - 12;
 495
 496         mhp->attr.len = (u32) total_size;
 497         mhp->attr.pbl_size = npages;
 498         ret = iwch_register_mem(rhp, php, mhp, shift);
 499         if (ret)
 500                 goto err_pbl;
 501
 502         return &mhp->ibmr;
 503
 504 err_pbl:
 505         iwch_free_pbl(mhp);
 506
 507 err:
 508         kfree(mhp);
 509         return ERR_PTR(ret);
 510 }
 511
 512 static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 513                                       u64 virt, int acc, struct ib_udata *udata)
 514 {
 515         __be64 *pages;
 516         int shift, n, i;
 517         int err = 0;
 518         struct iwch_dev *rhp;
 519         struct iwch_pd *php;
 520         struct iwch_mr *mhp;
 521         struct iwch_reg_user_mr_resp uresp;
 522         struct sg_dma_page_iter sg_iter;
 523         pr_debug("%s ib_pd %p\n", __func__, pd);
 524
 525         php = to_iwch_pd(pd);
 526         rhp = php->rhp;
 527         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 528         if (!mhp)
 529                 return ERR_PTR(-ENOMEM);
 530
 531         mhp->rhp = rhp;
 532
 533         mhp->umem = ib_umem_get(udata, start, length, acc, 0);
 534         if (IS_ERR(mhp->umem)) {
 535                 err = PTR_ERR(mhp->umem);
 536                 kfree(mhp);
 537                 return ERR_PTR(err);
 538         }
 539
 540         shift = PAGE_SHIFT;
 541
 542         n = mhp->umem->nmap;
 543
 544         err = iwch_alloc_pbl(mhp, n);
 545         if (err)
 546                 goto err;
 547
 548         pages = (__be64 *) __get_free_page(GFP_KERNEL);
 549         if (!pages) {
 550                 err = -ENOMEM;
 551                 goto err_pbl;
 552         }
 553
 554         i = n = 0;
 555
 556         for_each_sg_dma_page(mhp->umem->sg_head.sgl, &sg_iter, mhp->umem->nmap, 0) {
 557                 pages[i++] = cpu_to_be64(sg_page_iter_dma_address(&sg_iter));
 558                 if (i == PAGE_SIZE / sizeof *pages) {
 559                         err = iwch_write_pbl(mhp, pages, i, n);
 560                         if (err)
 561                                 goto pbl_done;
 562                         n += i;
 563                         i = 0;
 564                 }
 565         }
 566
 567         if (i)
 568                 err = iwch_write_pbl(mhp, pages, i, n);
 569
 570 pbl_done:
 571         free_page((unsigned long) pages);
 572         if (err)
 573                 goto err_pbl;
 574
 575         mhp->attr.pdid = php->pdid;
 576         mhp->attr.zbva = 0;
 577         mhp->attr.perms = iwch_ib_to_tpt_access(acc);
 578         mhp->attr.va_fbo = virt;
 579         mhp->attr.page_size = shift - 12;
 580         mhp->attr.len = (u32) length;
 581
 582         err = iwch_register_mem(rhp, php, mhp, shift);
 583         if (err)
 584                 goto err_pbl;
 585
 586         if (udata && !t3a_device(rhp)) {
 587                 uresp.pbl_addr = (mhp->attr.pbl_addr -
 588                                  rhp->rdev.rnic_info.pbl_base) >> 3;
 589                 pr_debug("%s user resp pbl_addr 0x%x\n", __func__,
 590                          uresp.pbl_addr);
 591
 592                 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
 593                         iwch_dereg_mr(&mhp->ibmr);
 594                         err = -EFAULT;
 595                         goto err;
 596                 }
 597         }
 598
 599         return &mhp->ibmr;
 600
 601 err_pbl:
 602         iwch_free_pbl(mhp);
 603
 604 err:
 605         ib_umem_release(mhp->umem);
 606         kfree(mhp);
 607         return ERR_PTR(err);
 608 }
 609
 610 static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
 611                                    struct ib_udata *udata)
 612 {
 613         struct iwch_dev *rhp;
 614         struct iwch_pd *php;
 615         struct iwch_mw *mhp;
 616         u32 mmid;
 617         u32 stag = 0;
 618         int ret;
 619
 620         if (type != IB_MW_TYPE_1)
 621                 return ERR_PTR(-EINVAL);
 622
 623         php = to_iwch_pd(pd);
 624         rhp = php->rhp;
 625         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 626         if (!mhp)
 627                 return ERR_PTR(-ENOMEM);
 628         ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid);
 629         if (ret) {
 630                 kfree(mhp);
 631                 return ERR_PTR(ret);
 632         }
 633         mhp->rhp = rhp;
 634         mhp->attr.pdid = php->pdid;
 635         mhp->attr.type = TPT_MW;
 636         mhp->attr.stag = stag;
 637         mmid = (stag) >> 8;
 638         mhp->ibmw.rkey = stag;
 639         if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
 640                 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
 641                 kfree(mhp);
 642                 return ERR_PTR(-ENOMEM);
 643         }
 644         pr_debug("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
 645         return &(mhp->ibmw);
 646 }
 647
 648 static int iwch_dealloc_mw(struct ib_mw *mw)
 649 {
 650         struct iwch_dev *rhp;
 651         struct iwch_mw *mhp;
 652         u32 mmid;
 653
 654         mhp = to_iwch_mw(mw);
 655         rhp = mhp->rhp;
 656         mmid = (mw->rkey) >> 8;
 657         cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
 658         remove_handle(rhp, &rhp->mmidr, mmid);
 659         pr_debug("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);
 660         kfree(mhp);
 661         return 0;
 662 }
 663
 664 static struct ib_mr *iwch_alloc_mr(struct ib_pd *pd,
 665                                    enum ib_mr_type mr_type,
 666                                    u32 max_num_sg)
 667 {
 668         struct iwch_dev *rhp;
 669         struct iwch_pd *php;
 670         struct iwch_mr *mhp;
 671         u32 mmid;
 672         u32 stag = 0;
 673         int ret = -ENOMEM;
 674
 675         if (mr_type != IB_MR_TYPE_MEM_REG ||
 676             max_num_sg > T3_MAX_FASTREG_DEPTH)
 677                 return ERR_PTR(-EINVAL);
 678
 679         php = to_iwch_pd(pd);
 680         rhp = php->rhp;
 681         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 682         if (!mhp)
 683                 goto err;
 684
 685         mhp->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
 686         if (!mhp->pages)
 687                 goto pl_err;
 688
 689         mhp->rhp = rhp;
 690         ret = iwch_alloc_pbl(mhp, max_num_sg);
 691         if (ret)
 692                 goto err1;
 693         mhp->attr.pbl_size = max_num_sg;
 694         ret = cxio_allocate_stag(&rhp->rdev, &stag, php->pdid,
 695                                  mhp->attr.pbl_size, mhp->attr.pbl_addr);
 696         if (ret)
 697                 goto err2;
 698         mhp->attr.pdid = php->pdid;
 699         mhp->attr.type = TPT_NON_SHARED_MR;
 700         mhp->attr.stag = stag;
 701         mhp->attr.state = 1;
 702         mmid = (stag) >> 8;
 703         mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
 704         ret = insert_handle(rhp, &rhp->mmidr, mhp, mmid);
 705         if (ret)
 706                 goto err3;
 707
 708         pr_debug("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
 709         return &(mhp->ibmr);
 710 err3:
 711         cxio_dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size,
 712                        mhp->attr.pbl_addr);
 713 err2:
 714         iwch_free_pbl(mhp);
 715 err1:
 716         kfree(mhp->pages);
 717 pl_err:
 718         kfree(mhp);
 719 err:
 720         return ERR_PTR(ret);
 721 }
 722
 723 static int iwch_set_page(struct ib_mr *ibmr, u64 addr)
 724 {
 725         struct iwch_mr *mhp = to_iwch_mr(ibmr);
 726
 727         if (unlikely(mhp->npages == mhp->attr.pbl_size))
 728                 return -ENOMEM;
 729
 730         mhp->pages[mhp->npages++] = addr;
 731
 732         return 0;
 733 }
 734
 735 static int iwch_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 736                           int sg_nents, unsigned int *sg_offset)
 737 {
 738         struct iwch_mr *mhp = to_iwch_mr(ibmr);
 739
 740         mhp->npages = 0;
 741
 742         return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, iwch_set_page);
 743 }
 744
 745 static int iwch_destroy_qp(struct ib_qp *ib_qp)
 746 {
 747         struct iwch_dev *rhp;
 748         struct iwch_qp *qhp;
 749         struct iwch_qp_attributes attrs;
 750         struct iwch_ucontext *ucontext;
 751
 752         qhp = to_iwch_qp(ib_qp);
 753         rhp = qhp->rhp;
 754
 755         attrs.next_state = IWCH_QP_STATE_ERROR;
 756         iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
 757         wait_event(qhp->wait, !qhp->ep);
 758
 759         remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid);
 760
 761         atomic_dec(&qhp->refcnt);
 762         wait_event(qhp->wait, !atomic_read(&qhp->refcnt));
 763
 764         ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context)
 765                                   : NULL;
 766         cxio_destroy_qp(&rhp->rdev, &qhp->wq,
 767                         ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
 768
 769         pr_debug("%s ib_qp %p qpid 0x%0x qhp %p\n", __func__,
 770                  ib_qp, qhp->wq.qpid, qhp);
 771         kfree(qhp);
 772         return 0;
 773 }
 774
 775 static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
 776                              struct ib_qp_init_attr *attrs,
 777                              struct ib_udata *udata)
 778 {
 779         struct iwch_dev *rhp;
 780         struct iwch_qp *qhp;
 781         struct iwch_pd *php;
 782         struct iwch_cq *schp;
 783         struct iwch_cq *rchp;
 784         struct iwch_create_qp_resp uresp;
 785         int wqsize, sqsize, rqsize;
 786         struct iwch_ucontext *ucontext;
 787
 788         pr_debug("%s ib_pd %p\n", __func__, pd);
 789         if (attrs->qp_type != IB_QPT_RC)
 790                 return ERR_PTR(-EINVAL);
 791         php = to_iwch_pd(pd);
 792         rhp = php->rhp;
 793         schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid);
 794         rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid);
 795         if (!schp || !rchp)
 796                 return ERR_PTR(-EINVAL);
 797
 798         /* The RQT size must be # of entries + 1 rounded up to a power of two */
 799         rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr);
 800         if (rqsize == attrs->cap.max_recv_wr)
 801                 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1);
 802
 803         /* T3 doesn't support RQT depth < 16 */
 804         if (rqsize < 16)
 805                 rqsize = 16;
 806
 807         if (rqsize > T3_MAX_RQ_SIZE)
 808                 return ERR_PTR(-EINVAL);
 809
 810         if (attrs->cap.max_inline_data > T3_MAX_INLINE)
 811                 return ERR_PTR(-EINVAL);
 812
 813         /*
 814          * NOTE: The SQ and total WQ sizes don't need to be
 815          * a power of two.  However, all the code assumes
 816          * they are. EG: Q_FREECNT() and friends.
 817          */
 818         sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
 819         wqsize = roundup_pow_of_two(rqsize + sqsize);
 820
 821         /*
 822          * Kernel users need more wq space for fastreg WRs which can take
 823          * 2 WR fragments.
 824          */
 825         ucontext = rdma_udata_to_drv_context(udata, struct iwch_ucontext,
 826                                              ibucontext);
 827         if (!ucontext && wqsize < (rqsize + (2 * sqsize)))
 828                 wqsize = roundup_pow_of_two(rqsize +
 829                                 roundup_pow_of_two(attrs->cap.max_send_wr * 2));
 830         pr_debug("%s wqsize %d sqsize %d rqsize %d\n", __func__,
 831                  wqsize, sqsize, rqsize);
 832         qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
 833         if (!qhp)
 834                 return ERR_PTR(-ENOMEM);
 835         qhp->wq.size_log2 = ilog2(wqsize);
 836         qhp->wq.rq_size_log2 = ilog2(rqsize);
 837         qhp->wq.sq_size_log2 = ilog2(sqsize);
 838         if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
 839                            ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
 840                 kfree(qhp);
 841                 return ERR_PTR(-ENOMEM);
 842         }
 843
 844         attrs->cap.max_recv_wr = rqsize - 1;
 845         attrs->cap.max_send_wr = sqsize;
 846         attrs->cap.max_inline_data = T3_MAX_INLINE;
 847
 848         qhp->rhp = rhp;
 849         qhp->attr.pd = php->pdid;
 850         qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
 851         qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid;
 852         qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
 853         qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
 854         qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
 855         qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
 856         qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
 857         qhp->attr.state = IWCH_QP_STATE_IDLE;
 858         qhp->attr.next_state = IWCH_QP_STATE_IDLE;
 859
 860         /*
 861          * XXX - These don't get passed in from the openib user
 862          * at create time.  The CM sets them via a QP modify.
 863          * Need to fix...  I think the CM should
 864          */
 865         qhp->attr.enable_rdma_read = 1;
 866         qhp->attr.enable_rdma_write = 1;
 867         qhp->attr.enable_bind = 1;
 868         qhp->attr.max_ord = 1;
 869         qhp->attr.max_ird = 1;
 870
 871         spin_lock_init(&qhp->lock);
 872         init_waitqueue_head(&qhp->wait);
 873         atomic_set(&qhp->refcnt, 1);
 874
 875         if (insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid)) {
 876                 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
 877                         ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
 878                 kfree(qhp);
 879                 return ERR_PTR(-ENOMEM);
 880         }
 881
 882         if (udata) {
 883
 884                 struct iwch_mm_entry *mm1, *mm2;
 885
 886                 mm1 = kmalloc(sizeof *mm1, GFP_KERNEL);
 887                 if (!mm1) {
 888                         iwch_destroy_qp(&qhp->ibqp);
 889                         return ERR_PTR(-ENOMEM);
 890                 }
 891
 892                 mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
 893                 if (!mm2) {
 894                         kfree(mm1);
 895                         iwch_destroy_qp(&qhp->ibqp);
 896                         return ERR_PTR(-ENOMEM);
 897                 }
 898
 899                 uresp.qpid = qhp->wq.qpid;
 900                 uresp.size_log2 = qhp->wq.size_log2;
 901                 uresp.sq_size_log2 = qhp->wq.sq_size_log2;
 902                 uresp.rq_size_log2 = qhp->wq.rq_size_log2;
 903                 spin_lock(&ucontext->mmap_lock);
 904                 uresp.key = ucontext->key;
 905                 ucontext->key += PAGE_SIZE;
 906                 uresp.db_key = ucontext->key;
 907                 ucontext->key += PAGE_SIZE;
 908                 spin_unlock(&ucontext->mmap_lock);
 909                 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
 910                         kfree(mm1);
 911                         kfree(mm2);
 912                         iwch_destroy_qp(&qhp->ibqp);
 913                         return ERR_PTR(-EFAULT);
 914                 }
 915                 mm1->key = uresp.key;
 916                 mm1->addr = virt_to_phys(qhp->wq.queue);
 917                 mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
 918                 insert_mmap(ucontext, mm1);
 919                 mm2->key = uresp.db_key;
 920                 mm2->addr = qhp->wq.udb & PAGE_MASK;
 921                 mm2->len = PAGE_SIZE;
 922                 insert_mmap(ucontext, mm2);
 923         }
 924         qhp->ibqp.qp_num = qhp->wq.qpid;
 925         pr_debug("%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n",
 926                  __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
 927                  qhp->wq.qpid, qhp, (unsigned long long)qhp->wq.dma_addr,
 928                  1 << qhp->wq.size_log2, qhp->wq.rq_addr);
 929         return &qhp->ibqp;
 930 }
 931
 932 static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 933                       int attr_mask, struct ib_udata *udata)
 934 {
 935         struct iwch_dev *rhp;
 936         struct iwch_qp *qhp;
 937         enum iwch_qp_attr_mask mask = 0;
 938         struct iwch_qp_attributes attrs;
 939
 940         pr_debug("%s ib_qp %p\n", __func__, ibqp);
 941
 942         /* iwarp does not support the RTR state */
 943         if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
 944                 attr_mask &= ~IB_QP_STATE;
 945
 946         /* Make sure we still have something left to do */
 947         if (!attr_mask)
 948                 return 0;
 949
 950         memset(&attrs, 0, sizeof attrs);
 951         qhp = to_iwch_qp(ibqp);
 952         rhp = qhp->rhp;
 953
 954         attrs.next_state = iwch_convert_state(attr->qp_state);
 955         attrs.enable_rdma_read = (attr->qp_access_flags &
 956                                IB_ACCESS_REMOTE_READ) ?  1 : 0;
 957         attrs.enable_rdma_write = (attr->qp_access_flags &
 958                                 IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
 959         attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
 960
 961
 962         mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;
 963         mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
 964                         (IWCH_QP_ATTR_ENABLE_RDMA_READ |
 965                          IWCH_QP_ATTR_ENABLE_RDMA_WRITE |
 966                          IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0;
 967
 968         return iwch_modify_qp(rhp, qhp, mask, &attrs, 0);
 969 }
 970
 971 void iwch_qp_add_ref(struct ib_qp *qp)
 972 {
 973         pr_debug("%s ib_qp %p\n", __func__, qp);
 974         atomic_inc(&(to_iwch_qp(qp)->refcnt));
 975 }
 976
 977 void iwch_qp_rem_ref(struct ib_qp *qp)
 978 {
 979         pr_debug("%s ib_qp %p\n", __func__, qp);
 980         if (atomic_dec_and_test(&(to_iwch_qp(qp)->refcnt)))
 981                 wake_up(&(to_iwch_qp(qp)->wait));
 982 }
 983
 984 static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)
 985 {
 986         pr_debug("%s ib_dev %p qpn 0x%x\n", __func__, dev, qpn);
 987         return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);
 988 }
 989
 990
 991 static int iwch_query_pkey(struct ib_device *ibdev,
 992                            u8 port, u16 index, u16 * pkey)
 993 {
 994         pr_debug("%s ibdev %p\n", __func__, ibdev);
 995         *pkey = 0;
 996         return 0;
 997 }
 998
 999 static int iwch_query_gid(struct ib_device *ibdev, u8 port,
1000                           int index, union ib_gid *gid)
1001 {
1002         struct iwch_dev *dev;
1003
1004         pr_debug("%s ibdev %p, port %d, index %d, gid %p\n",
1005                  __func__, ibdev, port, index, gid);
1006         dev = to_iwch_dev(ibdev);
1007         BUG_ON(port == 0 || port > 2);
1008         memset(&(gid->raw[0]), 0, sizeof(gid->raw));
1009         memcpy(&(gid->raw[0]), dev->rdev.port_info.lldevs[port-1]->dev_addr, 6);
1010         return 0;
1011 }
1012
1013 static u64 fw_vers_string_to_u64(struct iwch_dev *iwch_dev)
1014 {
1015         struct ethtool_drvinfo info;
1016         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1017         char *cp, *next;
1018         unsigned fw_maj, fw_min, fw_mic;
1019
1020         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1021
1022         next = info.fw_version + 1;
1023         cp = strsep(&next, ".");
1024         sscanf(cp, "%i", &fw_maj);
1025         cp = strsep(&next, ".");
1026         sscanf(cp, "%i", &fw_min);
1027         cp = strsep(&next, ".");
1028         sscanf(cp, "%i", &fw_mic);
1029
1030         return (((u64)fw_maj & 0xffff) << 32) | ((fw_min & 0xffff) << 16) |
1031                (fw_mic & 0xffff);
1032 }
1033
1034 static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
1035                              struct ib_udata *uhw)
1036 {
1037
1038         struct iwch_dev *dev;
1039
1040         pr_debug("%s ibdev %p\n", __func__, ibdev);
1041
1042         if (uhw->inlen || uhw->outlen)
1043                 return -EINVAL;
1044
1045         dev = to_iwch_dev(ibdev);
1046         memset(props, 0, sizeof *props);
1047         memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1048         props->hw_ver = dev->rdev.t3cdev_p->type;
1049         props->fw_ver = fw_vers_string_to_u64(dev);
1050         props->device_cap_flags = dev->device_cap_flags;
1051         props->page_size_cap = dev->attr.mem_pgsizes_bitmask;
1052         props->vendor_id = (u32)dev->rdev.rnic_info.pdev->vendor;
1053         props->vendor_part_id = (u32)dev->rdev.rnic_info.pdev->device;
1054         props->max_mr_size = dev->attr.max_mr_size;
1055         props->max_qp = dev->attr.max_qps;
1056         props->max_qp_wr = dev->attr.max_wrs;
1057         props->max_send_sge = dev->attr.max_sge_per_wr;
1058         props->max_recv_sge = dev->attr.max_sge_per_wr;
1059         props->max_sge_rd = 1;
1060         props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
1061         props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;
1062         props->max_cq = dev->attr.max_cqs;
1063         props->max_cqe = dev->attr.max_cqes_per_cq;
1064         props->max_mr = dev->attr.max_mem_regs;
1065         props->max_pd = dev->attr.max_pds;
1066         props->local_ca_ack_delay = 0;
1067         props->max_fast_reg_page_list_len = T3_MAX_FASTREG_DEPTH;
1068
1069         return 0;
1070 }
1071
1072 static int iwch_query_port(struct ib_device *ibdev,
1073                            u8 port, struct ib_port_attr *props)
1074 {
1075         struct iwch_dev *dev;
1076         struct net_device *netdev;
1077         struct in_device *inetdev;
1078
1079         pr_debug("%s ibdev %p\n", __func__, ibdev);
1080
1081         dev = to_iwch_dev(ibdev);
1082         netdev = dev->rdev.port_info.lldevs[port-1];
1083
1084         /* props being zeroed by the caller, avoid zeroing it here */
1085         props->max_mtu = IB_MTU_4096;
1086         props->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
1087
1088         if (!netif_carrier_ok(netdev))
1089                 props->state = IB_PORT_DOWN;
1090         else {
1091                 inetdev = in_dev_get(netdev);
1092                 if (inetdev) {
1093                         if (inetdev->ifa_list)
1094                                 props->state = IB_PORT_ACTIVE;
1095                         else
1096                                 props->state = IB_PORT_INIT;
1097                         in_dev_put(inetdev);
1098                 } else
1099                         props->state = IB_PORT_INIT;
1100         }
1101
1102         props->port_cap_flags =
1103             IB_PORT_CM_SUP |
1104             IB_PORT_SNMP_TUNNEL_SUP |
1105             IB_PORT_REINIT_SUP |
1106             IB_PORT_DEVICE_MGMT_SUP |
1107             IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
1108         props->gid_tbl_len = 1;
1109         props->pkey_tbl_len = 1;
1110         props->active_width = 2;
1111         props->active_speed = IB_SPEED_DDR;
1112         props->max_msg_sz = -1;
1113
1114         return 0;
1115 }
1116
1117 static ssize_t hw_rev_show(struct device *dev,
1118                            struct device_attribute *attr, char *buf)
1119 {
1120         struct iwch_dev *iwch_dev =
1121                         rdma_device_to_drv_device(dev, struct iwch_dev, ibdev);
1122
1123         pr_debug("%s dev 0x%p\n", __func__, dev);
1124         return sprintf(buf, "%d\n", iwch_dev->rdev.t3cdev_p->type);
1125 }
1126 static DEVICE_ATTR_RO(hw_rev);
1127
1128 static ssize_t hca_type_show(struct device *dev,
1129                              struct device_attribute *attr, char *buf)
1130 {
1131         struct iwch_dev *iwch_dev =
1132                         rdma_device_to_drv_device(dev, struct iwch_dev, ibdev);
1133         struct ethtool_drvinfo info;
1134         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1135
1136         pr_debug("%s dev 0x%p\n", __func__, dev);
1137         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1138         return sprintf(buf, "%s\n", info.driver);
1139 }
1140 static DEVICE_ATTR_RO(hca_type);
1141
1142 static ssize_t board_id_show(struct device *dev,
1143                              struct device_attribute *attr, char *buf)
1144 {
1145         struct iwch_dev *iwch_dev =
1146                         rdma_device_to_drv_device(dev, struct iwch_dev, ibdev);
1147
1148         pr_debug("%s dev 0x%p\n", __func__, dev);
1149         return sprintf(buf, "%x.%x\n", iwch_dev->rdev.rnic_info.pdev->vendor,
1150                        iwch_dev->rdev.rnic_info.pdev->device);
1151 }
1152 static DEVICE_ATTR_RO(board_id);
1153
1154 enum counters {
1155         IPINRECEIVES,
1156         IPINHDRERRORS,
1157         IPINADDRERRORS,
1158         IPINUNKNOWNPROTOS,
1159         IPINDISCARDS,
1160         IPINDELIVERS,
1161         IPOUTREQUESTS,
1162         IPOUTDISCARDS,
1163         IPOUTNOROUTES,
1164         IPREASMTIMEOUT,
1165         IPREASMREQDS,
1166         IPREASMOKS,
1167         IPREASMFAILS,
1168         TCPACTIVEOPENS,
1169         TCPPASSIVEOPENS,
1170         TCPATTEMPTFAILS,
1171         TCPESTABRESETS,
1172         TCPCURRESTAB,
1173         TCPINSEGS,
1174         TCPOUTSEGS,
1175         TCPRETRANSSEGS,
1176         TCPINERRS,
1177         TCPOUTRSTS,
1178         TCPRTOMIN,
1179         TCPRTOMAX,
1180         NR_COUNTERS
1181 };
1182
1183 static const char * const names[] = {
1184         [IPINRECEIVES] = "ipInReceives",
1185         [IPINHDRERRORS] = "ipInHdrErrors",
1186         [IPINADDRERRORS] = "ipInAddrErrors",
1187         [IPINUNKNOWNPROTOS] = "ipInUnknownProtos",
1188         [IPINDISCARDS] = "ipInDiscards",
1189         [IPINDELIVERS] = "ipInDelivers",
1190         [IPOUTREQUESTS] = "ipOutRequests",
1191         [IPOUTDISCARDS] = "ipOutDiscards",
1192         [IPOUTNOROUTES] = "ipOutNoRoutes",
1193         [IPREASMTIMEOUT] = "ipReasmTimeout",
1194         [IPREASMREQDS] = "ipReasmReqds",
1195         [IPREASMOKS] = "ipReasmOKs",
1196         [IPREASMFAILS] = "ipReasmFails",
1197         [TCPACTIVEOPENS] = "tcpActiveOpens",
1198         [TCPPASSIVEOPENS] = "tcpPassiveOpens",
1199         [TCPATTEMPTFAILS] = "tcpAttemptFails",
1200         [TCPESTABRESETS] = "tcpEstabResets",
1201         [TCPCURRESTAB] = "tcpCurrEstab",
1202         [TCPINSEGS] = "tcpInSegs",
1203         [TCPOUTSEGS] = "tcpOutSegs",
1204         [TCPRETRANSSEGS] = "tcpRetransSegs",
1205         [TCPINERRS] = "tcpInErrs",
1206         [TCPOUTRSTS] = "tcpOutRsts",
1207         [TCPRTOMIN] = "tcpRtoMin",
1208         [TCPRTOMAX] = "tcpRtoMax",
1209 };
1210
1211 static struct rdma_hw_stats *iwch_alloc_stats(struct ib_device *ibdev,
1212                                               u8 port_num)
1213 {
1214         BUILD_BUG_ON(ARRAY_SIZE(names) != NR_COUNTERS);
1215
1216         /* Our driver only supports device level stats */
1217         if (port_num != 0)
1218                 return NULL;
1219
1220         return rdma_alloc_hw_stats_struct(names, NR_COUNTERS,
1221                                           RDMA_HW_STATS_DEFAULT_LIFESPAN);
1222 }
1223
1224 static int iwch_get_mib(struct ib_device *ibdev, struct rdma_hw_stats *stats,
1225                         u8 port, int index)
1226 {
1227         struct iwch_dev *dev;
1228         struct tp_mib_stats m;
1229         int ret;
1230
1231         if (port != 0 || !stats)
1232                 return -ENOSYS;
1233
1234         pr_debug("%s ibdev %p\n", __func__, ibdev);
1235         dev = to_iwch_dev(ibdev);
1236         ret = dev->rdev.t3cdev_p->ctl(dev->rdev.t3cdev_p, RDMA_GET_MIB, &m);
1237         if (ret)
1238                 return -ENOSYS;
1239
1240         stats->value[IPINRECEIVES] = ((u64)m.ipInReceive_hi << 32) +    m.ipInReceive_lo;
1241         stats->value[IPINHDRERRORS] = ((u64)m.ipInHdrErrors_hi << 32) + m.ipInHdrErrors_lo;
1242         stats->value[IPINADDRERRORS] = ((u64)m.ipInAddrErrors_hi << 32) + m.ipInAddrErrors_lo;
1243         stats->value[IPINUNKNOWNPROTOS] = ((u64)m.ipInUnknownProtos_hi << 32) + m.ipInUnknownProtos_lo;
1244         stats->value[IPINDISCARDS] = ((u64)m.ipInDiscards_hi << 32) + m.ipInDiscards_lo;
1245         stats->value[IPINDELIVERS] = ((u64)m.ipInDelivers_hi << 32) + m.ipInDelivers_lo;
1246         stats->value[IPOUTREQUESTS] = ((u64)m.ipOutRequests_hi << 32) + m.ipOutRequests_lo;
1247         stats->value[IPOUTDISCARDS] = ((u64)m.ipOutDiscards_hi << 32) + m.ipOutDiscards_lo;
1248         stats->value[IPOUTNOROUTES] = ((u64)m.ipOutNoRoutes_hi << 32) + m.ipOutNoRoutes_lo;
1249         stats->value[IPREASMTIMEOUT] =  m.ipReasmTimeout;
1250         stats->value[IPREASMREQDS] = m.ipReasmReqds;
1251         stats->value[IPREASMOKS] = m.ipReasmOKs;
1252         stats->value[IPREASMFAILS] = m.ipReasmFails;
1253         stats->value[TCPACTIVEOPENS] =  m.tcpActiveOpens;
1254         stats->value[TCPPASSIVEOPENS] = m.tcpPassiveOpens;
1255         stats->value[TCPATTEMPTFAILS] = m.tcpAttemptFails;
1256         stats->value[TCPESTABRESETS] = m.tcpEstabResets;
1257         stats->value[TCPCURRESTAB] = m.tcpOutRsts;
1258         stats->value[TCPINSEGS] = m.tcpCurrEstab;
1259         stats->value[TCPOUTSEGS] = ((u64)m.tcpInSegs_hi << 32) + m.tcpInSegs_lo;
1260         stats->value[TCPRETRANSSEGS] = ((u64)m.tcpOutSegs_hi << 32) + m.tcpOutSegs_lo;
1261         stats->value[TCPINERRS] = ((u64)m.tcpRetransSeg_hi << 32) + m.tcpRetransSeg_lo,
1262         stats->value[TCPOUTRSTS] = ((u64)m.tcpInErrs_hi << 32) + m.tcpInErrs_lo;
1263         stats->value[TCPRTOMIN] = m.tcpRtoMin;
1264         stats->value[TCPRTOMAX] = m.tcpRtoMax;
1265
1266         return stats->num_counters;
1267 }
1268
1269 static struct attribute *iwch_class_attributes[] = {
1270         &dev_attr_hw_rev.attr,
1271         &dev_attr_hca_type.attr,
1272         &dev_attr_board_id.attr,
1273         NULL
1274 };
1275
1276 static const struct attribute_group iwch_attr_group = {
1277         .attrs = iwch_class_attributes,
1278 };
1279
1280 static int iwch_port_immutable(struct ib_device *ibdev, u8 port_num,
1281                                struct ib_port_immutable *immutable)
1282 {
1283         struct ib_port_attr attr;
1284         int err;
1285
1286         immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
1287
1288         err = ib_query_port(ibdev, port_num, &attr);
1289         if (err)
1290                 return err;
1291
1292         immutable->pkey_tbl_len = attr.pkey_tbl_len;
1293         immutable->gid_tbl_len = attr.gid_tbl_len;
1294
1295         return 0;
1296 }
1297
1298 static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str)
1299 {
1300         struct iwch_dev *iwch_dev = to_iwch_dev(ibdev);
1301         struct ethtool_drvinfo info;
1302         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1303
1304         pr_debug("%s dev 0x%p\n", __func__, iwch_dev);
1305         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1306         snprintf(str, IB_FW_VERSION_NAME_MAX, "%s", info.fw_version);
1307 }
1308
1309 static const struct ib_device_ops iwch_dev_ops = {
1310         .alloc_hw_stats = iwch_alloc_stats,
1311         .alloc_mr = iwch_alloc_mr,
1312         .alloc_mw = iwch_alloc_mw,
1313         .alloc_pd = iwch_allocate_pd,
1314         .alloc_ucontext = iwch_alloc_ucontext,
1315         .create_cq = iwch_create_cq,
1316         .create_qp = iwch_create_qp,
1317         .dealloc_mw = iwch_dealloc_mw,
1318         .dealloc_pd = iwch_deallocate_pd,
1319         .dealloc_ucontext = iwch_dealloc_ucontext,
1320         .dereg_mr = iwch_dereg_mr,
1321         .destroy_cq = iwch_destroy_cq,
1322         .destroy_qp = iwch_destroy_qp,
1323         .get_dev_fw_str = get_dev_fw_ver_str,
1324         .get_dma_mr = iwch_get_dma_mr,
1325         .get_hw_stats = iwch_get_mib,
1326         .get_port_immutable = iwch_port_immutable,
1327         .map_mr_sg = iwch_map_mr_sg,
1328         .mmap = iwch_mmap,
1329         .modify_qp = iwch_ib_modify_qp,
1330         .poll_cq = iwch_poll_cq,
1331         .post_recv = iwch_post_receive,
1332         .post_send = iwch_post_send,
1333         .query_device = iwch_query_device,
1334         .query_gid = iwch_query_gid,
1335         .query_pkey = iwch_query_pkey,
1336         .query_port = iwch_query_port,
1337         .reg_user_mr = iwch_reg_user_mr,
1338         .req_notify_cq = iwch_arm_cq,
1339         .resize_cq = iwch_resize_cq,
1340         INIT_RDMA_OBJ_SIZE(ib_pd, iwch_pd, ibpd),
1341         INIT_RDMA_OBJ_SIZE(ib_ucontext, iwch_ucontext, ibucontext),
1342 };
1343
1344 int iwch_register_device(struct iwch_dev *dev)
1345 {
1346         int ret;
1347
1348         pr_debug("%s iwch_dev %p\n", __func__, dev);
1349         memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
1350         memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1351         dev->ibdev.owner = THIS_MODULE;
1352         dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
1353                                 IB_DEVICE_MEM_WINDOW |
1354                                 IB_DEVICE_MEM_MGT_EXTENSIONS;
1355
1356         /* cxgb3 supports STag 0. */
1357         dev->ibdev.local_dma_lkey = 0;
1358
1359         dev->ibdev.uverbs_cmd_mask =
1360             (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1361             (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1362             (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1363             (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1364             (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1365             (1ull << IB_USER_VERBS_CMD_REG_MR) |
1366             (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1367             (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1368             (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1369             (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1370             (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1371             (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1372             (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1373             (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
1374             (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1375             (1ull << IB_USER_VERBS_CMD_POST_SEND) |
1376             (1ull << IB_USER_VERBS_CMD_POST_RECV);
1377         dev->ibdev.node_type = RDMA_NODE_RNIC;
1378         BUILD_BUG_ON(sizeof(IWCH_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
1379         memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
1380         dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports;
1381         dev->ibdev.num_comp_vectors = 1;
1382         dev->ibdev.dev.parent = &dev->rdev.rnic_info.pdev->dev;
1383         dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
1384
1385         dev->ibdev.iwcm = kzalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
1386         if (!dev->ibdev.iwcm)
1387                 return -ENOMEM;
1388
1389         dev->ibdev.iwcm->connect = iwch_connect;
1390         dev->ibdev.iwcm->accept = iwch_accept_cr;
1391         dev->ibdev.iwcm->reject = iwch_reject_cr;
1392         dev->ibdev.iwcm->create_listen = iwch_create_listen;
1393         dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
1394         dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
1395         dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
1396         dev->ibdev.iwcm->get_qp = iwch_get_qp;
1397         memcpy(dev->ibdev.iwcm->ifname, dev->rdev.t3cdev_p->lldev->name,
1398                sizeof(dev->ibdev.iwcm->ifname));
1399
1400         dev->ibdev.driver_id = RDMA_DRIVER_CXGB3;
1401         rdma_set_device_sysfs_group(&dev->ibdev, &iwch_attr_group);
1402         ib_set_device_ops(&dev->ibdev, &iwch_dev_ops);
1403         ret = ib_register_device(&dev->ibdev, "cxgb3_%d");
1404         if (ret)
1405                 kfree(dev->ibdev.iwcm);
1406         return ret;
1407 }
1408
1409 void iwch_unregister_device(struct iwch_dev *dev)
1410 {
1411         pr_debug("%s iwch_dev %p\n", __func__, dev);
1412         ib_unregister_device(&dev->ibdev);
1413         kfree(dev->ibdev.iwcm);
1414         return;
1415 }