drivers/infiniband/hw/cxgb3/iwch_provider.c

   1 /*
   2  * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/device.h>
  35 #include <linux/netdevice.h>
  36 #include <linux/etherdevice.h>
  37 #include <linux/delay.h>
  38 #include <linux/errno.h>
  39 #include <linux/list.h>
  40 #include <linux/sched/mm.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/ethtool.h>
  43 #include <linux/rtnetlink.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/slab.h>
  46
  47 #include <asm/io.h>
  48 #include <asm/irq.h>
  49 #include <asm/byteorder.h>
  50
  51 #include <rdma/iw_cm.h>
  52 #include <rdma/ib_verbs.h>
  53 #include <rdma/ib_smi.h>
  54 #include <rdma/ib_umem.h>
  55 #include <rdma/ib_user_verbs.h>
  56 #include <rdma/uverbs_ioctl.h>
  57
  58 #include "cxio_hal.h"
  59 #include "iwch.h"
  60 #include "iwch_provider.h"
  61 #include "iwch_cm.h"
  62 #include <rdma/cxgb3-abi.h>
  63 #include "common.h"
  64
  65 static void iwch_dealloc_ucontext(struct ib_ucontext *context)
  66 {
  67         struct iwch_dev *rhp = to_iwch_dev(context->device);
  68         struct iwch_ucontext *ucontext = to_iwch_ucontext(context);
  69         struct iwch_mm_entry *mm, *tmp;
  70
  71         pr_debug("%s context %p\n", __func__, context);
  72         list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)
  73                 kfree(mm);
  74         cxio_release_ucontext(&rhp->rdev, &ucontext->uctx);
  75 }
  76
  77 static int iwch_alloc_ucontext(struct ib_ucontext *ucontext,
  78                                struct ib_udata *udata)
  79 {
  80         struct ib_device *ibdev = ucontext->device;
  81         struct iwch_ucontext *context = to_iwch_ucontext(ucontext);
  82         struct iwch_dev *rhp = to_iwch_dev(ibdev);
  83
  84         pr_debug("%s ibdev %p\n", __func__, ibdev);
  85         cxio_init_ucontext(&rhp->rdev, &context->uctx);
  86         INIT_LIST_HEAD(&context->mmaps);
  87         spin_lock_init(&context->mmap_lock);
  88         return 0;
  89 }
  90
  91 static int iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
  92 {
  93         struct iwch_cq *chp;
  94
  95         pr_debug("%s ib_cq %p\n", __func__, ib_cq);
  96         chp = to_iwch_cq(ib_cq);
  97
  98         xa_erase_irq(&chp->rhp->cqs, chp->cq.cqid);
  99         atomic_dec(&chp->refcnt);
 100         wait_event(chp->wait, !atomic_read(&chp->refcnt));
 101
 102         cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
 103         kfree(chp);
 104         return 0;
 105 }
 106
 107 static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
 108                                     const struct ib_cq_init_attr *attr,
 109                                     struct ib_udata *udata)
 110 {
 111         int entries = attr->cqe;
 112         struct iwch_dev *rhp;
 113         struct iwch_cq *chp;
 114         struct iwch_create_cq_resp uresp;
 115         struct iwch_create_cq_req ureq;
 116         static int warned;
 117         size_t resplen;
 118
 119         pr_debug("%s ib_dev %p entries %d\n", __func__, ibdev, entries);
 120         if (attr->flags)
 121                 return ERR_PTR(-EINVAL);
 122
 123         rhp = to_iwch_dev(ibdev);
 124         chp = kzalloc(sizeof(*chp), GFP_KERNEL);
 125         if (!chp)
 126                 return ERR_PTR(-ENOMEM);
 127
 128         if (udata) {
 129                 if (!t3a_device(rhp)) {
 130                         if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
 131                                 kfree(chp);
 132                                 return ERR_PTR(-EFAULT);
 133                         }
 134                         chp->user_rptr_addr = (u32 __user *)(unsigned long)ureq.user_rptr_addr;
 135                 }
 136         }
 137
 138         if (t3a_device(rhp)) {
 139
 140                 /*
 141                  * T3A: Add some fluff to handle extra CQEs inserted
 142                  * for various errors.
 143                  * Additional CQE possibilities:
 144                  *      TERMINATE,
 145                  *      incoming RDMA WRITE Failures
 146                  *      incoming RDMA READ REQUEST FAILUREs
 147                  * NOTE: We cannot ensure the CQ won't overflow.
 148                  */
 149                 entries += 16;
 150         }
 151         entries = roundup_pow_of_two(entries);
 152         chp->cq.size_log2 = ilog2(entries);
 153
 154         if (cxio_create_cq(&rhp->rdev, &chp->cq, !udata)) {
 155                 kfree(chp);
 156                 return ERR_PTR(-ENOMEM);
 157         }
 158         chp->rhp = rhp;
 159         chp->ibcq.cqe = 1 << chp->cq.size_log2;
 160         spin_lock_init(&chp->lock);
 161         spin_lock_init(&chp->comp_handler_lock);
 162         atomic_set(&chp->refcnt, 1);
 163         init_waitqueue_head(&chp->wait);
 164         if (xa_store_irq(&rhp->cqs, chp->cq.cqid, chp, GFP_KERNEL)) {
 165                 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
 166                 kfree(chp);
 167                 return ERR_PTR(-ENOMEM);
 168         }
 169
 170         if (udata) {
 171                 struct iwch_mm_entry *mm;
 172                 struct iwch_ucontext *ucontext = rdma_udata_to_drv_context(
 173                         udata, struct iwch_ucontext, ibucontext);
 174
 175                 mm = kmalloc(sizeof *mm, GFP_KERNEL);
 176                 if (!mm) {
 177                         iwch_destroy_cq(&chp->ibcq, udata);
 178                         return ERR_PTR(-ENOMEM);
 179                 }
 180                 uresp.cqid = chp->cq.cqid;
 181                 uresp.size_log2 = chp->cq.size_log2;
 182                 spin_lock(&ucontext->mmap_lock);
 183                 uresp.key = ucontext->key;
 184                 ucontext->key += PAGE_SIZE;
 185                 spin_unlock(&ucontext->mmap_lock);
 186                 mm->key = uresp.key;
 187                 mm->addr = virt_to_phys(chp->cq.queue);
 188                 if (udata->outlen < sizeof uresp) {
 189                         if (!warned++)
 190                                 pr_warn("Warning - downlevel libcxgb3 (non-fatal)\n");
 191                         mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
 192                                              sizeof(struct t3_cqe));
 193                         resplen = sizeof(struct iwch_create_cq_resp_v0);
 194                 } else {
 195                         mm->len = PAGE_ALIGN(((1UL << uresp.size_log2) + 1) *
 196                                              sizeof(struct t3_cqe));
 197                         uresp.memsize = mm->len;
 198                         uresp.reserved = 0;
 199                         resplen = sizeof uresp;
 200                 }
 201                 if (ib_copy_to_udata(udata, &uresp, resplen)) {
 202                         kfree(mm);
 203                         iwch_destroy_cq(&chp->ibcq, udata);
 204                         return ERR_PTR(-EFAULT);
 205                 }
 206                 insert_mmap(ucontext, mm);
 207         }
 208         pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n",
 209                  chp->cq.cqid, chp, (1 << chp->cq.size_log2),
 210                  (unsigned long long)chp->cq.dma_addr);
 211         return &chp->ibcq;
 212 }
 213
 214 static int iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
 215 {
 216 #ifdef notyet
 217         struct iwch_cq *chp = to_iwch_cq(cq);
 218         struct t3_cq oldcq, newcq;
 219         int ret;
 220
 221         pr_debug("%s ib_cq %p cqe %d\n", __func__, cq, cqe);
 222
 223         /* We don't downsize... */
 224         if (cqe <= cq->cqe)
 225                 return 0;
 226
 227         /* create new t3_cq with new size */
 228         cqe = roundup_pow_of_two(cqe+1);
 229         newcq.size_log2 = ilog2(cqe);
 230
 231         /* Dont allow resize to less than the current wce count */
 232         if (cqe < Q_COUNT(chp->cq.rptr, chp->cq.wptr)) {
 233                 return -ENOMEM;
 234         }
 235
 236         /* Quiesce all QPs using this CQ */
 237         ret = iwch_quiesce_qps(chp);
 238         if (ret) {
 239                 return ret;
 240         }
 241
 242         ret = cxio_create_cq(&chp->rhp->rdev, &newcq);
 243         if (ret) {
 244                 return ret;
 245         }
 246
 247         /* copy CQEs */
 248         memcpy(newcq.queue, chp->cq.queue, (1 << chp->cq.size_log2) *
 249                                         sizeof(struct t3_cqe));
 250
 251         /* old iwch_qp gets new t3_cq but keeps old cqid */
 252         oldcq = chp->cq;
 253         chp->cq = newcq;
 254         chp->cq.cqid = oldcq.cqid;
 255
 256         /* resize new t3_cq to update the HW context */
 257         ret = cxio_resize_cq(&chp->rhp->rdev, &chp->cq);
 258         if (ret) {
 259                 chp->cq = oldcq;
 260                 return ret;
 261         }
 262         chp->ibcq.cqe = (1<<chp->cq.size_log2) - 1;
 263
 264         /* destroy old t3_cq */
 265         oldcq.cqid = newcq.cqid;
 266         ret = cxio_destroy_cq(&chp->rhp->rdev, &oldcq);
 267         if (ret) {
 268                 pr_err("%s - cxio_destroy_cq failed %d\n", __func__, ret);
 269         }
 270
 271         /* add user hooks here */
 272
 273         /* resume qps */
 274         ret = iwch_resume_qps(chp);
 275         return ret;
 276 #else
 277         return -ENOSYS;
 278 #endif
 279 }
 280
 281 static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 282 {
 283         struct iwch_dev *rhp;
 284         struct iwch_cq *chp;
 285         enum t3_cq_opcode cq_op;
 286         int err;
 287         unsigned long flag;
 288         u32 rptr;
 289
 290         chp = to_iwch_cq(ibcq);
 291         rhp = chp->rhp;
 292         if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
 293                 cq_op = CQ_ARM_SE;
 294         else
 295                 cq_op = CQ_ARM_AN;
 296         if (chp->user_rptr_addr) {
 297                 if (get_user(rptr, chp->user_rptr_addr))
 298                         return -EFAULT;
 299                 spin_lock_irqsave(&chp->lock, flag);
 300                 chp->cq.rptr = rptr;
 301         } else
 302                 spin_lock_irqsave(&chp->lock, flag);
 303         pr_debug("%s rptr 0x%x\n", __func__, chp->cq.rptr);
 304         err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0);
 305         spin_unlock_irqrestore(&chp->lock, flag);
 306         if (err < 0)
 307                 pr_err("Error %d rearming CQID 0x%x\n", err, chp->cq.cqid);
 308         if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
 309                 err = 0;
 310         return err;
 311 }
 312
 313 static int iwch_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 314 {
 315         int len = vma->vm_end - vma->vm_start;
 316         u32 key = vma->vm_pgoff << PAGE_SHIFT;
 317         struct cxio_rdev *rdev_p;
 318         int ret = 0;
 319         struct iwch_mm_entry *mm;
 320         struct iwch_ucontext *ucontext;
 321         u64 addr;
 322
 323         pr_debug("%s pgoff 0x%lx key 0x%x len %d\n", __func__, vma->vm_pgoff,
 324                  key, len);
 325
 326         if (vma->vm_start & (PAGE_SIZE-1)) {
 327                 return -EINVAL;
 328         }
 329
 330         rdev_p = &(to_iwch_dev(context->device)->rdev);
 331         ucontext = to_iwch_ucontext(context);
 332
 333         mm = remove_mmap(ucontext, key, len);
 334         if (!mm)
 335                 return -EINVAL;
 336         addr = mm->addr;
 337         kfree(mm);
 338
 339         if ((addr >= rdev_p->rnic_info.udbell_physbase) &&
 340             (addr < (rdev_p->rnic_info.udbell_physbase +
 341                        rdev_p->rnic_info.udbell_len))) {
 342
 343                 /*
 344                  * Map T3 DB register.
 345                  */
 346                 if (vma->vm_flags & VM_READ) {
 347                         return -EPERM;
 348                 }
 349
 350                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 351                 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
 352                 vma->vm_flags &= ~VM_MAYREAD;
 353                 ret = io_remap_pfn_range(vma, vma->vm_start,
 354                                          addr >> PAGE_SHIFT,
 355                                          len, vma->vm_page_prot);
 356         } else {
 357
 358                 /*
 359                  * Map WQ or CQ contig dma memory...
 360                  */
 361                 ret = remap_pfn_range(vma, vma->vm_start,
 362                                       addr >> PAGE_SHIFT,
 363                                       len, vma->vm_page_prot);
 364         }
 365
 366         return ret;
 367 }
 368
 369 static void iwch_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)
 370 {
 371         struct iwch_dev *rhp;
 372         struct iwch_pd *php;
 373
 374         php = to_iwch_pd(pd);
 375         rhp = php->rhp;
 376         pr_debug("%s ibpd %p pdid 0x%x\n", __func__, pd, php->pdid);
 377         cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
 378 }
 379
 380 static int iwch_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)
 381 {
 382         struct iwch_pd *php = to_iwch_pd(pd);
 383         struct ib_device *ibdev = pd->device;
 384         u32 pdid;
 385         struct iwch_dev *rhp;
 386
 387         pr_debug("%s ibdev %p\n", __func__, ibdev);
 388         rhp = (struct iwch_dev *) ibdev;
 389         pdid = cxio_hal_get_pdid(rhp->rdev.rscp);
 390         if (!pdid)
 391                 return -EINVAL;
 392
 393         php->pdid = pdid;
 394         php->rhp = rhp;
 395         if (udata) {
 396                 struct iwch_alloc_pd_resp resp = {.pdid = php->pdid};
 397
 398                 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
 399                         iwch_deallocate_pd(&php->ibpd, udata);
 400                         return -EFAULT;
 401                 }
 402         }
 403         pr_debug("%s pdid 0x%0x ptr 0x%p\n", __func__, pdid, php);
 404         return 0;
 405 }
 406
 407 static int iwch_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 408 {
 409         struct iwch_dev *rhp;
 410         struct iwch_mr *mhp;
 411         u32 mmid;
 412
 413         pr_debug("%s ib_mr %p\n", __func__, ib_mr);
 414
 415         mhp = to_iwch_mr(ib_mr);
 416         kfree(mhp->pages);
 417         rhp = mhp->rhp;
 418         mmid = mhp->attr.stag >> 8;
 419         cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
 420                        mhp->attr.pbl_addr);
 421         iwch_free_pbl(mhp);
 422         xa_erase_irq(&rhp->mrs, mmid);
 423         if (mhp->kva)
 424                 kfree((void *) (unsigned long) mhp->kva);
 425         if (mhp->umem)
 426                 ib_umem_release(mhp->umem);
 427         pr_debug("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp);
 428         kfree(mhp);
 429         return 0;
 430 }
 431
 432 static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
 433 {
 434         const u64 total_size = 0xffffffff;
 435         const u64 mask = (total_size + PAGE_SIZE - 1) & PAGE_MASK;
 436         struct iwch_pd *php = to_iwch_pd(pd);
 437         struct iwch_dev *rhp = php->rhp;
 438         struct iwch_mr *mhp;
 439         __be64 *page_list;
 440         int shift = 26, npages, ret, i;
 441
 442         pr_debug("%s ib_pd %p\n", __func__, pd);
 443
 444         /*
 445          * T3 only supports 32 bits of size.
 446          */
 447         if (sizeof(phys_addr_t) > 4) {
 448                 pr_warn_once("Cannot support dma_mrs on this platform\n");
 449                 return ERR_PTR(-ENOTSUPP);
 450         }
 451
 452         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 453         if (!mhp)
 454                 return ERR_PTR(-ENOMEM);
 455
 456         mhp->rhp = rhp;
 457
 458         npages = (total_size + (1ULL << shift) - 1) >> shift;
 459         if (!npages) {
 460                 ret = -EINVAL;
 461                 goto err;
 462         }
 463
 464         page_list = kmalloc_array(npages, sizeof(u64), GFP_KERNEL);
 465         if (!page_list) {
 466                 ret = -ENOMEM;
 467                 goto err;
 468         }
 469
 470         for (i = 0; i < npages; i++)
 471                 page_list[i] = cpu_to_be64((u64)i << shift);
 472
 473         pr_debug("%s mask 0x%llx shift %d len %lld pbl_size %d\n",
 474                  __func__, mask, shift, total_size, npages);
 475
 476         ret = iwch_alloc_pbl(mhp, npages);
 477         if (ret) {
 478                 kfree(page_list);
 479                 goto err_pbl;
 480         }
 481
 482         ret = iwch_write_pbl(mhp, page_list, npages, 0);
 483         kfree(page_list);
 484         if (ret)
 485                 goto err_pbl;
 486
 487         mhp->attr.pdid = php->pdid;
 488         mhp->attr.zbva = 0;
 489
 490         mhp->attr.perms = iwch_ib_to_tpt_access(acc);
 491         mhp->attr.va_fbo = 0;
 492         mhp->attr.page_size = shift - 12;
 493
 494         mhp->attr.len = (u32) total_size;
 495         mhp->attr.pbl_size = npages;
 496         ret = iwch_register_mem(rhp, php, mhp, shift);
 497         if (ret)
 498                 goto err_pbl;
 499
 500         return &mhp->ibmr;
 501
 502 err_pbl:
 503         iwch_free_pbl(mhp);
 504
 505 err:
 506         kfree(mhp);
 507         return ERR_PTR(ret);
 508 }
 509
 510 static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 511                                       u64 virt, int acc, struct ib_udata *udata)
 512 {
 513         __be64 *pages;
 514         int shift, n, i;
 515         int err = 0;
 516         struct iwch_dev *rhp;
 517         struct iwch_pd *php;
 518         struct iwch_mr *mhp;
 519         struct iwch_reg_user_mr_resp uresp;
 520         struct sg_dma_page_iter sg_iter;
 521         pr_debug("%s ib_pd %p\n", __func__, pd);
 522
 523         php = to_iwch_pd(pd);
 524         rhp = php->rhp;
 525         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 526         if (!mhp)
 527                 return ERR_PTR(-ENOMEM);
 528
 529         mhp->rhp = rhp;
 530
 531         mhp->umem = ib_umem_get(udata, start, length, acc, 0);
 532         if (IS_ERR(mhp->umem)) {
 533                 err = PTR_ERR(mhp->umem);
 534                 kfree(mhp);
 535                 return ERR_PTR(err);
 536         }
 537
 538         shift = PAGE_SHIFT;
 539
 540         n = ib_umem_num_pages(mhp->umem);
 541
 542         err = iwch_alloc_pbl(mhp, n);
 543         if (err)
 544                 goto err;
 545
 546         pages = (__be64 *) __get_free_page(GFP_KERNEL);
 547         if (!pages) {
 548                 err = -ENOMEM;
 549                 goto err_pbl;
 550         }
 551
 552         i = n = 0;
 553
 554         for_each_sg_dma_page(mhp->umem->sg_head.sgl, &sg_iter, mhp->umem->nmap, 0) {
 555                 pages[i++] = cpu_to_be64(sg_page_iter_dma_address(&sg_iter));
 556                 if (i == PAGE_SIZE / sizeof *pages) {
 557                         err = iwch_write_pbl(mhp, pages, i, n);
 558                         if (err)
 559                                 goto pbl_done;
 560                         n += i;
 561                         i = 0;
 562                 }
 563         }
 564
 565         if (i)
 566                 err = iwch_write_pbl(mhp, pages, i, n);
 567
 568 pbl_done:
 569         free_page((unsigned long) pages);
 570         if (err)
 571                 goto err_pbl;
 572
 573         mhp->attr.pdid = php->pdid;
 574         mhp->attr.zbva = 0;
 575         mhp->attr.perms = iwch_ib_to_tpt_access(acc);
 576         mhp->attr.va_fbo = virt;
 577         mhp->attr.page_size = shift - 12;
 578         mhp->attr.len = (u32) length;
 579
 580         err = iwch_register_mem(rhp, php, mhp, shift);
 581         if (err)
 582                 goto err_pbl;
 583
 584         if (udata && !t3a_device(rhp)) {
 585                 uresp.pbl_addr = (mhp->attr.pbl_addr -
 586                                  rhp->rdev.rnic_info.pbl_base) >> 3;
 587                 pr_debug("%s user resp pbl_addr 0x%x\n", __func__,
 588                          uresp.pbl_addr);
 589
 590                 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
 591                         iwch_dereg_mr(&mhp->ibmr, udata);
 592                         err = -EFAULT;
 593                         goto err;
 594                 }
 595         }
 596
 597         return &mhp->ibmr;
 598
 599 err_pbl:
 600         iwch_free_pbl(mhp);
 601
 602 err:
 603         ib_umem_release(mhp->umem);
 604         kfree(mhp);
 605         return ERR_PTR(err);
 606 }
 607
 608 static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
 609                                    struct ib_udata *udata)
 610 {
 611         struct iwch_dev *rhp;
 612         struct iwch_pd *php;
 613         struct iwch_mw *mhp;
 614         u32 mmid;
 615         u32 stag = 0;
 616         int ret;
 617
 618         if (type != IB_MW_TYPE_1)
 619                 return ERR_PTR(-EINVAL);
 620
 621         php = to_iwch_pd(pd);
 622         rhp = php->rhp;
 623         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 624         if (!mhp)
 625                 return ERR_PTR(-ENOMEM);
 626         ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid);
 627         if (ret) {
 628                 kfree(mhp);
 629                 return ERR_PTR(ret);
 630         }
 631         mhp->rhp = rhp;
 632         mhp->attr.pdid = php->pdid;
 633         mhp->attr.type = TPT_MW;
 634         mhp->attr.stag = stag;
 635         mmid = (stag) >> 8;
 636         mhp->ibmw.rkey = stag;
 637         if (xa_insert_irq(&rhp->mrs, mmid, mhp, GFP_KERNEL)) {
 638                 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
 639                 kfree(mhp);
 640                 return ERR_PTR(-ENOMEM);
 641         }
 642         pr_debug("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
 643         return &(mhp->ibmw);
 644 }
 645
 646 static int iwch_dealloc_mw(struct ib_mw *mw)
 647 {
 648         struct iwch_dev *rhp;
 649         struct iwch_mw *mhp;
 650         u32 mmid;
 651
 652         mhp = to_iwch_mw(mw);
 653         rhp = mhp->rhp;
 654         mmid = (mw->rkey) >> 8;
 655         cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
 656         xa_erase_irq(&rhp->mrs, mmid);
 657         pr_debug("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);
 658         kfree(mhp);
 659         return 0;
 660 }
 661
 662 static struct ib_mr *iwch_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
 663                                    u32 max_num_sg, struct ib_udata *udata)
 664 {
 665         struct iwch_dev *rhp;
 666         struct iwch_pd *php;
 667         struct iwch_mr *mhp;
 668         u32 mmid;
 669         u32 stag = 0;
 670         int ret = -ENOMEM;
 671
 672         if (mr_type != IB_MR_TYPE_MEM_REG ||
 673             max_num_sg > T3_MAX_FASTREG_DEPTH)
 674                 return ERR_PTR(-EINVAL);
 675
 676         php = to_iwch_pd(pd);
 677         rhp = php->rhp;
 678         mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
 679         if (!mhp)
 680                 goto err;
 681
 682         mhp->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
 683         if (!mhp->pages)
 684                 goto pl_err;
 685
 686         mhp->rhp = rhp;
 687         ret = iwch_alloc_pbl(mhp, max_num_sg);
 688         if (ret)
 689                 goto err1;
 690         mhp->attr.pbl_size = max_num_sg;
 691         ret = cxio_allocate_stag(&rhp->rdev, &stag, php->pdid,
 692                                  mhp->attr.pbl_size, mhp->attr.pbl_addr);
 693         if (ret)
 694                 goto err2;
 695         mhp->attr.pdid = php->pdid;
 696         mhp->attr.type = TPT_NON_SHARED_MR;
 697         mhp->attr.stag = stag;
 698         mhp->attr.state = 1;
 699         mmid = (stag) >> 8;
 700         mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
 701         ret = xa_insert_irq(&rhp->mrs, mmid, mhp, GFP_KERNEL);
 702         if (ret)
 703                 goto err3;
 704
 705         pr_debug("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
 706         return &(mhp->ibmr);
 707 err3:
 708         cxio_dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size,
 709                        mhp->attr.pbl_addr);
 710 err2:
 711         iwch_free_pbl(mhp);
 712 err1:
 713         kfree(mhp->pages);
 714 pl_err:
 715         kfree(mhp);
 716 err:
 717         return ERR_PTR(ret);
 718 }
 719
 720 static int iwch_set_page(struct ib_mr *ibmr, u64 addr)
 721 {
 722         struct iwch_mr *mhp = to_iwch_mr(ibmr);
 723
 724         if (unlikely(mhp->npages == mhp->attr.pbl_size))
 725                 return -ENOMEM;
 726
 727         mhp->pages[mhp->npages++] = addr;
 728
 729         return 0;
 730 }
 731
 732 static int iwch_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 733                           int sg_nents, unsigned int *sg_offset)
 734 {
 735         struct iwch_mr *mhp = to_iwch_mr(ibmr);
 736
 737         mhp->npages = 0;
 738
 739         return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, iwch_set_page);
 740 }
 741
 742 static int iwch_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 743 {
 744         struct iwch_dev *rhp;
 745         struct iwch_qp *qhp;
 746         struct iwch_qp_attributes attrs;
 747         struct iwch_ucontext *ucontext;
 748
 749         qhp = to_iwch_qp(ib_qp);
 750         rhp = qhp->rhp;
 751
 752         attrs.next_state = IWCH_QP_STATE_ERROR;
 753         iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
 754         wait_event(qhp->wait, !qhp->ep);
 755
 756         xa_erase_irq(&rhp->qps, qhp->wq.qpid);
 757
 758         atomic_dec(&qhp->refcnt);
 759         wait_event(qhp->wait, !atomic_read(&qhp->refcnt));
 760
 761         ucontext = rdma_udata_to_drv_context(udata, struct iwch_ucontext,
 762                                              ibucontext);
 763         cxio_destroy_qp(&rhp->rdev, &qhp->wq,
 764                         ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
 765
 766         pr_debug("%s ib_qp %p qpid 0x%0x qhp %p\n", __func__,
 767                  ib_qp, qhp->wq.qpid, qhp);
 768         kfree(qhp);
 769         return 0;
 770 }
 771
 772 static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
 773                              struct ib_qp_init_attr *attrs,
 774                              struct ib_udata *udata)
 775 {
 776         struct iwch_dev *rhp;
 777         struct iwch_qp *qhp;
 778         struct iwch_pd *php;
 779         struct iwch_cq *schp;
 780         struct iwch_cq *rchp;
 781         struct iwch_create_qp_resp uresp;
 782         int wqsize, sqsize, rqsize;
 783         struct iwch_ucontext *ucontext;
 784
 785         pr_debug("%s ib_pd %p\n", __func__, pd);
 786         if (attrs->qp_type != IB_QPT_RC)
 787                 return ERR_PTR(-EINVAL);
 788         php = to_iwch_pd(pd);
 789         rhp = php->rhp;
 790         schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid);
 791         rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid);
 792         if (!schp || !rchp)
 793                 return ERR_PTR(-EINVAL);
 794
 795         /* The RQT size must be # of entries + 1 rounded up to a power of two */
 796         rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr);
 797         if (rqsize == attrs->cap.max_recv_wr)
 798                 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1);
 799
 800         /* T3 doesn't support RQT depth < 16 */
 801         if (rqsize < 16)
 802                 rqsize = 16;
 803
 804         if (rqsize > T3_MAX_RQ_SIZE)
 805                 return ERR_PTR(-EINVAL);
 806
 807         if (attrs->cap.max_inline_data > T3_MAX_INLINE)
 808                 return ERR_PTR(-EINVAL);
 809
 810         /*
 811          * NOTE: The SQ and total WQ sizes don't need to be
 812          * a power of two.  However, all the code assumes
 813          * they are. EG: Q_FREECNT() and friends.
 814          */
 815         sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
 816         wqsize = roundup_pow_of_two(rqsize + sqsize);
 817
 818         /*
 819          * Kernel users need more wq space for fastreg WRs which can take
 820          * 2 WR fragments.
 821          */
 822         ucontext = rdma_udata_to_drv_context(udata, struct iwch_ucontext,
 823                                              ibucontext);
 824         if (!ucontext && wqsize < (rqsize + (2 * sqsize)))
 825                 wqsize = roundup_pow_of_two(rqsize +
 826                                 roundup_pow_of_two(attrs->cap.max_send_wr * 2));
 827         pr_debug("%s wqsize %d sqsize %d rqsize %d\n", __func__,
 828                  wqsize, sqsize, rqsize);
 829         qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
 830         if (!qhp)
 831                 return ERR_PTR(-ENOMEM);
 832         qhp->wq.size_log2 = ilog2(wqsize);
 833         qhp->wq.rq_size_log2 = ilog2(rqsize);
 834         qhp->wq.sq_size_log2 = ilog2(sqsize);
 835         if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
 836                            ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
 837                 kfree(qhp);
 838                 return ERR_PTR(-ENOMEM);
 839         }
 840
 841         attrs->cap.max_recv_wr = rqsize - 1;
 842         attrs->cap.max_send_wr = sqsize;
 843         attrs->cap.max_inline_data = T3_MAX_INLINE;
 844
 845         qhp->rhp = rhp;
 846         qhp->attr.pd = php->pdid;
 847         qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
 848         qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid;
 849         qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
 850         qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
 851         qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
 852         qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
 853         qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
 854         qhp->attr.state = IWCH_QP_STATE_IDLE;
 855         qhp->attr.next_state = IWCH_QP_STATE_IDLE;
 856
 857         /*
 858          * XXX - These don't get passed in from the openib user
 859          * at create time.  The CM sets them via a QP modify.
 860          * Need to fix...  I think the CM should
 861          */
 862         qhp->attr.enable_rdma_read = 1;
 863         qhp->attr.enable_rdma_write = 1;
 864         qhp->attr.enable_bind = 1;
 865         qhp->attr.max_ord = 1;
 866         qhp->attr.max_ird = 1;
 867
 868         spin_lock_init(&qhp->lock);
 869         init_waitqueue_head(&qhp->wait);
 870         atomic_set(&qhp->refcnt, 1);
 871
 872         if (xa_store_irq(&rhp->qps, qhp->wq.qpid, qhp, GFP_KERNEL)) {
 873                 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
 874                         ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
 875                 kfree(qhp);
 876                 return ERR_PTR(-ENOMEM);
 877         }
 878
 879         if (udata) {
 880
 881                 struct iwch_mm_entry *mm1, *mm2;
 882
 883                 mm1 = kmalloc(sizeof *mm1, GFP_KERNEL);
 884                 if (!mm1) {
 885                         iwch_destroy_qp(&qhp->ibqp, udata);
 886                         return ERR_PTR(-ENOMEM);
 887                 }
 888
 889                 mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
 890                 if (!mm2) {
 891                         kfree(mm1);
 892                         iwch_destroy_qp(&qhp->ibqp, udata);
 893                         return ERR_PTR(-ENOMEM);
 894                 }
 895
 896                 uresp.qpid = qhp->wq.qpid;
 897                 uresp.size_log2 = qhp->wq.size_log2;
 898                 uresp.sq_size_log2 = qhp->wq.sq_size_log2;
 899                 uresp.rq_size_log2 = qhp->wq.rq_size_log2;
 900                 spin_lock(&ucontext->mmap_lock);
 901                 uresp.key = ucontext->key;
 902                 ucontext->key += PAGE_SIZE;
 903                 uresp.db_key = ucontext->key;
 904                 ucontext->key += PAGE_SIZE;
 905                 spin_unlock(&ucontext->mmap_lock);
 906                 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
 907                         kfree(mm1);
 908                         kfree(mm2);
 909                         iwch_destroy_qp(&qhp->ibqp, udata);
 910                         return ERR_PTR(-EFAULT);
 911                 }
 912                 mm1->key = uresp.key;
 913                 mm1->addr = virt_to_phys(qhp->wq.queue);
 914                 mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
 915                 insert_mmap(ucontext, mm1);
 916                 mm2->key = uresp.db_key;
 917                 mm2->addr = qhp->wq.udb & PAGE_MASK;
 918                 mm2->len = PAGE_SIZE;
 919                 insert_mmap(ucontext, mm2);
 920         }
 921         qhp->ibqp.qp_num = qhp->wq.qpid;
 922         pr_debug("%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n",
 923                  __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
 924                  qhp->wq.qpid, qhp, (unsigned long long)qhp->wq.dma_addr,
 925                  1 << qhp->wq.size_log2, qhp->wq.rq_addr);
 926         return &qhp->ibqp;
 927 }
 928
 929 static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 930                       int attr_mask, struct ib_udata *udata)
 931 {
 932         struct iwch_dev *rhp;
 933         struct iwch_qp *qhp;
 934         enum iwch_qp_attr_mask mask = 0;
 935         struct iwch_qp_attributes attrs;
 936
 937         pr_debug("%s ib_qp %p\n", __func__, ibqp);
 938
 939         /* iwarp does not support the RTR state */
 940         if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
 941                 attr_mask &= ~IB_QP_STATE;
 942
 943         /* Make sure we still have something left to do */
 944         if (!attr_mask)
 945                 return 0;
 946
 947         memset(&attrs, 0, sizeof attrs);
 948         qhp = to_iwch_qp(ibqp);
 949         rhp = qhp->rhp;
 950
 951         attrs.next_state = iwch_convert_state(attr->qp_state);
 952         attrs.enable_rdma_read = (attr->qp_access_flags &
 953                                IB_ACCESS_REMOTE_READ) ?  1 : 0;
 954         attrs.enable_rdma_write = (attr->qp_access_flags &
 955                                 IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
 956         attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
 957
 958
 959         mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;
 960         mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
 961                         (IWCH_QP_ATTR_ENABLE_RDMA_READ |
 962                          IWCH_QP_ATTR_ENABLE_RDMA_WRITE |
 963                          IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0;
 964
 965         return iwch_modify_qp(rhp, qhp, mask, &attrs, 0);
 966 }
 967
 968 void iwch_qp_add_ref(struct ib_qp *qp)
 969 {
 970         pr_debug("%s ib_qp %p\n", __func__, qp);
 971         atomic_inc(&(to_iwch_qp(qp)->refcnt));
 972 }
 973
 974 void iwch_qp_rem_ref(struct ib_qp *qp)
 975 {
 976         pr_debug("%s ib_qp %p\n", __func__, qp);
 977         if (atomic_dec_and_test(&(to_iwch_qp(qp)->refcnt)))
 978                 wake_up(&(to_iwch_qp(qp)->wait));
 979 }
 980
 981 static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)
 982 {
 983         pr_debug("%s ib_dev %p qpn 0x%x\n", __func__, dev, qpn);
 984         return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);
 985 }
 986
 987
 988 static int iwch_query_pkey(struct ib_device *ibdev,
 989                            u8 port, u16 index, u16 * pkey)
 990 {
 991         pr_debug("%s ibdev %p\n", __func__, ibdev);
 992         *pkey = 0;
 993         return 0;
 994 }
 995
 996 static int iwch_query_gid(struct ib_device *ibdev, u8 port,
 997                           int index, union ib_gid *gid)
 998 {
 999         struct iwch_dev *dev;
1000
1001         pr_debug("%s ibdev %p, port %d, index %d, gid %p\n",
1002                  __func__, ibdev, port, index, gid);
1003         dev = to_iwch_dev(ibdev);
1004         BUG_ON(port == 0 || port > 2);
1005         memset(&(gid->raw[0]), 0, sizeof(gid->raw));
1006         memcpy(&(gid->raw[0]), dev->rdev.port_info.lldevs[port-1]->dev_addr, 6);
1007         return 0;
1008 }
1009
1010 static u64 fw_vers_string_to_u64(struct iwch_dev *iwch_dev)
1011 {
1012         struct ethtool_drvinfo info;
1013         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1014         char *cp, *next;
1015         unsigned fw_maj, fw_min, fw_mic;
1016
1017         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1018
1019         next = info.fw_version + 1;
1020         cp = strsep(&next, ".");
1021         sscanf(cp, "%i", &fw_maj);
1022         cp = strsep(&next, ".");
1023         sscanf(cp, "%i", &fw_min);
1024         cp = strsep(&next, ".");
1025         sscanf(cp, "%i", &fw_mic);
1026
1027         return (((u64)fw_maj & 0xffff) << 32) | ((fw_min & 0xffff) << 16) |
1028                (fw_mic & 0xffff);
1029 }
1030
1031 static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
1032                              struct ib_udata *uhw)
1033 {
1034
1035         struct iwch_dev *dev;
1036
1037         pr_debug("%s ibdev %p\n", __func__, ibdev);
1038
1039         if (uhw->inlen || uhw->outlen)
1040                 return -EINVAL;
1041
1042         dev = to_iwch_dev(ibdev);
1043         memset(props, 0, sizeof *props);
1044         memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1045         props->hw_ver = dev->rdev.t3cdev_p->type;
1046         props->fw_ver = fw_vers_string_to_u64(dev);
1047         props->device_cap_flags = dev->device_cap_flags;
1048         props->page_size_cap = dev->attr.mem_pgsizes_bitmask;
1049         props->vendor_id = (u32)dev->rdev.rnic_info.pdev->vendor;
1050         props->vendor_part_id = (u32)dev->rdev.rnic_info.pdev->device;
1051         props->max_mr_size = dev->attr.max_mr_size;
1052         props->max_qp = dev->attr.max_qps;
1053         props->max_qp_wr = dev->attr.max_wrs;
1054         props->max_send_sge = dev->attr.max_sge_per_wr;
1055         props->max_recv_sge = dev->attr.max_sge_per_wr;
1056         props->max_sge_rd = 1;
1057         props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
1058         props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;
1059         props->max_cq = dev->attr.max_cqs;
1060         props->max_cqe = dev->attr.max_cqes_per_cq;
1061         props->max_mr = dev->attr.max_mem_regs;
1062         props->max_pd = dev->attr.max_pds;
1063         props->local_ca_ack_delay = 0;
1064         props->max_fast_reg_page_list_len = T3_MAX_FASTREG_DEPTH;
1065
1066         return 0;
1067 }
1068
1069 static int iwch_query_port(struct ib_device *ibdev,
1070                            u8 port, struct ib_port_attr *props)
1071 {
1072         struct iwch_dev *dev;
1073         struct net_device *netdev;
1074         struct in_device *inetdev;
1075
1076         pr_debug("%s ibdev %p\n", __func__, ibdev);
1077
1078         dev = to_iwch_dev(ibdev);
1079         netdev = dev->rdev.port_info.lldevs[port-1];
1080
1081         /* props being zeroed by the caller, avoid zeroing it here */
1082         props->max_mtu = IB_MTU_4096;
1083         props->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
1084
1085         if (!netif_carrier_ok(netdev))
1086                 props->state = IB_PORT_DOWN;
1087         else {
1088                 inetdev = in_dev_get(netdev);
1089                 if (inetdev) {
1090                         if (inetdev->ifa_list)
1091                                 props->state = IB_PORT_ACTIVE;
1092                         else
1093                                 props->state = IB_PORT_INIT;
1094                         in_dev_put(inetdev);
1095                 } else
1096                         props->state = IB_PORT_INIT;
1097         }
1098
1099         props->port_cap_flags =
1100             IB_PORT_CM_SUP |
1101             IB_PORT_SNMP_TUNNEL_SUP |
1102             IB_PORT_REINIT_SUP |
1103             IB_PORT_DEVICE_MGMT_SUP |
1104             IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
1105         props->gid_tbl_len = 1;
1106         props->pkey_tbl_len = 1;
1107         props->active_width = 2;
1108         props->active_speed = IB_SPEED_DDR;
1109         props->max_msg_sz = -1;
1110
1111         return 0;
1112 }
1113
1114 static ssize_t hw_rev_show(struct device *dev,
1115                            struct device_attribute *attr, char *buf)
1116 {
1117         struct iwch_dev *iwch_dev =
1118                         rdma_device_to_drv_device(dev, struct iwch_dev, ibdev);
1119
1120         pr_debug("%s dev 0x%p\n", __func__, dev);
1121         return sprintf(buf, "%d\n", iwch_dev->rdev.t3cdev_p->type);
1122 }
1123 static DEVICE_ATTR_RO(hw_rev);
1124
1125 static ssize_t hca_type_show(struct device *dev,
1126                              struct device_attribute *attr, char *buf)
1127 {
1128         struct iwch_dev *iwch_dev =
1129                         rdma_device_to_drv_device(dev, struct iwch_dev, ibdev);
1130         struct ethtool_drvinfo info;
1131         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1132
1133         pr_debug("%s dev 0x%p\n", __func__, dev);
1134         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1135         return sprintf(buf, "%s\n", info.driver);
1136 }
1137 static DEVICE_ATTR_RO(hca_type);
1138
1139 static ssize_t board_id_show(struct device *dev,
1140                              struct device_attribute *attr, char *buf)
1141 {
1142         struct iwch_dev *iwch_dev =
1143                         rdma_device_to_drv_device(dev, struct iwch_dev, ibdev);
1144
1145         pr_debug("%s dev 0x%p\n", __func__, dev);
1146         return sprintf(buf, "%x.%x\n", iwch_dev->rdev.rnic_info.pdev->vendor,
1147                        iwch_dev->rdev.rnic_info.pdev->device);
1148 }
1149 static DEVICE_ATTR_RO(board_id);
1150
1151 enum counters {
1152         IPINRECEIVES,
1153         IPINHDRERRORS,
1154         IPINADDRERRORS,
1155         IPINUNKNOWNPROTOS,
1156         IPINDISCARDS,
1157         IPINDELIVERS,
1158         IPOUTREQUESTS,
1159         IPOUTDISCARDS,
1160         IPOUTNOROUTES,
1161         IPREASMTIMEOUT,
1162         IPREASMREQDS,
1163         IPREASMOKS,
1164         IPREASMFAILS,
1165         TCPACTIVEOPENS,
1166         TCPPASSIVEOPENS,
1167         TCPATTEMPTFAILS,
1168         TCPESTABRESETS,
1169         TCPCURRESTAB,
1170         TCPINSEGS,
1171         TCPOUTSEGS,
1172         TCPRETRANSSEGS,
1173         TCPINERRS,
1174         TCPOUTRSTS,
1175         TCPRTOMIN,
1176         TCPRTOMAX,
1177         NR_COUNTERS
1178 };
1179
1180 static const char * const names[] = {
1181         [IPINRECEIVES] = "ipInReceives",
1182         [IPINHDRERRORS] = "ipInHdrErrors",
1183         [IPINADDRERRORS] = "ipInAddrErrors",
1184         [IPINUNKNOWNPROTOS] = "ipInUnknownProtos",
1185         [IPINDISCARDS] = "ipInDiscards",
1186         [IPINDELIVERS] = "ipInDelivers",
1187         [IPOUTREQUESTS] = "ipOutRequests",
1188         [IPOUTDISCARDS] = "ipOutDiscards",
1189         [IPOUTNOROUTES] = "ipOutNoRoutes",
1190         [IPREASMTIMEOUT] = "ipReasmTimeout",
1191         [IPREASMREQDS] = "ipReasmReqds",
1192         [IPREASMOKS] = "ipReasmOKs",
1193         [IPREASMFAILS] = "ipReasmFails",
1194         [TCPACTIVEOPENS] = "tcpActiveOpens",
1195         [TCPPASSIVEOPENS] = "tcpPassiveOpens",
1196         [TCPATTEMPTFAILS] = "tcpAttemptFails",
1197         [TCPESTABRESETS] = "tcpEstabResets",
1198         [TCPCURRESTAB] = "tcpCurrEstab",
1199         [TCPINSEGS] = "tcpInSegs",
1200         [TCPOUTSEGS] = "tcpOutSegs",
1201         [TCPRETRANSSEGS] = "tcpRetransSegs",
1202         [TCPINERRS] = "tcpInErrs",
1203         [TCPOUTRSTS] = "tcpOutRsts",
1204         [TCPRTOMIN] = "tcpRtoMin",
1205         [TCPRTOMAX] = "tcpRtoMax",
1206 };
1207
1208 static struct rdma_hw_stats *iwch_alloc_stats(struct ib_device *ibdev,
1209                                               u8 port_num)
1210 {
1211         BUILD_BUG_ON(ARRAY_SIZE(names) != NR_COUNTERS);
1212
1213         /* Our driver only supports device level stats */
1214         if (port_num != 0)
1215                 return NULL;
1216
1217         return rdma_alloc_hw_stats_struct(names, NR_COUNTERS,
1218                                           RDMA_HW_STATS_DEFAULT_LIFESPAN);
1219 }
1220
1221 static int iwch_get_mib(struct ib_device *ibdev, struct rdma_hw_stats *stats,
1222                         u8 port, int index)
1223 {
1224         struct iwch_dev *dev;
1225         struct tp_mib_stats m;
1226         int ret;
1227
1228         if (port != 0 || !stats)
1229                 return -ENOSYS;
1230
1231         pr_debug("%s ibdev %p\n", __func__, ibdev);
1232         dev = to_iwch_dev(ibdev);
1233         ret = dev->rdev.t3cdev_p->ctl(dev->rdev.t3cdev_p, RDMA_GET_MIB, &m);
1234         if (ret)
1235                 return -ENOSYS;
1236
1237         stats->value[IPINRECEIVES] = ((u64)m.ipInReceive_hi << 32) +    m.ipInReceive_lo;
1238         stats->value[IPINHDRERRORS] = ((u64)m.ipInHdrErrors_hi << 32) + m.ipInHdrErrors_lo;
1239         stats->value[IPINADDRERRORS] = ((u64)m.ipInAddrErrors_hi << 32) + m.ipInAddrErrors_lo;
1240         stats->value[IPINUNKNOWNPROTOS] = ((u64)m.ipInUnknownProtos_hi << 32) + m.ipInUnknownProtos_lo;
1241         stats->value[IPINDISCARDS] = ((u64)m.ipInDiscards_hi << 32) + m.ipInDiscards_lo;
1242         stats->value[IPINDELIVERS] = ((u64)m.ipInDelivers_hi << 32) + m.ipInDelivers_lo;
1243         stats->value[IPOUTREQUESTS] = ((u64)m.ipOutRequests_hi << 32) + m.ipOutRequests_lo;
1244         stats->value[IPOUTDISCARDS] = ((u64)m.ipOutDiscards_hi << 32) + m.ipOutDiscards_lo;
1245         stats->value[IPOUTNOROUTES] = ((u64)m.ipOutNoRoutes_hi << 32) + m.ipOutNoRoutes_lo;
1246         stats->value[IPREASMTIMEOUT] =  m.ipReasmTimeout;
1247         stats->value[IPREASMREQDS] = m.ipReasmReqds;
1248         stats->value[IPREASMOKS] = m.ipReasmOKs;
1249         stats->value[IPREASMFAILS] = m.ipReasmFails;
1250         stats->value[TCPACTIVEOPENS] =  m.tcpActiveOpens;
1251         stats->value[TCPPASSIVEOPENS] = m.tcpPassiveOpens;
1252         stats->value[TCPATTEMPTFAILS] = m.tcpAttemptFails;
1253         stats->value[TCPESTABRESETS] = m.tcpEstabResets;
1254         stats->value[TCPCURRESTAB] = m.tcpOutRsts;
1255         stats->value[TCPINSEGS] = m.tcpCurrEstab;
1256         stats->value[TCPOUTSEGS] = ((u64)m.tcpInSegs_hi << 32) + m.tcpInSegs_lo;
1257         stats->value[TCPRETRANSSEGS] = ((u64)m.tcpOutSegs_hi << 32) + m.tcpOutSegs_lo;
1258         stats->value[TCPINERRS] = ((u64)m.tcpRetransSeg_hi << 32) + m.tcpRetransSeg_lo,
1259         stats->value[TCPOUTRSTS] = ((u64)m.tcpInErrs_hi << 32) + m.tcpInErrs_lo;
1260         stats->value[TCPRTOMIN] = m.tcpRtoMin;
1261         stats->value[TCPRTOMAX] = m.tcpRtoMax;
1262
1263         return stats->num_counters;
1264 }
1265
1266 static struct attribute *iwch_class_attributes[] = {
1267         &dev_attr_hw_rev.attr,
1268         &dev_attr_hca_type.attr,
1269         &dev_attr_board_id.attr,
1270         NULL
1271 };
1272
1273 static const struct attribute_group iwch_attr_group = {
1274         .attrs = iwch_class_attributes,
1275 };
1276
1277 static int iwch_port_immutable(struct ib_device *ibdev, u8 port_num,
1278                                struct ib_port_immutable *immutable)
1279 {
1280         struct ib_port_attr attr;
1281         int err;
1282
1283         immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
1284
1285         err = ib_query_port(ibdev, port_num, &attr);
1286         if (err)
1287                 return err;
1288
1289         immutable->pkey_tbl_len = attr.pkey_tbl_len;
1290         immutable->gid_tbl_len = attr.gid_tbl_len;
1291
1292         return 0;
1293 }
1294
1295 static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str)
1296 {
1297         struct iwch_dev *iwch_dev = to_iwch_dev(ibdev);
1298         struct ethtool_drvinfo info;
1299         struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1300
1301         pr_debug("%s dev 0x%p\n", __func__, iwch_dev);
1302         lldev->ethtool_ops->get_drvinfo(lldev, &info);
1303         snprintf(str, IB_FW_VERSION_NAME_MAX, "%s", info.fw_version);
1304 }
1305
1306 static const struct ib_device_ops iwch_dev_ops = {
1307         .alloc_hw_stats = iwch_alloc_stats,
1308         .alloc_mr = iwch_alloc_mr,
1309         .alloc_mw = iwch_alloc_mw,
1310         .alloc_pd = iwch_allocate_pd,
1311         .alloc_ucontext = iwch_alloc_ucontext,
1312         .create_cq = iwch_create_cq,
1313         .create_qp = iwch_create_qp,
1314         .dealloc_mw = iwch_dealloc_mw,
1315         .dealloc_pd = iwch_deallocate_pd,
1316         .dealloc_ucontext = iwch_dealloc_ucontext,
1317         .dereg_mr = iwch_dereg_mr,
1318         .destroy_cq = iwch_destroy_cq,
1319         .destroy_qp = iwch_destroy_qp,
1320         .get_dev_fw_str = get_dev_fw_ver_str,
1321         .get_dma_mr = iwch_get_dma_mr,
1322         .get_hw_stats = iwch_get_mib,
1323         .get_port_immutable = iwch_port_immutable,
1324         .iw_accept = iwch_accept_cr,
1325         .iw_add_ref = iwch_qp_add_ref,
1326         .iw_connect = iwch_connect,
1327         .iw_create_listen = iwch_create_listen,
1328         .iw_destroy_listen = iwch_destroy_listen,
1329         .iw_get_qp = iwch_get_qp,
1330         .iw_reject = iwch_reject_cr,
1331         .iw_rem_ref = iwch_qp_rem_ref,
1332         .map_mr_sg = iwch_map_mr_sg,
1333         .mmap = iwch_mmap,
1334         .modify_qp = iwch_ib_modify_qp,
1335         .poll_cq = iwch_poll_cq,
1336         .post_recv = iwch_post_receive,
1337         .post_send = iwch_post_send,
1338         .query_device = iwch_query_device,
1339         .query_gid = iwch_query_gid,
1340         .query_pkey = iwch_query_pkey,
1341         .query_port = iwch_query_port,
1342         .reg_user_mr = iwch_reg_user_mr,
1343         .req_notify_cq = iwch_arm_cq,
1344         .resize_cq = iwch_resize_cq,
1345         INIT_RDMA_OBJ_SIZE(ib_pd, iwch_pd, ibpd),
1346         INIT_RDMA_OBJ_SIZE(ib_ucontext, iwch_ucontext, ibucontext),
1347 };
1348
1349 int iwch_register_device(struct iwch_dev *dev)
1350 {
1351         pr_debug("%s iwch_dev %p\n", __func__, dev);
1352         memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
1353         memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1354         dev->ibdev.owner = THIS_MODULE;
1355         dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
1356                                 IB_DEVICE_MEM_WINDOW |
1357                                 IB_DEVICE_MEM_MGT_EXTENSIONS;
1358
1359         /* cxgb3 supports STag 0. */
1360         dev->ibdev.local_dma_lkey = 0;
1361
1362         dev->ibdev.uverbs_cmd_mask =
1363             (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1364             (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1365             (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1366             (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1367             (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1368             (1ull << IB_USER_VERBS_CMD_REG_MR) |
1369             (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1370             (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1371             (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1372             (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1373             (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1374             (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1375             (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1376             (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
1377             (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1378             (1ull << IB_USER_VERBS_CMD_POST_SEND) |
1379             (1ull << IB_USER_VERBS_CMD_POST_RECV);
1380         dev->ibdev.node_type = RDMA_NODE_RNIC;
1381         BUILD_BUG_ON(sizeof(IWCH_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
1382         memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
1383         dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports;
1384         dev->ibdev.num_comp_vectors = 1;
1385         dev->ibdev.dev.parent = &dev->rdev.rnic_info.pdev->dev;
1386         dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
1387
1388         memcpy(dev->ibdev.iw_ifname, dev->rdev.t3cdev_p->lldev->name,
1389                sizeof(dev->ibdev.iw_ifname));
1390
1391         dev->ibdev.driver_id = RDMA_DRIVER_CXGB3;
1392         rdma_set_device_sysfs_group(&dev->ibdev, &iwch_attr_group);
1393         ib_set_device_ops(&dev->ibdev, &iwch_dev_ops);
1394         return ib_register_device(&dev->ibdev, "cxgb3_%d");
1395 }
1396
1397 void iwch_unregister_device(struct iwch_dev *dev)
1398 {
1399         pr_debug("%s iwch_dev %p\n", __func__, dev);
1400         ib_unregister_device(&dev->ibdev);
1401         return;
1402 }