drivers/infiniband/hw/bnxt_re/qplib_res.c

   1 /*
   2  * Broadcom NetXtreme-E RoCE driver.
   3  *
   4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * BSD license below:
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in
  21  *    the documentation and/or other materials provided with the
  22  *    distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Description: QPLib resource manager
  37  */
  38
  39 #define dev_fmt(fmt) "QPLIB: " fmt
  40
  41 #include <linux/spinlock.h>
  42 #include <linux/pci.h>
  43 #include <linux/interrupt.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/dma-mapping.h>
  46 #include <linux/if_vlan.h>
  47 #include <linux/vmalloc.h>
  48 #include <rdma/ib_verbs.h>
  49 #include <rdma/ib_umem.h>
  50
  51 #include "roce_hsi.h"
  52 #include "qplib_res.h"
  53 #include "qplib_sp.h"
  54 #include "qplib_rcfw.h"
  55
  56 static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
  57                                       struct bnxt_qplib_stats *stats);
  58 static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
  59                                       struct bnxt_qplib_chip_ctx *cctx,
  60                                       struct bnxt_qplib_stats *stats);
  61
  62 /* PBL */
  63 static void __free_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl,
  64                        bool is_umem)
  65 {
  66         struct pci_dev *pdev = res->pdev;
  67         int i;
  68
  69         if (!is_umem) {
  70                 for (i = 0; i < pbl->pg_count; i++) {
  71                         if (pbl->pg_arr[i])
  72                                 dma_free_coherent(&pdev->dev, pbl->pg_size,
  73                                                   (void *)((unsigned long)
  74                                                    pbl->pg_arr[i] &
  75                                                   PAGE_MASK),
  76                                                   pbl->pg_map_arr[i]);
  77                         else
  78                                 dev_warn(&pdev->dev,
  79                                          "PBL free pg_arr[%d] empty?!\n", i);
  80                         pbl->pg_arr[i] = NULL;
  81                 }
  82         }
  83         vfree(pbl->pg_arr);
  84         pbl->pg_arr = NULL;
  85         vfree(pbl->pg_map_arr);
  86         pbl->pg_map_arr = NULL;
  87         pbl->pg_count = 0;
  88         pbl->pg_size = 0;
  89 }
  90
  91 static void bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,
  92                                            struct bnxt_qplib_sg_info *sginfo)
  93 {
  94         struct ib_block_iter biter;
  95         int i = 0;
  96
  97         rdma_umem_for_each_dma_block(sginfo->umem, &biter, sginfo->pgsize) {
  98                 pbl->pg_map_arr[i] = rdma_block_iter_dma_address(&biter);
  99                 pbl->pg_arr[i] = NULL;
 100                 pbl->pg_count++;
 101                 i++;
 102         }
 103 }
 104
 105 static int __alloc_pbl(struct bnxt_qplib_res *res,
 106                        struct bnxt_qplib_pbl *pbl,
 107                        struct bnxt_qplib_sg_info *sginfo)
 108 {
 109         struct pci_dev *pdev = res->pdev;
 110         bool is_umem = false;
 111         u32 pages;
 112         int i;
 113
 114         if (sginfo->nopte)
 115                 return 0;
 116         if (sginfo->umem)
 117                 pages = ib_umem_num_dma_blocks(sginfo->umem, sginfo->pgsize);
 118         else
 119                 pages = sginfo->npages;
 120         /* page ptr arrays */
 121         pbl->pg_arr = vmalloc(pages * sizeof(void *));
 122         if (!pbl->pg_arr)
 123                 return -ENOMEM;
 124
 125         pbl->pg_map_arr = vmalloc(pages * sizeof(dma_addr_t));
 126         if (!pbl->pg_map_arr) {
 127                 vfree(pbl->pg_arr);
 128                 pbl->pg_arr = NULL;
 129                 return -ENOMEM;
 130         }
 131         pbl->pg_count = 0;
 132         pbl->pg_size = sginfo->pgsize;
 133
 134         if (!sginfo->umem) {
 135                 for (i = 0; i < pages; i++) {
 136                         pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
 137                                                             pbl->pg_size,
 138                                                             &pbl->pg_map_arr[i],
 139                                                             GFP_KERNEL);
 140                         if (!pbl->pg_arr[i])
 141                                 goto fail;
 142                         pbl->pg_count++;
 143                 }
 144         } else {
 145                 is_umem = true;
 146                 bnxt_qplib_fill_user_dma_pages(pbl, sginfo);
 147         }
 148
 149         return 0;
 150 fail:
 151         __free_pbl(res, pbl, is_umem);
 152         return -ENOMEM;
 153 }
 154
 155 /* HWQ */
 156 void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
 157                          struct bnxt_qplib_hwq *hwq)
 158 {
 159         int i;
 160
 161         if (!hwq->max_elements)
 162                 return;
 163         if (hwq->level >= PBL_LVL_MAX)
 164                 return;
 165
 166         for (i = 0; i < hwq->level + 1; i++) {
 167                 if (i == hwq->level)
 168                         __free_pbl(res, &hwq->pbl[i], hwq->is_user);
 169                 else
 170                         __free_pbl(res, &hwq->pbl[i], false);
 171         }
 172
 173         hwq->level = PBL_LVL_MAX;
 174         hwq->max_elements = 0;
 175         hwq->element_size = 0;
 176         hwq->prod = 0;
 177         hwq->cons = 0;
 178         hwq->cp_bit = 0;
 179 }
 180
 181 /* All HWQs are power of 2 in size */
 182
 183 int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
 184                               struct bnxt_qplib_hwq_attr *hwq_attr)
 185 {
 186         u32 npages, aux_slots, pg_size, aux_pages = 0, aux_size = 0;
 187         struct bnxt_qplib_sg_info sginfo = {};
 188         u32 depth, stride, npbl, npde;
 189         dma_addr_t *src_phys_ptr, **dst_virt_ptr;
 190         struct bnxt_qplib_res *res;
 191         struct pci_dev *pdev;
 192         int i, rc, lvl;
 193
 194         res = hwq_attr->res;
 195         pdev = res->pdev;
 196         pg_size = hwq_attr->sginfo->pgsize;
 197         hwq->level = PBL_LVL_MAX;
 198
 199         depth = roundup_pow_of_two(hwq_attr->depth);
 200         stride = roundup_pow_of_two(hwq_attr->stride);
 201         if (hwq_attr->aux_depth) {
 202                 aux_slots = hwq_attr->aux_depth;
 203                 aux_size = roundup_pow_of_two(hwq_attr->aux_stride);
 204                 aux_pages = (aux_slots * aux_size) / pg_size;
 205                 if ((aux_slots * aux_size) % pg_size)
 206                         aux_pages++;
 207         }
 208
 209         if (!hwq_attr->sginfo->umem) {
 210                 hwq->is_user = false;
 211                 npages = (depth * stride) / pg_size + aux_pages;
 212                 if ((depth * stride) % pg_size)
 213                         npages++;
 214                 if (!npages)
 215                         return -EINVAL;
 216                 hwq_attr->sginfo->npages = npages;
 217         } else {
 218                 unsigned long sginfo_num_pages = ib_umem_num_dma_blocks(
 219                         hwq_attr->sginfo->umem, hwq_attr->sginfo->pgsize);
 220
 221                 hwq->is_user = true;
 222                 npages = sginfo_num_pages;
 223                 npages = (npages * PAGE_SIZE) /
 224                           BIT_ULL(hwq_attr->sginfo->pgshft);
 225                 if ((sginfo_num_pages * PAGE_SIZE) %
 226                      BIT_ULL(hwq_attr->sginfo->pgshft))
 227                         if (!npages)
 228                                 npages++;
 229         }
 230
 231         if (npages == MAX_PBL_LVL_0_PGS) {
 232                 /* This request is Level 0, map PTE */
 233                 rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo);
 234                 if (rc)
 235                         goto fail;
 236                 hwq->level = PBL_LVL_0;
 237         }
 238
 239         if (npages > MAX_PBL_LVL_0_PGS) {
 240                 if (npages > MAX_PBL_LVL_1_PGS) {
 241                         u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
 242                                     0 : PTU_PTE_VALID;
 243                         /* 2 levels of indirection */
 244                         npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
 245                         if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
 246                                 npbl++;
 247                         npde = npbl >> MAX_PDL_LVL_SHIFT;
 248                         if (npbl % BIT(MAX_PDL_LVL_SHIFT))
 249                                 npde++;
 250                         /* Alloc PDE pages */
 251                         sginfo.pgsize = npde * pg_size;
 252                         sginfo.npages = 1;
 253                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
 254
 255                         /* Alloc PBL pages */
 256                         sginfo.npages = npbl;
 257                         sginfo.pgsize = PAGE_SIZE;
 258                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo);
 259                         if (rc)
 260                                 goto fail;
 261                         /* Fill PDL with PBL page pointers */
 262                         dst_virt_ptr =
 263                                 (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
 264                         src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
 265                         if (hwq_attr->type == HWQ_TYPE_MR) {
 266                         /* For MR it is expected that we supply only 1 contigous
 267                          * page i.e only 1 entry in the PDL that will contain
 268                          * all the PBLs for the user supplied memory region
 269                          */
 270                                 for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count;
 271                                      i++)
 272                                         dst_virt_ptr[0][i] = src_phys_ptr[i] |
 273                                                 flag;
 274                         } else {
 275                                 for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count;
 276                                      i++)
 277                                         dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
 278                                                 src_phys_ptr[i] |
 279                                                 PTU_PDE_VALID;
 280                         }
 281                         /* Alloc or init PTEs */
 282                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2],
 283                                          hwq_attr->sginfo);
 284                         if (rc)
 285                                 goto fail;
 286                         hwq->level = PBL_LVL_2;
 287                         if (hwq_attr->sginfo->nopte)
 288                                 goto done;
 289                         /* Fill PBLs with PTE pointers */
 290                         dst_virt_ptr =
 291                                 (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr;
 292                         src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr;
 293                         for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) {
 294                                 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
 295                                         src_phys_ptr[i] | PTU_PTE_VALID;
 296                         }
 297                         if (hwq_attr->type == HWQ_TYPE_QUEUE) {
 298                                 /* Find the last pg of the size */
 299                                 i = hwq->pbl[PBL_LVL_2].pg_count;
 300                                 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
 301                                                                   PTU_PTE_LAST;
 302                                 if (i > 1)
 303                                         dst_virt_ptr[PTR_PG(i - 2)]
 304                                                     [PTR_IDX(i - 2)] |=
 305                                                     PTU_PTE_NEXT_TO_LAST;
 306                         }
 307                 } else { /* pages < 512 npbl = 1, npde = 0 */
 308                         u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
 309                                     0 : PTU_PTE_VALID;
 310
 311                         /* 1 level of indirection */
 312                         npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
 313                         if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
 314                                 npbl++;
 315                         sginfo.npages = npbl;
 316                         sginfo.pgsize = PAGE_SIZE;
 317                         /* Alloc PBL page */
 318                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
 319                         if (rc)
 320                                 goto fail;
 321                         /* Alloc or init  PTEs */
 322                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1],
 323                                          hwq_attr->sginfo);
 324                         if (rc)
 325                                 goto fail;
 326                         hwq->level = PBL_LVL_1;
 327                         if (hwq_attr->sginfo->nopte)
 328                                 goto done;
 329                         /* Fill PBL with PTE pointers */
 330                         dst_virt_ptr =
 331                                 (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
 332                         src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
 333                         for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
 334                                 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
 335                                         src_phys_ptr[i] | flag;
 336                         if (hwq_attr->type == HWQ_TYPE_QUEUE) {
 337                                 /* Find the last pg of the size */
 338                                 i = hwq->pbl[PBL_LVL_1].pg_count;
 339                                 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
 340                                                                   PTU_PTE_LAST;
 341                                 if (i > 1)
 342                                         dst_virt_ptr[PTR_PG(i - 2)]
 343                                                     [PTR_IDX(i - 2)] |=
 344                                                     PTU_PTE_NEXT_TO_LAST;
 345                         }
 346                 }
 347         }
 348 done:
 349         hwq->prod = 0;
 350         hwq->cons = 0;
 351         hwq->pdev = pdev;
 352         hwq->depth = hwq_attr->depth;
 353         hwq->max_elements = depth;
 354         hwq->element_size = stride;
 355         hwq->qe_ppg = pg_size / stride;
 356         /* For direct access to the elements */
 357         lvl = hwq->level;
 358         if (hwq_attr->sginfo->nopte && hwq->level)
 359                 lvl = hwq->level - 1;
 360         hwq->pbl_ptr = hwq->pbl[lvl].pg_arr;
 361         hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr;
 362         spin_lock_init(&hwq->lock);
 363
 364         return 0;
 365 fail:
 366         bnxt_qplib_free_hwq(res, hwq);
 367         return -ENOMEM;
 368 }
 369
 370 /* Context Tables */
 371 void bnxt_qplib_free_ctx(struct bnxt_qplib_res *res,
 372                          struct bnxt_qplib_ctx *ctx)
 373 {
 374         int i;
 375
 376         bnxt_qplib_free_hwq(res, &ctx->qpc_tbl);
 377         bnxt_qplib_free_hwq(res, &ctx->mrw_tbl);
 378         bnxt_qplib_free_hwq(res, &ctx->srqc_tbl);
 379         bnxt_qplib_free_hwq(res, &ctx->cq_tbl);
 380         bnxt_qplib_free_hwq(res, &ctx->tim_tbl);
 381         for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
 382                 bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.qtbl[i]);
 383         /* restore original pde level before destroy */
 384         ctx->tqm_ctx.pde.level = ctx->tqm_ctx.pde_level;
 385         bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.pde);
 386         bnxt_qplib_free_stats_ctx(res->pdev, &ctx->stats);
 387 }
 388
 389 static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res,
 390                                       struct bnxt_qplib_ctx *ctx)
 391 {
 392         struct bnxt_qplib_hwq_attr hwq_attr = {};
 393         struct bnxt_qplib_sg_info sginfo = {};
 394         struct bnxt_qplib_tqm_ctx *tqmctx;
 395         int rc = 0;
 396         int i;
 397
 398         tqmctx = &ctx->tqm_ctx;
 399
 400         sginfo.pgsize = PAGE_SIZE;
 401         sginfo.pgshft = PAGE_SHIFT;
 402         hwq_attr.sginfo = &sginfo;
 403         hwq_attr.res = res;
 404         hwq_attr.type = HWQ_TYPE_CTX;
 405         hwq_attr.depth = 512;
 406         hwq_attr.stride = sizeof(u64);
 407         /* Alloc pdl buffer */
 408         rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr);
 409         if (rc)
 410                 goto out;
 411         /* Save original pdl level */
 412         tqmctx->pde_level = tqmctx->pde.level;
 413
 414         hwq_attr.stride = 1;
 415         for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
 416                 if (!tqmctx->qcount[i])
 417                         continue;
 418                 hwq_attr.depth = ctx->qpc_count * tqmctx->qcount[i];
 419                 rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr);
 420                 if (rc)
 421                         goto out;
 422         }
 423 out:
 424         return rc;
 425 }
 426
 427 static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx)
 428 {
 429         struct bnxt_qplib_hwq *tbl;
 430         dma_addr_t *dma_ptr;
 431         __le64 **pbl_ptr, *ptr;
 432         int i, j, k;
 433         int fnz_idx = -1;
 434         int pg_count;
 435
 436         pbl_ptr = (__le64 **)ctx->pde.pbl_ptr;
 437
 438         for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
 439              i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
 440                 tbl = &ctx->qtbl[i];
 441                 if (!tbl->max_elements)
 442                         continue;
 443                 if (fnz_idx == -1)
 444                         fnz_idx = i; /* first non-zero index */
 445                 switch (tbl->level) {
 446                 case PBL_LVL_2:
 447                         pg_count = tbl->pbl[PBL_LVL_1].pg_count;
 448                         for (k = 0; k < pg_count; k++) {
 449                                 ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)];
 450                                 dma_ptr = &tbl->pbl[PBL_LVL_1].pg_map_arr[k];
 451                                 *ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID);
 452                         }
 453                         break;
 454                 case PBL_LVL_1:
 455                 case PBL_LVL_0:
 456                 default:
 457                         ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)];
 458                         *ptr = cpu_to_le64(tbl->pbl[PBL_LVL_0].pg_map_arr[0] |
 459                                            PTU_PTE_VALID);
 460                         break;
 461                 }
 462         }
 463         if (fnz_idx == -1)
 464                 fnz_idx = 0;
 465         /* update pde level as per page table programming */
 466         ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 :
 467                           ctx->qtbl[fnz_idx].level + 1;
 468 }
 469
 470 static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res,
 471                                       struct bnxt_qplib_ctx *ctx)
 472 {
 473         int rc = 0;
 474
 475         rc = bnxt_qplib_alloc_tqm_rings(res, ctx);
 476         if (rc)
 477                 goto fail;
 478
 479         bnxt_qplib_map_tqm_pgtbl(&ctx->tqm_ctx);
 480 fail:
 481         return rc;
 482 }
 483
 484 /*
 485  * Routine: bnxt_qplib_alloc_ctx
 486  * Description:
 487  *     Context tables are memories which are used by the chip fw.
 488  *     The 6 tables defined are:
 489  *             QPC ctx - holds QP states
 490  *             MRW ctx - holds memory region and window
 491  *             SRQ ctx - holds shared RQ states
 492  *             CQ ctx - holds completion queue states
 493  *             TQM ctx - holds Tx Queue Manager context
 494  *             TIM ctx - holds timer context
 495  *     Depending on the size of the tbl requested, either a 1 Page Buffer List
 496  *     or a 1-to-2-stage indirection Page Directory List + 1 PBL is used
 497  *     instead.
 498  *     Table might be employed as follows:
 499  *             For 0      < ctx size <= 1 PAGE, 0 level of ind is used
 500  *             For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used
 501  *             For 512    < ctx size <= MAX, 2 levels of ind is used
 502  * Returns:
 503  *     0 if success, else -ERRORS
 504  */
 505 int bnxt_qplib_alloc_ctx(struct bnxt_qplib_res *res,
 506                          struct bnxt_qplib_ctx *ctx,
 507                          bool virt_fn, bool is_p5)
 508 {
 509         struct bnxt_qplib_hwq_attr hwq_attr = {};
 510         struct bnxt_qplib_sg_info sginfo = {};
 511         int rc = 0;
 512
 513         if (virt_fn || is_p5)
 514                 goto stats_alloc;
 515
 516         /* QPC Tables */
 517         sginfo.pgsize = PAGE_SIZE;
 518         sginfo.pgshft = PAGE_SHIFT;
 519         hwq_attr.sginfo = &sginfo;
 520
 521         hwq_attr.res = res;
 522         hwq_attr.depth = ctx->qpc_count;
 523         hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE;
 524         hwq_attr.type = HWQ_TYPE_CTX;
 525         rc = bnxt_qplib_alloc_init_hwq(&ctx->qpc_tbl, &hwq_attr);
 526         if (rc)
 527                 goto fail;
 528
 529         /* MRW Tables */
 530         hwq_attr.depth = ctx->mrw_count;
 531         hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE;
 532         rc = bnxt_qplib_alloc_init_hwq(&ctx->mrw_tbl, &hwq_attr);
 533         if (rc)
 534                 goto fail;
 535
 536         /* SRQ Tables */
 537         hwq_attr.depth = ctx->srqc_count;
 538         hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE;
 539         rc = bnxt_qplib_alloc_init_hwq(&ctx->srqc_tbl, &hwq_attr);
 540         if (rc)
 541                 goto fail;
 542
 543         /* CQ Tables */
 544         hwq_attr.depth = ctx->cq_count;
 545         hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE;
 546         rc = bnxt_qplib_alloc_init_hwq(&ctx->cq_tbl, &hwq_attr);
 547         if (rc)
 548                 goto fail;
 549
 550         /* TQM Buffer */
 551         rc = bnxt_qplib_setup_tqm_rings(res, ctx);
 552         if (rc)
 553                 goto fail;
 554         /* TIM Buffer */
 555         ctx->tim_tbl.max_elements = ctx->qpc_count * 16;
 556         hwq_attr.depth = ctx->qpc_count * 16;
 557         hwq_attr.stride = 1;
 558         rc = bnxt_qplib_alloc_init_hwq(&ctx->tim_tbl, &hwq_attr);
 559         if (rc)
 560                 goto fail;
 561 stats_alloc:
 562         /* Stats */
 563         rc = bnxt_qplib_alloc_stats_ctx(res->pdev, res->cctx, &ctx->stats);
 564         if (rc)
 565                 goto fail;
 566
 567         return 0;
 568
 569 fail:
 570         bnxt_qplib_free_ctx(res, ctx);
 571         return rc;
 572 }
 573
 574 /* GUID */
 575 void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid)
 576 {
 577         u8 mac[ETH_ALEN];
 578
 579         /* MAC-48 to EUI-64 mapping */
 580         memcpy(mac, dev_addr, ETH_ALEN);
 581         guid[0] = mac[0] ^ 2;
 582         guid[1] = mac[1];
 583         guid[2] = mac[2];
 584         guid[3] = 0xff;
 585         guid[4] = 0xfe;
 586         guid[5] = mac[3];
 587         guid[6] = mac[4];
 588         guid[7] = mac[5];
 589 }
 590
 591 static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res,
 592                                      struct bnxt_qplib_sgid_tbl *sgid_tbl)
 593 {
 594         kfree(sgid_tbl->tbl);
 595         kfree(sgid_tbl->hw_id);
 596         kfree(sgid_tbl->ctx);
 597         kfree(sgid_tbl->vlan);
 598         sgid_tbl->tbl = NULL;
 599         sgid_tbl->hw_id = NULL;
 600         sgid_tbl->ctx = NULL;
 601         sgid_tbl->vlan = NULL;
 602         sgid_tbl->max = 0;
 603         sgid_tbl->active = 0;
 604 }
 605
 606 static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res,
 607                                      struct bnxt_qplib_sgid_tbl *sgid_tbl,
 608                                      u16 max)
 609 {
 610         sgid_tbl->tbl = kcalloc(max, sizeof(*sgid_tbl->tbl), GFP_KERNEL);
 611         if (!sgid_tbl->tbl)
 612                 return -ENOMEM;
 613
 614         sgid_tbl->hw_id = kcalloc(max, sizeof(u16), GFP_KERNEL);
 615         if (!sgid_tbl->hw_id)
 616                 goto out_free1;
 617
 618         sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL);
 619         if (!sgid_tbl->ctx)
 620                 goto out_free2;
 621
 622         sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL);
 623         if (!sgid_tbl->vlan)
 624                 goto out_free3;
 625
 626         sgid_tbl->max = max;
 627         return 0;
 628 out_free3:
 629         kfree(sgid_tbl->ctx);
 630         sgid_tbl->ctx = NULL;
 631 out_free2:
 632         kfree(sgid_tbl->hw_id);
 633         sgid_tbl->hw_id = NULL;
 634 out_free1:
 635         kfree(sgid_tbl->tbl);
 636         sgid_tbl->tbl = NULL;
 637         return -ENOMEM;
 638 };
 639
 640 static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,
 641                                         struct bnxt_qplib_sgid_tbl *sgid_tbl)
 642 {
 643         int i;
 644
 645         for (i = 0; i < sgid_tbl->max; i++) {
 646                 if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
 647                            sizeof(bnxt_qplib_gid_zero)))
 648                         bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i].gid,
 649                                             sgid_tbl->tbl[i].vlan_id, true);
 650         }
 651         memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max);
 652         memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
 653         memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max);
 654         sgid_tbl->active = 0;
 655 }
 656
 657 static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl,
 658                                      struct net_device *netdev)
 659 {
 660         u32 i;
 661
 662         for (i = 0; i < sgid_tbl->max; i++)
 663                 sgid_tbl->tbl[i].vlan_id = 0xffff;
 664
 665         memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
 666 }
 667
 668 static void bnxt_qplib_free_pkey_tbl(struct bnxt_qplib_res *res,
 669                                      struct bnxt_qplib_pkey_tbl *pkey_tbl)
 670 {
 671         if (!pkey_tbl->tbl)
 672                 dev_dbg(&res->pdev->dev, "PKEY tbl not present\n");
 673         else
 674                 kfree(pkey_tbl->tbl);
 675
 676         pkey_tbl->tbl = NULL;
 677         pkey_tbl->max = 0;
 678         pkey_tbl->active = 0;
 679 }
 680
 681 static int bnxt_qplib_alloc_pkey_tbl(struct bnxt_qplib_res *res,
 682                                      struct bnxt_qplib_pkey_tbl *pkey_tbl,
 683                                      u16 max)
 684 {
 685         pkey_tbl->tbl = kcalloc(max, sizeof(u16), GFP_KERNEL);
 686         if (!pkey_tbl->tbl)
 687                 return -ENOMEM;
 688
 689         pkey_tbl->max = max;
 690         return 0;
 691 };
 692
 693 /* PDs */
 694 int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd)
 695 {
 696         u32 bit_num;
 697
 698         bit_num = find_first_bit(pdt->tbl, pdt->max);
 699         if (bit_num == pdt->max)
 700                 return -ENOMEM;
 701
 702         /* Found unused PD */
 703         clear_bit(bit_num, pdt->tbl);
 704         pd->id = bit_num;
 705         return 0;
 706 }
 707
 708 int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
 709                           struct bnxt_qplib_pd_tbl *pdt,
 710                           struct bnxt_qplib_pd *pd)
 711 {
 712         if (test_and_set_bit(pd->id, pdt->tbl)) {
 713                 dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n",
 714                          pd->id);
 715                 return -EINVAL;
 716         }
 717         pd->id = 0;
 718         return 0;
 719 }
 720
 721 static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt)
 722 {
 723         kfree(pdt->tbl);
 724         pdt->tbl = NULL;
 725         pdt->max = 0;
 726 }
 727
 728 static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
 729                                    struct bnxt_qplib_pd_tbl *pdt,
 730                                    u32 max)
 731 {
 732         u32 bytes;
 733
 734         bytes = max >> 3;
 735         if (!bytes)
 736                 bytes = 1;
 737         pdt->tbl = kmalloc(bytes, GFP_KERNEL);
 738         if (!pdt->tbl)
 739                 return -ENOMEM;
 740
 741         pdt->max = max;
 742         memset((u8 *)pdt->tbl, 0xFF, bytes);
 743
 744         return 0;
 745 }
 746
 747 /* DPIs */
 748 int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit,
 749                          struct bnxt_qplib_dpi     *dpi,
 750                          void                      *app)
 751 {
 752         u32 bit_num;
 753
 754         bit_num = find_first_bit(dpit->tbl, dpit->max);
 755         if (bit_num == dpit->max)
 756                 return -ENOMEM;
 757
 758         /* Found unused DPI */
 759         clear_bit(bit_num, dpit->tbl);
 760         dpit->app_tbl[bit_num] = app;
 761
 762         dpi->dpi = bit_num;
 763         dpi->dbr = dpit->dbr_bar_reg_iomem + (bit_num * PAGE_SIZE);
 764         dpi->umdbr = dpit->unmapped_dbr + (bit_num * PAGE_SIZE);
 765
 766         return 0;
 767 }
 768
 769 int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
 770                            struct bnxt_qplib_dpi_tbl *dpit,
 771                            struct bnxt_qplib_dpi     *dpi)
 772 {
 773         if (dpi->dpi >= dpit->max) {
 774                 dev_warn(&res->pdev->dev, "Invalid DPI? dpi = %d\n", dpi->dpi);
 775                 return -EINVAL;
 776         }
 777         if (test_and_set_bit(dpi->dpi, dpit->tbl)) {
 778                 dev_warn(&res->pdev->dev, "Freeing an unused DPI? dpi = %d\n",
 779                          dpi->dpi);
 780                 return -EINVAL;
 781         }
 782         if (dpit->app_tbl)
 783                 dpit->app_tbl[dpi->dpi] = NULL;
 784         memset(dpi, 0, sizeof(*dpi));
 785
 786         return 0;
 787 }
 788
 789 static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res     *res,
 790                                     struct bnxt_qplib_dpi_tbl *dpit)
 791 {
 792         kfree(dpit->tbl);
 793         kfree(dpit->app_tbl);
 794         if (dpit->dbr_bar_reg_iomem)
 795                 pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
 796         memset(dpit, 0, sizeof(*dpit));
 797 }
 798
 799 static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res     *res,
 800                                     struct bnxt_qplib_dpi_tbl *dpit,
 801                                     u32                       dbr_offset)
 802 {
 803         u32 dbr_bar_reg = RCFW_DBR_PCI_BAR_REGION;
 804         resource_size_t bar_reg_base;
 805         u32 dbr_len, bytes;
 806
 807         if (dpit->dbr_bar_reg_iomem) {
 808                 dev_err(&res->pdev->dev, "DBR BAR region %d already mapped\n",
 809                         dbr_bar_reg);
 810                 return -EALREADY;
 811         }
 812
 813         bar_reg_base = pci_resource_start(res->pdev, dbr_bar_reg);
 814         if (!bar_reg_base) {
 815                 dev_err(&res->pdev->dev, "BAR region %d resc start failed\n",
 816                         dbr_bar_reg);
 817                 return -ENOMEM;
 818         }
 819
 820         dbr_len = pci_resource_len(res->pdev, dbr_bar_reg) - dbr_offset;
 821         if (!dbr_len || ((dbr_len & (PAGE_SIZE - 1)) != 0)) {
 822                 dev_err(&res->pdev->dev, "Invalid DBR length %d\n", dbr_len);
 823                 return -ENOMEM;
 824         }
 825
 826         dpit->dbr_bar_reg_iomem = ioremap(bar_reg_base + dbr_offset,
 827                                                   dbr_len);
 828         if (!dpit->dbr_bar_reg_iomem) {
 829                 dev_err(&res->pdev->dev,
 830                         "FP: DBR BAR region %d mapping failed\n", dbr_bar_reg);
 831                 return -ENOMEM;
 832         }
 833
 834         dpit->unmapped_dbr = bar_reg_base + dbr_offset;
 835         dpit->max = dbr_len / PAGE_SIZE;
 836
 837         dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL);
 838         if (!dpit->app_tbl)
 839                 goto unmap_io;
 840
 841         bytes = dpit->max >> 3;
 842         if (!bytes)
 843                 bytes = 1;
 844
 845         dpit->tbl = kmalloc(bytes, GFP_KERNEL);
 846         if (!dpit->tbl) {
 847                 kfree(dpit->app_tbl);
 848                 dpit->app_tbl = NULL;
 849                 goto unmap_io;
 850         }
 851
 852         memset((u8 *)dpit->tbl, 0xFF, bytes);
 853
 854         return 0;
 855
 856 unmap_io:
 857         pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
 858         dpit->dbr_bar_reg_iomem = NULL;
 859         return -ENOMEM;
 860 }
 861
 862 /* PKEYs */
 863 static void bnxt_qplib_cleanup_pkey_tbl(struct bnxt_qplib_pkey_tbl *pkey_tbl)
 864 {
 865         memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max);
 866         pkey_tbl->active = 0;
 867 }
 868
 869 static void bnxt_qplib_init_pkey_tbl(struct bnxt_qplib_res *res,
 870                                      struct bnxt_qplib_pkey_tbl *pkey_tbl)
 871 {
 872         u16 pkey = 0xFFFF;
 873
 874         memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max);
 875
 876         /* pkey default = 0xFFFF */
 877         bnxt_qplib_add_pkey(res, pkey_tbl, &pkey, false);
 878 }
 879
 880 /* Stats */
 881 static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
 882                                       struct bnxt_qplib_stats *stats)
 883 {
 884         if (stats->dma) {
 885                 dma_free_coherent(&pdev->dev, stats->size,
 886                                   stats->dma, stats->dma_map);
 887         }
 888         memset(stats, 0, sizeof(*stats));
 889         stats->fw_id = -1;
 890 }
 891
 892 static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
 893                                       struct bnxt_qplib_chip_ctx *cctx,
 894                                       struct bnxt_qplib_stats *stats)
 895 {
 896         memset(stats, 0, sizeof(*stats));
 897         stats->fw_id = -1;
 898         stats->size = cctx->hw_stats_size;
 899         stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
 900                                         &stats->dma_map, GFP_KERNEL);
 901         if (!stats->dma) {
 902                 dev_err(&pdev->dev, "Stats DMA allocation failed\n");
 903                 return -ENOMEM;
 904         }
 905         return 0;
 906 }
 907
 908 void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res)
 909 {
 910         bnxt_qplib_cleanup_pkey_tbl(&res->pkey_tbl);
 911         bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);
 912 }
 913
 914 int bnxt_qplib_init_res(struct bnxt_qplib_res *res)
 915 {
 916         bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);
 917         bnxt_qplib_init_pkey_tbl(res, &res->pkey_tbl);
 918
 919         return 0;
 920 }
 921
 922 void bnxt_qplib_free_res(struct bnxt_qplib_res *res)
 923 {
 924         bnxt_qplib_free_pkey_tbl(res, &res->pkey_tbl);
 925         bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);
 926         bnxt_qplib_free_pd_tbl(&res->pd_tbl);
 927         bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);
 928 }
 929
 930 int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,
 931                          struct net_device *netdev,
 932                          struct bnxt_qplib_dev_attr *dev_attr)
 933 {
 934         int rc = 0;
 935
 936         res->pdev = pdev;
 937         res->netdev = netdev;
 938
 939         rc = bnxt_qplib_alloc_sgid_tbl(res, &res->sgid_tbl, dev_attr->max_sgid);
 940         if (rc)
 941                 goto fail;
 942
 943         rc = bnxt_qplib_alloc_pkey_tbl(res, &res->pkey_tbl, dev_attr->max_pkey);
 944         if (rc)
 945                 goto fail;
 946
 947         rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd);
 948         if (rc)
 949                 goto fail;
 950
 951         rc = bnxt_qplib_alloc_dpi_tbl(res, &res->dpi_tbl, dev_attr->l2_db_size);
 952         if (rc)
 953                 goto fail;
 954
 955         return 0;
 956 fail:
 957         bnxt_qplib_free_res(res);
 958         return rc;
 959 }
 960
 961 int bnxt_qplib_determine_atomics(struct pci_dev *dev)
 962 {
 963         int comp;
 964         u16 ctl2;
 965
 966         comp = pci_enable_atomic_ops_to_root(dev,
 967                                              PCI_EXP_DEVCAP2_ATOMIC_COMP32);
 968         if (comp)
 969                 return -EOPNOTSUPP;
 970         comp = pci_enable_atomic_ops_to_root(dev,
 971                                              PCI_EXP_DEVCAP2_ATOMIC_COMP64);
 972         if (comp)
 973                 return -EOPNOTSUPP;
 974         pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &ctl2);
 975         return !(ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
 976 }