drivers/infiniband/hw/bnxt_re/qplib_res.c

   1 /*
   2  * Broadcom NetXtreme-E RoCE driver.
   3  *
   4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * BSD license below:
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in
  21  *    the documentation and/or other materials provided with the
  22  *    distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Description: QPLib resource manager
  37  */
  38
  39 #define dev_fmt(fmt) "QPLIB: " fmt
  40
  41 #include <linux/spinlock.h>
  42 #include <linux/pci.h>
  43 #include <linux/interrupt.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/dma-mapping.h>
  46 #include <linux/if_vlan.h>
  47 #include <linux/vmalloc.h>
  48 #include <rdma/ib_verbs.h>
  49 #include <rdma/ib_umem.h>
  50
  51 #include "roce_hsi.h"
  52 #include "qplib_res.h"
  53 #include "qplib_sp.h"
  54 #include "qplib_rcfw.h"
  55
  56 static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
  57                                       struct bnxt_qplib_stats *stats);
  58 static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
  59                                       struct bnxt_qplib_stats *stats);
  60
  61 /* PBL */
  62 static void __free_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl,
  63                        bool is_umem)
  64 {
  65         struct pci_dev *pdev = res->pdev;
  66         int i;
  67
  68         if (!is_umem) {
  69                 for (i = 0; i < pbl->pg_count; i++) {
  70                         if (pbl->pg_arr[i])
  71                                 dma_free_coherent(&pdev->dev, pbl->pg_size,
  72                                                   (void *)((unsigned long)
  73                                                    pbl->pg_arr[i] &
  74                                                   PAGE_MASK),
  75                                                   pbl->pg_map_arr[i]);
  76                         else
  77                                 dev_warn(&pdev->dev,
  78                                          "PBL free pg_arr[%d] empty?!\n", i);
  79                         pbl->pg_arr[i] = NULL;
  80                 }
  81         }
  82         vfree(pbl->pg_arr);
  83         pbl->pg_arr = NULL;
  84         vfree(pbl->pg_map_arr);
  85         pbl->pg_map_arr = NULL;
  86         pbl->pg_count = 0;
  87         pbl->pg_size = 0;
  88 }
  89
  90 static void bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,
  91                                            struct bnxt_qplib_sg_info *sginfo)
  92 {
  93         struct ib_block_iter biter;
  94         int i = 0;
  95
  96         rdma_umem_for_each_dma_block(sginfo->umem, &biter, sginfo->pgsize) {
  97                 pbl->pg_map_arr[i] = rdma_block_iter_dma_address(&biter);
  98                 pbl->pg_arr[i] = NULL;
  99                 pbl->pg_count++;
 100                 i++;
 101         }
 102 }
 103
 104 static int __alloc_pbl(struct bnxt_qplib_res *res,
 105                        struct bnxt_qplib_pbl *pbl,
 106                        struct bnxt_qplib_sg_info *sginfo)
 107 {
 108         struct pci_dev *pdev = res->pdev;
 109         bool is_umem = false;
 110         u32 pages;
 111         int i;
 112
 113         if (sginfo->nopte)
 114                 return 0;
 115         if (sginfo->umem)
 116                 pages = ib_umem_num_dma_blocks(sginfo->umem, sginfo->pgsize);
 117         else
 118                 pages = sginfo->npages;
 119         /* page ptr arrays */
 120         pbl->pg_arr = vmalloc(pages * sizeof(void *));
 121         if (!pbl->pg_arr)
 122                 return -ENOMEM;
 123
 124         pbl->pg_map_arr = vmalloc(pages * sizeof(dma_addr_t));
 125         if (!pbl->pg_map_arr) {
 126                 vfree(pbl->pg_arr);
 127                 pbl->pg_arr = NULL;
 128                 return -ENOMEM;
 129         }
 130         pbl->pg_count = 0;
 131         pbl->pg_size = sginfo->pgsize;
 132
 133         if (!sginfo->umem) {
 134                 for (i = 0; i < pages; i++) {
 135                         pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
 136                                                             pbl->pg_size,
 137                                                             &pbl->pg_map_arr[i],
 138                                                             GFP_KERNEL);
 139                         if (!pbl->pg_arr[i])
 140                                 goto fail;
 141                         pbl->pg_count++;
 142                 }
 143         } else {
 144                 is_umem = true;
 145                 bnxt_qplib_fill_user_dma_pages(pbl, sginfo);
 146         }
 147
 148         return 0;
 149 fail:
 150         __free_pbl(res, pbl, is_umem);
 151         return -ENOMEM;
 152 }
 153
 154 /* HWQ */
 155 void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
 156                          struct bnxt_qplib_hwq *hwq)
 157 {
 158         int i;
 159
 160         if (!hwq->max_elements)
 161                 return;
 162         if (hwq->level >= PBL_LVL_MAX)
 163                 return;
 164
 165         for (i = 0; i < hwq->level + 1; i++) {
 166                 if (i == hwq->level)
 167                         __free_pbl(res, &hwq->pbl[i], hwq->is_user);
 168                 else
 169                         __free_pbl(res, &hwq->pbl[i], false);
 170         }
 171
 172         hwq->level = PBL_LVL_MAX;
 173         hwq->max_elements = 0;
 174         hwq->element_size = 0;
 175         hwq->prod = 0;
 176         hwq->cons = 0;
 177         hwq->cp_bit = 0;
 178 }
 179
 180 /* All HWQs are power of 2 in size */
 181
 182 int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
 183                               struct bnxt_qplib_hwq_attr *hwq_attr)
 184 {
 185         u32 npages, aux_slots, pg_size, aux_pages = 0, aux_size = 0;
 186         struct bnxt_qplib_sg_info sginfo = {};
 187         u32 depth, stride, npbl, npde;
 188         dma_addr_t *src_phys_ptr, **dst_virt_ptr;
 189         struct bnxt_qplib_res *res;
 190         struct pci_dev *pdev;
 191         int i, rc, lvl;
 192
 193         res = hwq_attr->res;
 194         pdev = res->pdev;
 195         pg_size = hwq_attr->sginfo->pgsize;
 196         hwq->level = PBL_LVL_MAX;
 197
 198         depth = roundup_pow_of_two(hwq_attr->depth);
 199         stride = roundup_pow_of_two(hwq_attr->stride);
 200         if (hwq_attr->aux_depth) {
 201                 aux_slots = hwq_attr->aux_depth;
 202                 aux_size = roundup_pow_of_two(hwq_attr->aux_stride);
 203                 aux_pages = (aux_slots * aux_size) / pg_size;
 204                 if ((aux_slots * aux_size) % pg_size)
 205                         aux_pages++;
 206         }
 207
 208         if (!hwq_attr->sginfo->umem) {
 209                 hwq->is_user = false;
 210                 npages = (depth * stride) / pg_size + aux_pages;
 211                 if ((depth * stride) % pg_size)
 212                         npages++;
 213                 if (!npages)
 214                         return -EINVAL;
 215                 hwq_attr->sginfo->npages = npages;
 216         } else {
 217                 unsigned long sginfo_num_pages = ib_umem_num_dma_blocks(
 218                         hwq_attr->sginfo->umem, hwq_attr->sginfo->pgsize);
 219
 220                 hwq->is_user = true;
 221                 npages = sginfo_num_pages;
 222                 npages = (npages * PAGE_SIZE) /
 223                           BIT_ULL(hwq_attr->sginfo->pgshft);
 224                 if ((sginfo_num_pages * PAGE_SIZE) %
 225                      BIT_ULL(hwq_attr->sginfo->pgshft))
 226                         if (!npages)
 227                                 npages++;
 228         }
 229
 230         if (npages == MAX_PBL_LVL_0_PGS) {
 231                 /* This request is Level 0, map PTE */
 232                 rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo);
 233                 if (rc)
 234                         goto fail;
 235                 hwq->level = PBL_LVL_0;
 236         }
 237
 238         if (npages > MAX_PBL_LVL_0_PGS) {
 239                 if (npages > MAX_PBL_LVL_1_PGS) {
 240                         u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
 241                                     0 : PTU_PTE_VALID;
 242                         /* 2 levels of indirection */
 243                         npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
 244                         if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
 245                                 npbl++;
 246                         npde = npbl >> MAX_PDL_LVL_SHIFT;
 247                         if (npbl % BIT(MAX_PDL_LVL_SHIFT))
 248                                 npde++;
 249                         /* Alloc PDE pages */
 250                         sginfo.pgsize = npde * pg_size;
 251                         sginfo.npages = 1;
 252                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
 253
 254                         /* Alloc PBL pages */
 255                         sginfo.npages = npbl;
 256                         sginfo.pgsize = PAGE_SIZE;
 257                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo);
 258                         if (rc)
 259                                 goto fail;
 260                         /* Fill PDL with PBL page pointers */
 261                         dst_virt_ptr =
 262                                 (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
 263                         src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
 264                         if (hwq_attr->type == HWQ_TYPE_MR) {
 265                         /* For MR it is expected that we supply only 1 contigous
 266                          * page i.e only 1 entry in the PDL that will contain
 267                          * all the PBLs for the user supplied memory region
 268                          */
 269                                 for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count;
 270                                      i++)
 271                                         dst_virt_ptr[0][i] = src_phys_ptr[i] |
 272                                                 flag;
 273                         } else {
 274                                 for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count;
 275                                      i++)
 276                                         dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
 277                                                 src_phys_ptr[i] |
 278                                                 PTU_PDE_VALID;
 279                         }
 280                         /* Alloc or init PTEs */
 281                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2],
 282                                          hwq_attr->sginfo);
 283                         if (rc)
 284                                 goto fail;
 285                         hwq->level = PBL_LVL_2;
 286                         if (hwq_attr->sginfo->nopte)
 287                                 goto done;
 288                         /* Fill PBLs with PTE pointers */
 289                         dst_virt_ptr =
 290                                 (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr;
 291                         src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr;
 292                         for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) {
 293                                 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
 294                                         src_phys_ptr[i] | PTU_PTE_VALID;
 295                         }
 296                         if (hwq_attr->type == HWQ_TYPE_QUEUE) {
 297                                 /* Find the last pg of the size */
 298                                 i = hwq->pbl[PBL_LVL_2].pg_count;
 299                                 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
 300                                                                   PTU_PTE_LAST;
 301                                 if (i > 1)
 302                                         dst_virt_ptr[PTR_PG(i - 2)]
 303                                                     [PTR_IDX(i - 2)] |=
 304                                                     PTU_PTE_NEXT_TO_LAST;
 305                         }
 306                 } else { /* pages < 512 npbl = 1, npde = 0 */
 307                         u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
 308                                     0 : PTU_PTE_VALID;
 309
 310                         /* 1 level of indirection */
 311                         npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
 312                         if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
 313                                 npbl++;
 314                         sginfo.npages = npbl;
 315                         sginfo.pgsize = PAGE_SIZE;
 316                         /* Alloc PBL page */
 317                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
 318                         if (rc)
 319                                 goto fail;
 320                         /* Alloc or init  PTEs */
 321                         rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1],
 322                                          hwq_attr->sginfo);
 323                         if (rc)
 324                                 goto fail;
 325                         hwq->level = PBL_LVL_1;
 326                         if (hwq_attr->sginfo->nopte)
 327                                 goto done;
 328                         /* Fill PBL with PTE pointers */
 329                         dst_virt_ptr =
 330                                 (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
 331                         src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
 332                         for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
 333                                 dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
 334                                         src_phys_ptr[i] | flag;
 335                         if (hwq_attr->type == HWQ_TYPE_QUEUE) {
 336                                 /* Find the last pg of the size */
 337                                 i = hwq->pbl[PBL_LVL_1].pg_count;
 338                                 dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
 339                                                                   PTU_PTE_LAST;
 340                                 if (i > 1)
 341                                         dst_virt_ptr[PTR_PG(i - 2)]
 342                                                     [PTR_IDX(i - 2)] |=
 343                                                     PTU_PTE_NEXT_TO_LAST;
 344                         }
 345                 }
 346         }
 347 done:
 348         hwq->prod = 0;
 349         hwq->cons = 0;
 350         hwq->pdev = pdev;
 351         hwq->depth = hwq_attr->depth;
 352         hwq->max_elements = depth;
 353         hwq->element_size = stride;
 354         hwq->qe_ppg = pg_size / stride;
 355         /* For direct access to the elements */
 356         lvl = hwq->level;
 357         if (hwq_attr->sginfo->nopte && hwq->level)
 358                 lvl = hwq->level - 1;
 359         hwq->pbl_ptr = hwq->pbl[lvl].pg_arr;
 360         hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr;
 361         spin_lock_init(&hwq->lock);
 362
 363         return 0;
 364 fail:
 365         bnxt_qplib_free_hwq(res, hwq);
 366         return -ENOMEM;
 367 }
 368
 369 /* Context Tables */
 370 void bnxt_qplib_free_ctx(struct bnxt_qplib_res *res,
 371                          struct bnxt_qplib_ctx *ctx)
 372 {
 373         int i;
 374
 375         bnxt_qplib_free_hwq(res, &ctx->qpc_tbl);
 376         bnxt_qplib_free_hwq(res, &ctx->mrw_tbl);
 377         bnxt_qplib_free_hwq(res, &ctx->srqc_tbl);
 378         bnxt_qplib_free_hwq(res, &ctx->cq_tbl);
 379         bnxt_qplib_free_hwq(res, &ctx->tim_tbl);
 380         for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
 381                 bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.qtbl[i]);
 382         /* restore original pde level before destroy */
 383         ctx->tqm_ctx.pde.level = ctx->tqm_ctx.pde_level;
 384         bnxt_qplib_free_hwq(res, &ctx->tqm_ctx.pde);
 385         bnxt_qplib_free_stats_ctx(res->pdev, &ctx->stats);
 386 }
 387
 388 static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res,
 389                                       struct bnxt_qplib_ctx *ctx)
 390 {
 391         struct bnxt_qplib_hwq_attr hwq_attr = {};
 392         struct bnxt_qplib_sg_info sginfo = {};
 393         struct bnxt_qplib_tqm_ctx *tqmctx;
 394         int rc = 0;
 395         int i;
 396
 397         tqmctx = &ctx->tqm_ctx;
 398
 399         sginfo.pgsize = PAGE_SIZE;
 400         sginfo.pgshft = PAGE_SHIFT;
 401         hwq_attr.sginfo = &sginfo;
 402         hwq_attr.res = res;
 403         hwq_attr.type = HWQ_TYPE_CTX;
 404         hwq_attr.depth = 512;
 405         hwq_attr.stride = sizeof(u64);
 406         /* Alloc pdl buffer */
 407         rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr);
 408         if (rc)
 409                 goto out;
 410         /* Save original pdl level */
 411         tqmctx->pde_level = tqmctx->pde.level;
 412
 413         hwq_attr.stride = 1;
 414         for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
 415                 if (!tqmctx->qcount[i])
 416                         continue;
 417                 hwq_attr.depth = ctx->qpc_count * tqmctx->qcount[i];
 418                 rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr);
 419                 if (rc)
 420                         goto out;
 421         }
 422 out:
 423         return rc;
 424 }
 425
 426 static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx)
 427 {
 428         struct bnxt_qplib_hwq *tbl;
 429         dma_addr_t *dma_ptr;
 430         __le64 **pbl_ptr, *ptr;
 431         int i, j, k;
 432         int fnz_idx = -1;
 433         int pg_count;
 434
 435         pbl_ptr = (__le64 **)ctx->pde.pbl_ptr;
 436
 437         for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
 438              i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
 439                 tbl = &ctx->qtbl[i];
 440                 if (!tbl->max_elements)
 441                         continue;
 442                 if (fnz_idx == -1)
 443                         fnz_idx = i; /* first non-zero index */
 444                 switch (tbl->level) {
 445                 case PBL_LVL_2:
 446                         pg_count = tbl->pbl[PBL_LVL_1].pg_count;
 447                         for (k = 0; k < pg_count; k++) {
 448                                 ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)];
 449                                 dma_ptr = &tbl->pbl[PBL_LVL_1].pg_map_arr[k];
 450                                 *ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID);
 451                         }
 452                         break;
 453                 case PBL_LVL_1:
 454                 case PBL_LVL_0:
 455                 default:
 456                         ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)];
 457                         *ptr = cpu_to_le64(tbl->pbl[PBL_LVL_0].pg_map_arr[0] |
 458                                            PTU_PTE_VALID);
 459                         break;
 460                 }
 461         }
 462         if (fnz_idx == -1)
 463                 fnz_idx = 0;
 464         /* update pde level as per page table programming */
 465         ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 :
 466                           ctx->qtbl[fnz_idx].level + 1;
 467 }
 468
 469 static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res,
 470                                       struct bnxt_qplib_ctx *ctx)
 471 {
 472         int rc = 0;
 473
 474         rc = bnxt_qplib_alloc_tqm_rings(res, ctx);
 475         if (rc)
 476                 goto fail;
 477
 478         bnxt_qplib_map_tqm_pgtbl(&ctx->tqm_ctx);
 479 fail:
 480         return rc;
 481 }
 482
 483 /*
 484  * Routine: bnxt_qplib_alloc_ctx
 485  * Description:
 486  *     Context tables are memories which are used by the chip fw.
 487  *     The 6 tables defined are:
 488  *             QPC ctx - holds QP states
 489  *             MRW ctx - holds memory region and window
 490  *             SRQ ctx - holds shared RQ states
 491  *             CQ ctx - holds completion queue states
 492  *             TQM ctx - holds Tx Queue Manager context
 493  *             TIM ctx - holds timer context
 494  *     Depending on the size of the tbl requested, either a 1 Page Buffer List
 495  *     or a 1-to-2-stage indirection Page Directory List + 1 PBL is used
 496  *     instead.
 497  *     Table might be employed as follows:
 498  *             For 0      < ctx size <= 1 PAGE, 0 level of ind is used
 499  *             For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used
 500  *             For 512    < ctx size <= MAX, 2 levels of ind is used
 501  * Returns:
 502  *     0 if success, else -ERRORS
 503  */
 504 int bnxt_qplib_alloc_ctx(struct bnxt_qplib_res *res,
 505                          struct bnxt_qplib_ctx *ctx,
 506                          bool virt_fn, bool is_p5)
 507 {
 508         struct bnxt_qplib_hwq_attr hwq_attr = {};
 509         struct bnxt_qplib_sg_info sginfo = {};
 510         int rc = 0;
 511
 512         if (virt_fn || is_p5)
 513                 goto stats_alloc;
 514
 515         /* QPC Tables */
 516         sginfo.pgsize = PAGE_SIZE;
 517         sginfo.pgshft = PAGE_SHIFT;
 518         hwq_attr.sginfo = &sginfo;
 519
 520         hwq_attr.res = res;
 521         hwq_attr.depth = ctx->qpc_count;
 522         hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE;
 523         hwq_attr.type = HWQ_TYPE_CTX;
 524         rc = bnxt_qplib_alloc_init_hwq(&ctx->qpc_tbl, &hwq_attr);
 525         if (rc)
 526                 goto fail;
 527
 528         /* MRW Tables */
 529         hwq_attr.depth = ctx->mrw_count;
 530         hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE;
 531         rc = bnxt_qplib_alloc_init_hwq(&ctx->mrw_tbl, &hwq_attr);
 532         if (rc)
 533                 goto fail;
 534
 535         /* SRQ Tables */
 536         hwq_attr.depth = ctx->srqc_count;
 537         hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE;
 538         rc = bnxt_qplib_alloc_init_hwq(&ctx->srqc_tbl, &hwq_attr);
 539         if (rc)
 540                 goto fail;
 541
 542         /* CQ Tables */
 543         hwq_attr.depth = ctx->cq_count;
 544         hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE;
 545         rc = bnxt_qplib_alloc_init_hwq(&ctx->cq_tbl, &hwq_attr);
 546         if (rc)
 547                 goto fail;
 548
 549         /* TQM Buffer */
 550         rc = bnxt_qplib_setup_tqm_rings(res, ctx);
 551         if (rc)
 552                 goto fail;
 553         /* TIM Buffer */
 554         ctx->tim_tbl.max_elements = ctx->qpc_count * 16;
 555         hwq_attr.depth = ctx->qpc_count * 16;
 556         hwq_attr.stride = 1;
 557         rc = bnxt_qplib_alloc_init_hwq(&ctx->tim_tbl, &hwq_attr);
 558         if (rc)
 559                 goto fail;
 560 stats_alloc:
 561         /* Stats */
 562         rc = bnxt_qplib_alloc_stats_ctx(res->pdev, &ctx->stats);
 563         if (rc)
 564                 goto fail;
 565
 566         return 0;
 567
 568 fail:
 569         bnxt_qplib_free_ctx(res, ctx);
 570         return rc;
 571 }
 572
 573 /* GUID */
 574 void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid)
 575 {
 576         u8 mac[ETH_ALEN];
 577
 578         /* MAC-48 to EUI-64 mapping */
 579         memcpy(mac, dev_addr, ETH_ALEN);
 580         guid[0] = mac[0] ^ 2;
 581         guid[1] = mac[1];
 582         guid[2] = mac[2];
 583         guid[3] = 0xff;
 584         guid[4] = 0xfe;
 585         guid[5] = mac[3];
 586         guid[6] = mac[4];
 587         guid[7] = mac[5];
 588 }
 589
 590 static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res,
 591                                      struct bnxt_qplib_sgid_tbl *sgid_tbl)
 592 {
 593         kfree(sgid_tbl->tbl);
 594         kfree(sgid_tbl->hw_id);
 595         kfree(sgid_tbl->ctx);
 596         kfree(sgid_tbl->vlan);
 597         sgid_tbl->tbl = NULL;
 598         sgid_tbl->hw_id = NULL;
 599         sgid_tbl->ctx = NULL;
 600         sgid_tbl->vlan = NULL;
 601         sgid_tbl->max = 0;
 602         sgid_tbl->active = 0;
 603 }
 604
 605 static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res,
 606                                      struct bnxt_qplib_sgid_tbl *sgid_tbl,
 607                                      u16 max)
 608 {
 609         sgid_tbl->tbl = kcalloc(max, sizeof(*sgid_tbl->tbl), GFP_KERNEL);
 610         if (!sgid_tbl->tbl)
 611                 return -ENOMEM;
 612
 613         sgid_tbl->hw_id = kcalloc(max, sizeof(u16), GFP_KERNEL);
 614         if (!sgid_tbl->hw_id)
 615                 goto out_free1;
 616
 617         sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL);
 618         if (!sgid_tbl->ctx)
 619                 goto out_free2;
 620
 621         sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL);
 622         if (!sgid_tbl->vlan)
 623                 goto out_free3;
 624
 625         sgid_tbl->max = max;
 626         return 0;
 627 out_free3:
 628         kfree(sgid_tbl->ctx);
 629         sgid_tbl->ctx = NULL;
 630 out_free2:
 631         kfree(sgid_tbl->hw_id);
 632         sgid_tbl->hw_id = NULL;
 633 out_free1:
 634         kfree(sgid_tbl->tbl);
 635         sgid_tbl->tbl = NULL;
 636         return -ENOMEM;
 637 };
 638
 639 static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,
 640                                         struct bnxt_qplib_sgid_tbl *sgid_tbl)
 641 {
 642         int i;
 643
 644         for (i = 0; i < sgid_tbl->max; i++) {
 645                 if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
 646                            sizeof(bnxt_qplib_gid_zero)))
 647                         bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i].gid,
 648                                             sgid_tbl->tbl[i].vlan_id, true);
 649         }
 650         memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max);
 651         memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
 652         memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max);
 653         sgid_tbl->active = 0;
 654 }
 655
 656 static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl,
 657                                      struct net_device *netdev)
 658 {
 659         u32 i;
 660
 661         for (i = 0; i < sgid_tbl->max; i++)
 662                 sgid_tbl->tbl[i].vlan_id = 0xffff;
 663
 664         memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
 665 }
 666
 667 static void bnxt_qplib_free_pkey_tbl(struct bnxt_qplib_res *res,
 668                                      struct bnxt_qplib_pkey_tbl *pkey_tbl)
 669 {
 670         if (!pkey_tbl->tbl)
 671                 dev_dbg(&res->pdev->dev, "PKEY tbl not present\n");
 672         else
 673                 kfree(pkey_tbl->tbl);
 674
 675         pkey_tbl->tbl = NULL;
 676         pkey_tbl->max = 0;
 677         pkey_tbl->active = 0;
 678 }
 679
 680 static int bnxt_qplib_alloc_pkey_tbl(struct bnxt_qplib_res *res,
 681                                      struct bnxt_qplib_pkey_tbl *pkey_tbl,
 682                                      u16 max)
 683 {
 684         pkey_tbl->tbl = kcalloc(max, sizeof(u16), GFP_KERNEL);
 685         if (!pkey_tbl->tbl)
 686                 return -ENOMEM;
 687
 688         pkey_tbl->max = max;
 689         return 0;
 690 };
 691
 692 /* PDs */
 693 int bnxt_qplib_alloc_pd(struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd)
 694 {
 695         u32 bit_num;
 696
 697         bit_num = find_first_bit(pdt->tbl, pdt->max);
 698         if (bit_num == pdt->max)
 699                 return -ENOMEM;
 700
 701         /* Found unused PD */
 702         clear_bit(bit_num, pdt->tbl);
 703         pd->id = bit_num;
 704         return 0;
 705 }
 706
 707 int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
 708                           struct bnxt_qplib_pd_tbl *pdt,
 709                           struct bnxt_qplib_pd *pd)
 710 {
 711         if (test_and_set_bit(pd->id, pdt->tbl)) {
 712                 dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n",
 713                          pd->id);
 714                 return -EINVAL;
 715         }
 716         pd->id = 0;
 717         return 0;
 718 }
 719
 720 static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt)
 721 {
 722         kfree(pdt->tbl);
 723         pdt->tbl = NULL;
 724         pdt->max = 0;
 725 }
 726
 727 static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
 728                                    struct bnxt_qplib_pd_tbl *pdt,
 729                                    u32 max)
 730 {
 731         u32 bytes;
 732
 733         bytes = max >> 3;
 734         if (!bytes)
 735                 bytes = 1;
 736         pdt->tbl = kmalloc(bytes, GFP_KERNEL);
 737         if (!pdt->tbl)
 738                 return -ENOMEM;
 739
 740         pdt->max = max;
 741         memset((u8 *)pdt->tbl, 0xFF, bytes);
 742
 743         return 0;
 744 }
 745
 746 /* DPIs */
 747 int bnxt_qplib_alloc_dpi(struct bnxt_qplib_dpi_tbl *dpit,
 748                          struct bnxt_qplib_dpi     *dpi,
 749                          void                      *app)
 750 {
 751         u32 bit_num;
 752
 753         bit_num = find_first_bit(dpit->tbl, dpit->max);
 754         if (bit_num == dpit->max)
 755                 return -ENOMEM;
 756
 757         /* Found unused DPI */
 758         clear_bit(bit_num, dpit->tbl);
 759         dpit->app_tbl[bit_num] = app;
 760
 761         dpi->dpi = bit_num;
 762         dpi->dbr = dpit->dbr_bar_reg_iomem + (bit_num * PAGE_SIZE);
 763         dpi->umdbr = dpit->unmapped_dbr + (bit_num * PAGE_SIZE);
 764
 765         return 0;
 766 }
 767
 768 int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
 769                            struct bnxt_qplib_dpi_tbl *dpit,
 770                            struct bnxt_qplib_dpi     *dpi)
 771 {
 772         if (dpi->dpi >= dpit->max) {
 773                 dev_warn(&res->pdev->dev, "Invalid DPI? dpi = %d\n", dpi->dpi);
 774                 return -EINVAL;
 775         }
 776         if (test_and_set_bit(dpi->dpi, dpit->tbl)) {
 777                 dev_warn(&res->pdev->dev, "Freeing an unused DPI? dpi = %d\n",
 778                          dpi->dpi);
 779                 return -EINVAL;
 780         }
 781         if (dpit->app_tbl)
 782                 dpit->app_tbl[dpi->dpi] = NULL;
 783         memset(dpi, 0, sizeof(*dpi));
 784
 785         return 0;
 786 }
 787
 788 static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_res     *res,
 789                                     struct bnxt_qplib_dpi_tbl *dpit)
 790 {
 791         kfree(dpit->tbl);
 792         kfree(dpit->app_tbl);
 793         if (dpit->dbr_bar_reg_iomem)
 794                 pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
 795         memset(dpit, 0, sizeof(*dpit));
 796 }
 797
 798 static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res     *res,
 799                                     struct bnxt_qplib_dpi_tbl *dpit,
 800                                     u32                       dbr_offset)
 801 {
 802         u32 dbr_bar_reg = RCFW_DBR_PCI_BAR_REGION;
 803         resource_size_t bar_reg_base;
 804         u32 dbr_len, bytes;
 805
 806         if (dpit->dbr_bar_reg_iomem) {
 807                 dev_err(&res->pdev->dev, "DBR BAR region %d already mapped\n",
 808                         dbr_bar_reg);
 809                 return -EALREADY;
 810         }
 811
 812         bar_reg_base = pci_resource_start(res->pdev, dbr_bar_reg);
 813         if (!bar_reg_base) {
 814                 dev_err(&res->pdev->dev, "BAR region %d resc start failed\n",
 815                         dbr_bar_reg);
 816                 return -ENOMEM;
 817         }
 818
 819         dbr_len = pci_resource_len(res->pdev, dbr_bar_reg) - dbr_offset;
 820         if (!dbr_len || ((dbr_len & (PAGE_SIZE - 1)) != 0)) {
 821                 dev_err(&res->pdev->dev, "Invalid DBR length %d\n", dbr_len);
 822                 return -ENOMEM;
 823         }
 824
 825         dpit->dbr_bar_reg_iomem = ioremap(bar_reg_base + dbr_offset,
 826                                                   dbr_len);
 827         if (!dpit->dbr_bar_reg_iomem) {
 828                 dev_err(&res->pdev->dev,
 829                         "FP: DBR BAR region %d mapping failed\n", dbr_bar_reg);
 830                 return -ENOMEM;
 831         }
 832
 833         dpit->unmapped_dbr = bar_reg_base + dbr_offset;
 834         dpit->max = dbr_len / PAGE_SIZE;
 835
 836         dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL);
 837         if (!dpit->app_tbl)
 838                 goto unmap_io;
 839
 840         bytes = dpit->max >> 3;
 841         if (!bytes)
 842                 bytes = 1;
 843
 844         dpit->tbl = kmalloc(bytes, GFP_KERNEL);
 845         if (!dpit->tbl) {
 846                 kfree(dpit->app_tbl);
 847                 dpit->app_tbl = NULL;
 848                 goto unmap_io;
 849         }
 850
 851         memset((u8 *)dpit->tbl, 0xFF, bytes);
 852
 853         return 0;
 854
 855 unmap_io:
 856         pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
 857         dpit->dbr_bar_reg_iomem = NULL;
 858         return -ENOMEM;
 859 }
 860
 861 /* PKEYs */
 862 static void bnxt_qplib_cleanup_pkey_tbl(struct bnxt_qplib_pkey_tbl *pkey_tbl)
 863 {
 864         memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max);
 865         pkey_tbl->active = 0;
 866 }
 867
 868 static void bnxt_qplib_init_pkey_tbl(struct bnxt_qplib_res *res,
 869                                      struct bnxt_qplib_pkey_tbl *pkey_tbl)
 870 {
 871         u16 pkey = 0xFFFF;
 872
 873         memset(pkey_tbl->tbl, 0, sizeof(u16) * pkey_tbl->max);
 874
 875         /* pkey default = 0xFFFF */
 876         bnxt_qplib_add_pkey(res, pkey_tbl, &pkey, false);
 877 }
 878
 879 /* Stats */
 880 static void bnxt_qplib_free_stats_ctx(struct pci_dev *pdev,
 881                                       struct bnxt_qplib_stats *stats)
 882 {
 883         if (stats->dma) {
 884                 dma_free_coherent(&pdev->dev, stats->size,
 885                                   stats->dma, stats->dma_map);
 886         }
 887         memset(stats, 0, sizeof(*stats));
 888         stats->fw_id = -1;
 889 }
 890
 891 static int bnxt_qplib_alloc_stats_ctx(struct pci_dev *pdev,
 892                                       struct bnxt_qplib_stats *stats)
 893 {
 894         memset(stats, 0, sizeof(*stats));
 895         stats->fw_id = -1;
 896         /* 128 byte aligned context memory is required only for 57500.
 897          * However making this unconditional, it does not harm previous
 898          * generation.
 899          */
 900         stats->size = ALIGN(sizeof(struct ctx_hw_stats), 128);
 901         stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
 902                                         &stats->dma_map, GFP_KERNEL);
 903         if (!stats->dma) {
 904                 dev_err(&pdev->dev, "Stats DMA allocation failed\n");
 905                 return -ENOMEM;
 906         }
 907         return 0;
 908 }
 909
 910 void bnxt_qplib_cleanup_res(struct bnxt_qplib_res *res)
 911 {
 912         bnxt_qplib_cleanup_pkey_tbl(&res->pkey_tbl);
 913         bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);
 914 }
 915
 916 int bnxt_qplib_init_res(struct bnxt_qplib_res *res)
 917 {
 918         bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);
 919         bnxt_qplib_init_pkey_tbl(res, &res->pkey_tbl);
 920
 921         return 0;
 922 }
 923
 924 void bnxt_qplib_free_res(struct bnxt_qplib_res *res)
 925 {
 926         bnxt_qplib_free_pkey_tbl(res, &res->pkey_tbl);
 927         bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);
 928         bnxt_qplib_free_pd_tbl(&res->pd_tbl);
 929         bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);
 930 }
 931
 932 int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct pci_dev *pdev,
 933                          struct net_device *netdev,
 934                          struct bnxt_qplib_dev_attr *dev_attr)
 935 {
 936         int rc = 0;
 937
 938         res->pdev = pdev;
 939         res->netdev = netdev;
 940
 941         rc = bnxt_qplib_alloc_sgid_tbl(res, &res->sgid_tbl, dev_attr->max_sgid);
 942         if (rc)
 943                 goto fail;
 944
 945         rc = bnxt_qplib_alloc_pkey_tbl(res, &res->pkey_tbl, dev_attr->max_pkey);
 946         if (rc)
 947                 goto fail;
 948
 949         rc = bnxt_qplib_alloc_pd_tbl(res, &res->pd_tbl, dev_attr->max_pd);
 950         if (rc)
 951                 goto fail;
 952
 953         rc = bnxt_qplib_alloc_dpi_tbl(res, &res->dpi_tbl, dev_attr->l2_db_size);
 954         if (rc)
 955                 goto fail;
 956
 957         return 0;
 958 fail:
 959         bnxt_qplib_free_res(res);
 960         return rc;
 961 }
 962
 963 int bnxt_qplib_determine_atomics(struct pci_dev *dev)
 964 {
 965         int comp;
 966         u16 ctl2;
 967
 968         comp = pci_enable_atomic_ops_to_root(dev,
 969                                              PCI_EXP_DEVCAP2_ATOMIC_COMP32);
 970         if (comp)
 971                 return -EOPNOTSUPP;
 972         comp = pci_enable_atomic_ops_to_root(dev,
 973                                              PCI_EXP_DEVCAP2_ATOMIC_COMP64);
 974         if (comp)
 975                 return -EOPNOTSUPP;
 976         pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &ctl2);
 977         return !(ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
 978 }