drivers/net/ethernet/marvell/octeontx2/af/rvu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell OcteonTx2 RVU Admin Function driver
   3  *
   4  * Copyright (C) 2018 Marvell International Ltd.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/delay.h>
  14 #include <linux/irq.h>
  15 #include <linux/pci.h>
  16 #include <linux/sysfs.h>
  17
  18 #include "cgx.h"
  19 #include "rvu.h"
  20 #include "rvu_reg.h"
  21
  22 #define DRV_NAME        "octeontx2-af"
  23 #define DRV_STRING      "Marvell OcteonTX2 RVU Admin Function Driver"
  24 #define DRV_VERSION     "1.0"
  25
  26 static int rvu_get_hwvf(struct rvu *rvu, int pcifunc);
  27
  28 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
  29                                 struct rvu_block *block, int lf);
  30 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
  31                                   struct rvu_block *block, int lf);
  32 static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc);
  33
  34 static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
  35                          int type, int num,
  36                          void (mbox_handler)(struct work_struct *),
  37                          void (mbox_up_handler)(struct work_struct *));
  38 enum {
  39         TYPE_AFVF,
  40         TYPE_AFPF,
  41 };
  42
  43 /* Supported devices */
  44 static const struct pci_device_id rvu_id_table[] = {
  45         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_AF) },
  46         { 0, }  /* end of table */
  47 };
  48
  49 MODULE_AUTHOR("Marvell International Ltd.");
  50 MODULE_DESCRIPTION(DRV_STRING);
  51 MODULE_LICENSE("GPL v2");
  52 MODULE_VERSION(DRV_VERSION);
  53 MODULE_DEVICE_TABLE(pci, rvu_id_table);
  54
  55 static char *mkex_profile; /* MKEX profile name */
  56 module_param(mkex_profile, charp, 0000);
  57 MODULE_PARM_DESC(mkex_profile, "MKEX profile name string");
  58
  59 /* Poll a RVU block's register 'offset', for a 'zero'
  60  * or 'nonzero' at bits specified by 'mask'
  61  */
  62 int rvu_poll_reg(struct rvu *rvu, u64 block, u64 offset, u64 mask, bool zero)
  63 {
  64         unsigned long timeout = jiffies + usecs_to_jiffies(100);
  65         void __iomem *reg;
  66         u64 reg_val;
  67
  68         reg = rvu->afreg_base + ((block << 28) | offset);
  69         while (time_before(jiffies, timeout)) {
  70                 reg_val = readq(reg);
  71                 if (zero && !(reg_val & mask))
  72                         return 0;
  73                 if (!zero && (reg_val & mask))
  74                         return 0;
  75                 usleep_range(1, 5);
  76                 timeout--;
  77         }
  78         return -EBUSY;
  79 }
  80
  81 int rvu_alloc_rsrc(struct rsrc_bmap *rsrc)
  82 {
  83         int id;
  84
  85         if (!rsrc->bmap)
  86                 return -EINVAL;
  87
  88         id = find_first_zero_bit(rsrc->bmap, rsrc->max);
  89         if (id >= rsrc->max)
  90                 return -ENOSPC;
  91
  92         __set_bit(id, rsrc->bmap);
  93
  94         return id;
  95 }
  96
  97 int rvu_alloc_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc)
  98 {
  99         int start;
 100
 101         if (!rsrc->bmap)
 102                 return -EINVAL;
 103
 104         start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);
 105         if (start >= rsrc->max)
 106                 return -ENOSPC;
 107
 108         bitmap_set(rsrc->bmap, start, nrsrc);
 109         return start;
 110 }
 111
 112 static void rvu_free_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc, int start)
 113 {
 114         if (!rsrc->bmap)
 115                 return;
 116         if (start >= rsrc->max)
 117                 return;
 118
 119         bitmap_clear(rsrc->bmap, start, nrsrc);
 120 }
 121
 122 bool rvu_rsrc_check_contig(struct rsrc_bmap *rsrc, int nrsrc)
 123 {
 124         int start;
 125
 126         if (!rsrc->bmap)
 127                 return false;
 128
 129         start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);
 130         if (start >= rsrc->max)
 131                 return false;
 132
 133         return true;
 134 }
 135
 136 void rvu_free_rsrc(struct rsrc_bmap *rsrc, int id)
 137 {
 138         if (!rsrc->bmap)
 139                 return;
 140
 141         __clear_bit(id, rsrc->bmap);
 142 }
 143
 144 int rvu_rsrc_free_count(struct rsrc_bmap *rsrc)
 145 {
 146         int used;
 147
 148         if (!rsrc->bmap)
 149                 return 0;
 150
 151         used = bitmap_weight(rsrc->bmap, rsrc->max);
 152         return (rsrc->max - used);
 153 }
 154
 155 int rvu_alloc_bitmap(struct rsrc_bmap *rsrc)
 156 {
 157         rsrc->bmap = kcalloc(BITS_TO_LONGS(rsrc->max),
 158                              sizeof(long), GFP_KERNEL);
 159         if (!rsrc->bmap)
 160                 return -ENOMEM;
 161         return 0;
 162 }
 163
 164 /* Get block LF's HW index from a PF_FUNC's block slot number */
 165 int rvu_get_lf(struct rvu *rvu, struct rvu_block *block, u16 pcifunc, u16 slot)
 166 {
 167         u16 match = 0;
 168         int lf;
 169
 170         mutex_lock(&rvu->rsrc_lock);
 171         for (lf = 0; lf < block->lf.max; lf++) {
 172                 if (block->fn_map[lf] == pcifunc) {
 173                         if (slot == match) {
 174                                 mutex_unlock(&rvu->rsrc_lock);
 175                                 return lf;
 176                         }
 177                         match++;
 178                 }
 179         }
 180         mutex_unlock(&rvu->rsrc_lock);
 181         return -ENODEV;
 182 }
 183
 184 /* Convert BLOCK_TYPE_E to a BLOCK_ADDR_E.
 185  * Some silicon variants of OcteonTX2 supports
 186  * multiple blocks of same type.
 187  *
 188  * @pcifunc has to be zero when no LF is yet attached.
 189  */
 190 int rvu_get_blkaddr(struct rvu *rvu, int blktype, u16 pcifunc)
 191 {
 192         int devnum, blkaddr = -ENODEV;
 193         u64 cfg, reg;
 194         bool is_pf;
 195
 196         switch (blktype) {
 197         case BLKTYPE_NPC:
 198                 blkaddr = BLKADDR_NPC;
 199                 goto exit;
 200         case BLKTYPE_NPA:
 201                 blkaddr = BLKADDR_NPA;
 202                 goto exit;
 203         case BLKTYPE_NIX:
 204                 /* For now assume NIX0 */
 205                 if (!pcifunc) {
 206                         blkaddr = BLKADDR_NIX0;
 207                         goto exit;
 208                 }
 209                 break;
 210         case BLKTYPE_SSO:
 211                 blkaddr = BLKADDR_SSO;
 212                 goto exit;
 213         case BLKTYPE_SSOW:
 214                 blkaddr = BLKADDR_SSOW;
 215                 goto exit;
 216         case BLKTYPE_TIM:
 217                 blkaddr = BLKADDR_TIM;
 218                 goto exit;
 219         case BLKTYPE_CPT:
 220                 /* For now assume CPT0 */
 221                 if (!pcifunc) {
 222                         blkaddr = BLKADDR_CPT0;
 223                         goto exit;
 224                 }
 225                 break;
 226         }
 227
 228         /* Check if this is a RVU PF or VF */
 229         if (pcifunc & RVU_PFVF_FUNC_MASK) {
 230                 is_pf = false;
 231                 devnum = rvu_get_hwvf(rvu, pcifunc);
 232         } else {
 233                 is_pf = true;
 234                 devnum = rvu_get_pf(pcifunc);
 235         }
 236
 237         /* Check if the 'pcifunc' has a NIX LF from 'BLKADDR_NIX0' */
 238         if (blktype == BLKTYPE_NIX) {
 239                 reg = is_pf ? RVU_PRIV_PFX_NIX0_CFG : RVU_PRIV_HWVFX_NIX0_CFG;
 240                 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
 241                 if (cfg)
 242                         blkaddr = BLKADDR_NIX0;
 243         }
 244
 245         /* Check if the 'pcifunc' has a CPT LF from 'BLKADDR_CPT0' */
 246         if (blktype == BLKTYPE_CPT) {
 247                 reg = is_pf ? RVU_PRIV_PFX_CPT0_CFG : RVU_PRIV_HWVFX_CPT0_CFG;
 248                 cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));
 249                 if (cfg)
 250                         blkaddr = BLKADDR_CPT0;
 251         }
 252
 253 exit:
 254         if (is_block_implemented(rvu->hw, blkaddr))
 255                 return blkaddr;
 256         return -ENODEV;
 257 }
 258
 259 static void rvu_update_rsrc_map(struct rvu *rvu, struct rvu_pfvf *pfvf,
 260                                 struct rvu_block *block, u16 pcifunc,
 261                                 u16 lf, bool attach)
 262 {
 263         int devnum, num_lfs = 0;
 264         bool is_pf;
 265         u64 reg;
 266
 267         if (lf >= block->lf.max) {
 268                 dev_err(&rvu->pdev->dev,
 269                         "%s: FATAL: LF %d is >= %s's max lfs i.e %d\n",
 270                         __func__, lf, block->name, block->lf.max);
 271                 return;
 272         }
 273
 274         /* Check if this is for a RVU PF or VF */
 275         if (pcifunc & RVU_PFVF_FUNC_MASK) {
 276                 is_pf = false;
 277                 devnum = rvu_get_hwvf(rvu, pcifunc);
 278         } else {
 279                 is_pf = true;
 280                 devnum = rvu_get_pf(pcifunc);
 281         }
 282
 283         block->fn_map[lf] = attach ? pcifunc : 0;
 284
 285         switch (block->type) {
 286         case BLKTYPE_NPA:
 287                 pfvf->npalf = attach ? true : false;
 288                 num_lfs = pfvf->npalf;
 289                 break;
 290         case BLKTYPE_NIX:
 291                 pfvf->nixlf = attach ? true : false;
 292                 num_lfs = pfvf->nixlf;
 293                 break;
 294         case BLKTYPE_SSO:
 295                 attach ? pfvf->sso++ : pfvf->sso--;
 296                 num_lfs = pfvf->sso;
 297                 break;
 298         case BLKTYPE_SSOW:
 299                 attach ? pfvf->ssow++ : pfvf->ssow--;
 300                 num_lfs = pfvf->ssow;
 301                 break;
 302         case BLKTYPE_TIM:
 303                 attach ? pfvf->timlfs++ : pfvf->timlfs--;
 304                 num_lfs = pfvf->timlfs;
 305                 break;
 306         case BLKTYPE_CPT:
 307                 attach ? pfvf->cptlfs++ : pfvf->cptlfs--;
 308                 num_lfs = pfvf->cptlfs;
 309                 break;
 310         }
 311
 312         reg = is_pf ? block->pf_lfcnt_reg : block->vf_lfcnt_reg;
 313         rvu_write64(rvu, BLKADDR_RVUM, reg | (devnum << 16), num_lfs);
 314 }
 315
 316 inline int rvu_get_pf(u16 pcifunc)
 317 {
 318         return (pcifunc >> RVU_PFVF_PF_SHIFT) & RVU_PFVF_PF_MASK;
 319 }
 320
 321 void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf)
 322 {
 323         u64 cfg;
 324
 325         /* Get numVFs attached to this PF and first HWVF */
 326         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 327         *numvfs = (cfg >> 12) & 0xFF;
 328         *hwvf = cfg & 0xFFF;
 329 }
 330
 331 static int rvu_get_hwvf(struct rvu *rvu, int pcifunc)
 332 {
 333         int pf, func;
 334         u64 cfg;
 335
 336         pf = rvu_get_pf(pcifunc);
 337         func = pcifunc & RVU_PFVF_FUNC_MASK;
 338
 339         /* Get first HWVF attached to this PF */
 340         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 341
 342         return ((cfg & 0xFFF) + func - 1);
 343 }
 344
 345 struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc)
 346 {
 347         /* Check if it is a PF or VF */
 348         if (pcifunc & RVU_PFVF_FUNC_MASK)
 349                 return &rvu->hwvf[rvu_get_hwvf(rvu, pcifunc)];
 350         else
 351                 return &rvu->pf[rvu_get_pf(pcifunc)];
 352 }
 353
 354 static bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc)
 355 {
 356         int pf, vf, nvfs;
 357         u64 cfg;
 358
 359         pf = rvu_get_pf(pcifunc);
 360         if (pf >= rvu->hw->total_pfs)
 361                 return false;
 362
 363         if (!(pcifunc & RVU_PFVF_FUNC_MASK))
 364                 return true;
 365
 366         /* Check if VF is within number of VFs attached to this PF */
 367         vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
 368         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 369         nvfs = (cfg >> 12) & 0xFF;
 370         if (vf >= nvfs)
 371                 return false;
 372
 373         return true;
 374 }
 375
 376 bool is_block_implemented(struct rvu_hwinfo *hw, int blkaddr)
 377 {
 378         struct rvu_block *block;
 379
 380         if (blkaddr < BLKADDR_RVUM || blkaddr >= BLK_COUNT)
 381                 return false;
 382
 383         block = &hw->block[blkaddr];
 384         return block->implemented;
 385 }
 386
 387 static void rvu_check_block_implemented(struct rvu *rvu)
 388 {
 389         struct rvu_hwinfo *hw = rvu->hw;
 390         struct rvu_block *block;
 391         int blkid;
 392         u64 cfg;
 393
 394         /* For each block check if 'implemented' bit is set */
 395         for (blkid = 0; blkid < BLK_COUNT; blkid++) {
 396                 block = &hw->block[blkid];
 397                 cfg = rvupf_read64(rvu, RVU_PF_BLOCK_ADDRX_DISC(blkid));
 398                 if (cfg & BIT_ULL(11))
 399                         block->implemented = true;
 400         }
 401 }
 402
 403 int rvu_lf_reset(struct rvu *rvu, struct rvu_block *block, int lf)
 404 {
 405         int err;
 406
 407         if (!block->implemented)
 408                 return 0;
 409
 410         rvu_write64(rvu, block->addr, block->lfreset_reg, lf | BIT_ULL(12));
 411         err = rvu_poll_reg(rvu, block->addr, block->lfreset_reg, BIT_ULL(12),
 412                            true);
 413         return err;
 414 }
 415
 416 static void rvu_block_reset(struct rvu *rvu, int blkaddr, u64 rst_reg)
 417 {
 418         struct rvu_block *block = &rvu->hw->block[blkaddr];
 419
 420         if (!block->implemented)
 421                 return;
 422
 423         rvu_write64(rvu, blkaddr, rst_reg, BIT_ULL(0));
 424         rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true);
 425 }
 426
 427 static void rvu_reset_all_blocks(struct rvu *rvu)
 428 {
 429         /* Do a HW reset of all RVU blocks */
 430         rvu_block_reset(rvu, BLKADDR_NPA, NPA_AF_BLK_RST);
 431         rvu_block_reset(rvu, BLKADDR_NIX0, NIX_AF_BLK_RST);
 432         rvu_block_reset(rvu, BLKADDR_NPC, NPC_AF_BLK_RST);
 433         rvu_block_reset(rvu, BLKADDR_SSO, SSO_AF_BLK_RST);
 434         rvu_block_reset(rvu, BLKADDR_TIM, TIM_AF_BLK_RST);
 435         rvu_block_reset(rvu, BLKADDR_CPT0, CPT_AF_BLK_RST);
 436         rvu_block_reset(rvu, BLKADDR_NDC0, NDC_AF_BLK_RST);
 437         rvu_block_reset(rvu, BLKADDR_NDC1, NDC_AF_BLK_RST);
 438         rvu_block_reset(rvu, BLKADDR_NDC2, NDC_AF_BLK_RST);
 439 }
 440
 441 static void rvu_scan_block(struct rvu *rvu, struct rvu_block *block)
 442 {
 443         struct rvu_pfvf *pfvf;
 444         u64 cfg;
 445         int lf;
 446
 447         for (lf = 0; lf < block->lf.max; lf++) {
 448                 cfg = rvu_read64(rvu, block->addr,
 449                                  block->lfcfg_reg | (lf << block->lfshift));
 450                 if (!(cfg & BIT_ULL(63)))
 451                         continue;
 452
 453                 /* Set this resource as being used */
 454                 __set_bit(lf, block->lf.bmap);
 455
 456                 /* Get, to whom this LF is attached */
 457                 pfvf = rvu_get_pfvf(rvu, (cfg >> 8) & 0xFFFF);
 458                 rvu_update_rsrc_map(rvu, pfvf, block,
 459                                     (cfg >> 8) & 0xFFFF, lf, true);
 460
 461                 /* Set start MSIX vector for this LF within this PF/VF */
 462                 rvu_set_msix_offset(rvu, pfvf, block, lf);
 463         }
 464 }
 465
 466 static void rvu_check_min_msix_vec(struct rvu *rvu, int nvecs, int pf, int vf)
 467 {
 468         int min_vecs;
 469
 470         if (!vf)
 471                 goto check_pf;
 472
 473         if (!nvecs) {
 474                 dev_warn(rvu->dev,
 475                          "PF%d:VF%d is configured with zero msix vectors, %d\n",
 476                          pf, vf - 1, nvecs);
 477         }
 478         return;
 479
 480 check_pf:
 481         if (pf == 0)
 482                 min_vecs = RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT;
 483         else
 484                 min_vecs = RVU_PF_INT_VEC_CNT;
 485
 486         if (!(nvecs < min_vecs))
 487                 return;
 488         dev_warn(rvu->dev,
 489                  "PF%d is configured with too few vectors, %d, min is %d\n",
 490                  pf, nvecs, min_vecs);
 491 }
 492
 493 static int rvu_setup_msix_resources(struct rvu *rvu)
 494 {
 495         struct rvu_hwinfo *hw = rvu->hw;
 496         int pf, vf, numvfs, hwvf, err;
 497         int nvecs, offset, max_msix;
 498         struct rvu_pfvf *pfvf;
 499         u64 cfg, phy_addr;
 500         dma_addr_t iova;
 501
 502         for (pf = 0; pf < hw->total_pfs; pf++) {
 503                 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
 504                 /* If PF is not enabled, nothing to do */
 505                 if (!((cfg >> 20) & 0x01))
 506                         continue;
 507
 508                 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
 509
 510                 pfvf = &rvu->pf[pf];
 511                 /* Get num of MSIX vectors attached to this PF */
 512                 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_MSIX_CFG(pf));
 513                 pfvf->msix.max = ((cfg >> 32) & 0xFFF) + 1;
 514                 rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, 0);
 515
 516                 /* Alloc msix bitmap for this PF */
 517                 err = rvu_alloc_bitmap(&pfvf->msix);
 518                 if (err)
 519                         return err;
 520
 521                 /* Allocate memory for MSIX vector to RVU block LF mapping */
 522                 pfvf->msix_lfmap = devm_kcalloc(rvu->dev, pfvf->msix.max,
 523                                                 sizeof(u16), GFP_KERNEL);
 524                 if (!pfvf->msix_lfmap)
 525                         return -ENOMEM;
 526
 527                 /* For PF0 (AF) firmware will set msix vector offsets for
 528                  * AF, block AF and PF0_INT vectors, so jump to VFs.
 529                  */
 530                 if (!pf)
 531                         goto setup_vfmsix;
 532
 533                 /* Set MSIX offset for PF's 'RVU_PF_INT_VEC' vectors.
 534                  * These are allocated on driver init and never freed,
 535                  * so no need to set 'msix_lfmap' for these.
 536                  */
 537                 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(pf));
 538                 nvecs = (cfg >> 12) & 0xFF;
 539                 cfg &= ~0x7FFULL;
 540                 offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
 541                 rvu_write64(rvu, BLKADDR_RVUM,
 542                             RVU_PRIV_PFX_INT_CFG(pf), cfg | offset);
 543 setup_vfmsix:
 544                 /* Alloc msix bitmap for VFs */
 545                 for (vf = 0; vf < numvfs; vf++) {
 546                         pfvf =  &rvu->hwvf[hwvf + vf];
 547                         /* Get num of MSIX vectors attached to this VF */
 548                         cfg = rvu_read64(rvu, BLKADDR_RVUM,
 549                                          RVU_PRIV_PFX_MSIX_CFG(pf));
 550                         pfvf->msix.max = (cfg & 0xFFF) + 1;
 551                         rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, vf + 1);
 552
 553                         /* Alloc msix bitmap for this VF */
 554                         err = rvu_alloc_bitmap(&pfvf->msix);
 555                         if (err)
 556                                 return err;
 557
 558                         pfvf->msix_lfmap =
 559                                 devm_kcalloc(rvu->dev, pfvf->msix.max,
 560                                              sizeof(u16), GFP_KERNEL);
 561                         if (!pfvf->msix_lfmap)
 562                                 return -ENOMEM;
 563
 564                         /* Set MSIX offset for HWVF's 'RVU_VF_INT_VEC' vectors.
 565                          * These are allocated on driver init and never freed,
 566                          * so no need to set 'msix_lfmap' for these.
 567                          */
 568                         cfg = rvu_read64(rvu, BLKADDR_RVUM,
 569                                          RVU_PRIV_HWVFX_INT_CFG(hwvf + vf));
 570                         nvecs = (cfg >> 12) & 0xFF;
 571                         cfg &= ~0x7FFULL;
 572                         offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
 573                         rvu_write64(rvu, BLKADDR_RVUM,
 574                                     RVU_PRIV_HWVFX_INT_CFG(hwvf + vf),
 575                                     cfg | offset);
 576                 }
 577         }
 578
 579         /* HW interprets RVU_AF_MSIXTR_BASE address as an IOVA, hence
 580          * create a IOMMU mapping for the physcial address configured by
 581          * firmware and reconfig RVU_AF_MSIXTR_BASE with IOVA.
 582          */
 583         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
 584         max_msix = cfg & 0xFFFFF;
 585         phy_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE);
 586         iova = dma_map_resource(rvu->dev, phy_addr,
 587                                 max_msix * PCI_MSIX_ENTRY_SIZE,
 588                                 DMA_BIDIRECTIONAL, 0);
 589
 590         if (dma_mapping_error(rvu->dev, iova))
 591                 return -ENOMEM;
 592
 593         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE, (u64)iova);
 594         rvu->msix_base_iova = iova;
 595
 596         return 0;
 597 }
 598
 599 static void rvu_free_hw_resources(struct rvu *rvu)
 600 {
 601         struct rvu_hwinfo *hw = rvu->hw;
 602         struct rvu_block *block;
 603         struct rvu_pfvf  *pfvf;
 604         int id, max_msix;
 605         u64 cfg;
 606
 607         rvu_npa_freemem(rvu);
 608         rvu_npc_freemem(rvu);
 609         rvu_nix_freemem(rvu);
 610
 611         /* Free block LF bitmaps */
 612         for (id = 0; id < BLK_COUNT; id++) {
 613                 block = &hw->block[id];
 614                 kfree(block->lf.bmap);
 615         }
 616
 617         /* Free MSIX bitmaps */
 618         for (id = 0; id < hw->total_pfs; id++) {
 619                 pfvf = &rvu->pf[id];
 620                 kfree(pfvf->msix.bmap);
 621         }
 622
 623         for (id = 0; id < hw->total_vfs; id++) {
 624                 pfvf = &rvu->hwvf[id];
 625                 kfree(pfvf->msix.bmap);
 626         }
 627
 628         /* Unmap MSIX vector base IOVA mapping */
 629         if (!rvu->msix_base_iova)
 630                 return;
 631         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
 632         max_msix = cfg & 0xFFFFF;
 633         dma_unmap_resource(rvu->dev, rvu->msix_base_iova,
 634                            max_msix * PCI_MSIX_ENTRY_SIZE,
 635                            DMA_BIDIRECTIONAL, 0);
 636
 637         mutex_destroy(&rvu->rsrc_lock);
 638 }
 639
 640 static int rvu_setup_hw_resources(struct rvu *rvu)
 641 {
 642         struct rvu_hwinfo *hw = rvu->hw;
 643         struct rvu_block *block;
 644         int blkid, err;
 645         u64 cfg;
 646
 647         /* Get HW supported max RVU PF & VF count */
 648         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);
 649         hw->total_pfs = (cfg >> 32) & 0xFF;
 650         hw->total_vfs = (cfg >> 20) & 0xFFF;
 651         hw->max_vfs_per_pf = (cfg >> 40) & 0xFF;
 652
 653         /* Init NPA LF's bitmap */
 654         block = &hw->block[BLKADDR_NPA];
 655         if (!block->implemented)
 656                 goto nix;
 657         cfg = rvu_read64(rvu, BLKADDR_NPA, NPA_AF_CONST);
 658         block->lf.max = (cfg >> 16) & 0xFFF;
 659         block->addr = BLKADDR_NPA;
 660         block->type = BLKTYPE_NPA;
 661         block->lfshift = 8;
 662         block->lookup_reg = NPA_AF_RVU_LF_CFG_DEBUG;
 663         block->pf_lfcnt_reg = RVU_PRIV_PFX_NPA_CFG;
 664         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NPA_CFG;
 665         block->lfcfg_reg = NPA_PRIV_LFX_CFG;
 666         block->msixcfg_reg = NPA_PRIV_LFX_INT_CFG;
 667         block->lfreset_reg = NPA_AF_LF_RST;
 668         sprintf(block->name, "NPA");
 669         err = rvu_alloc_bitmap(&block->lf);
 670         if (err)
 671                 return err;
 672
 673 nix:
 674         /* Init NIX LF's bitmap */
 675         block = &hw->block[BLKADDR_NIX0];
 676         if (!block->implemented)
 677                 goto sso;
 678         cfg = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_CONST2);
 679         block->lf.max = cfg & 0xFFF;
 680         block->addr = BLKADDR_NIX0;
 681         block->type = BLKTYPE_NIX;
 682         block->lfshift = 8;
 683         block->lookup_reg = NIX_AF_RVU_LF_CFG_DEBUG;
 684         block->pf_lfcnt_reg = RVU_PRIV_PFX_NIX0_CFG;
 685         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NIX0_CFG;
 686         block->lfcfg_reg = NIX_PRIV_LFX_CFG;
 687         block->msixcfg_reg = NIX_PRIV_LFX_INT_CFG;
 688         block->lfreset_reg = NIX_AF_LF_RST;
 689         sprintf(block->name, "NIX");
 690         err = rvu_alloc_bitmap(&block->lf);
 691         if (err)
 692                 return err;
 693
 694 sso:
 695         /* Init SSO group's bitmap */
 696         block = &hw->block[BLKADDR_SSO];
 697         if (!block->implemented)
 698                 goto ssow;
 699         cfg = rvu_read64(rvu, BLKADDR_SSO, SSO_AF_CONST);
 700         block->lf.max = cfg & 0xFFFF;
 701         block->addr = BLKADDR_SSO;
 702         block->type = BLKTYPE_SSO;
 703         block->multislot = true;
 704         block->lfshift = 3;
 705         block->lookup_reg = SSO_AF_RVU_LF_CFG_DEBUG;
 706         block->pf_lfcnt_reg = RVU_PRIV_PFX_SSO_CFG;
 707         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSO_CFG;
 708         block->lfcfg_reg = SSO_PRIV_LFX_HWGRP_CFG;
 709         block->msixcfg_reg = SSO_PRIV_LFX_HWGRP_INT_CFG;
 710         block->lfreset_reg = SSO_AF_LF_HWGRP_RST;
 711         sprintf(block->name, "SSO GROUP");
 712         err = rvu_alloc_bitmap(&block->lf);
 713         if (err)
 714                 return err;
 715
 716 ssow:
 717         /* Init SSO workslot's bitmap */
 718         block = &hw->block[BLKADDR_SSOW];
 719         if (!block->implemented)
 720                 goto tim;
 721         block->lf.max = (cfg >> 56) & 0xFF;
 722         block->addr = BLKADDR_SSOW;
 723         block->type = BLKTYPE_SSOW;
 724         block->multislot = true;
 725         block->lfshift = 3;
 726         block->lookup_reg = SSOW_AF_RVU_LF_HWS_CFG_DEBUG;
 727         block->pf_lfcnt_reg = RVU_PRIV_PFX_SSOW_CFG;
 728         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSOW_CFG;
 729         block->lfcfg_reg = SSOW_PRIV_LFX_HWS_CFG;
 730         block->msixcfg_reg = SSOW_PRIV_LFX_HWS_INT_CFG;
 731         block->lfreset_reg = SSOW_AF_LF_HWS_RST;
 732         sprintf(block->name, "SSOWS");
 733         err = rvu_alloc_bitmap(&block->lf);
 734         if (err)
 735                 return err;
 736
 737 tim:
 738         /* Init TIM LF's bitmap */
 739         block = &hw->block[BLKADDR_TIM];
 740         if (!block->implemented)
 741                 goto cpt;
 742         cfg = rvu_read64(rvu, BLKADDR_TIM, TIM_AF_CONST);
 743         block->lf.max = cfg & 0xFFFF;
 744         block->addr = BLKADDR_TIM;
 745         block->type = BLKTYPE_TIM;
 746         block->multislot = true;
 747         block->lfshift = 3;
 748         block->lookup_reg = TIM_AF_RVU_LF_CFG_DEBUG;
 749         block->pf_lfcnt_reg = RVU_PRIV_PFX_TIM_CFG;
 750         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_TIM_CFG;
 751         block->lfcfg_reg = TIM_PRIV_LFX_CFG;
 752         block->msixcfg_reg = TIM_PRIV_LFX_INT_CFG;
 753         block->lfreset_reg = TIM_AF_LF_RST;
 754         sprintf(block->name, "TIM");
 755         err = rvu_alloc_bitmap(&block->lf);
 756         if (err)
 757                 return err;
 758
 759 cpt:
 760         /* Init CPT LF's bitmap */
 761         block = &hw->block[BLKADDR_CPT0];
 762         if (!block->implemented)
 763                 goto init;
 764         cfg = rvu_read64(rvu, BLKADDR_CPT0, CPT_AF_CONSTANTS0);
 765         block->lf.max = cfg & 0xFF;
 766         block->addr = BLKADDR_CPT0;
 767         block->type = BLKTYPE_CPT;
 768         block->multislot = true;
 769         block->lfshift = 3;
 770         block->lookup_reg = CPT_AF_RVU_LF_CFG_DEBUG;
 771         block->pf_lfcnt_reg = RVU_PRIV_PFX_CPT0_CFG;
 772         block->vf_lfcnt_reg = RVU_PRIV_HWVFX_CPT0_CFG;
 773         block->lfcfg_reg = CPT_PRIV_LFX_CFG;
 774         block->msixcfg_reg = CPT_PRIV_LFX_INT_CFG;
 775         block->lfreset_reg = CPT_AF_LF_RST;
 776         sprintf(block->name, "CPT");
 777         err = rvu_alloc_bitmap(&block->lf);
 778         if (err)
 779                 return err;
 780
 781 init:
 782         /* Allocate memory for PFVF data */
 783         rvu->pf = devm_kcalloc(rvu->dev, hw->total_pfs,
 784                                sizeof(struct rvu_pfvf), GFP_KERNEL);
 785         if (!rvu->pf)
 786                 return -ENOMEM;
 787
 788         rvu->hwvf = devm_kcalloc(rvu->dev, hw->total_vfs,
 789                                  sizeof(struct rvu_pfvf), GFP_KERNEL);
 790         if (!rvu->hwvf)
 791                 return -ENOMEM;
 792
 793         mutex_init(&rvu->rsrc_lock);
 794
 795         err = rvu_setup_msix_resources(rvu);
 796         if (err)
 797                 return err;
 798
 799         for (blkid = 0; blkid < BLK_COUNT; blkid++) {
 800                 block = &hw->block[blkid];
 801                 if (!block->lf.bmap)
 802                         continue;
 803
 804                 /* Allocate memory for block LF/slot to pcifunc mapping info */
 805                 block->fn_map = devm_kcalloc(rvu->dev, block->lf.max,
 806                                              sizeof(u16), GFP_KERNEL);
 807                 if (!block->fn_map)
 808                         return -ENOMEM;
 809
 810                 /* Scan all blocks to check if low level firmware has
 811                  * already provisioned any of the resources to a PF/VF.
 812                  */
 813                 rvu_scan_block(rvu, block);
 814         }
 815
 816         err = rvu_npc_init(rvu);
 817         if (err)
 818                 goto exit;
 819
 820         err = rvu_cgx_init(rvu);
 821         if (err)
 822                 goto exit;
 823
 824         err = rvu_npa_init(rvu);
 825         if (err)
 826                 goto cgx_err;
 827
 828         err = rvu_nix_init(rvu);
 829         if (err)
 830                 goto cgx_err;
 831
 832         return 0;
 833
 834 cgx_err:
 835         rvu_cgx_exit(rvu);
 836 exit:
 837         return err;
 838 }
 839
 840 /* NPA and NIX admin queue APIs */
 841 void rvu_aq_free(struct rvu *rvu, struct admin_queue *aq)
 842 {
 843         if (!aq)
 844                 return;
 845
 846         qmem_free(rvu->dev, aq->inst);
 847         qmem_free(rvu->dev, aq->res);
 848         devm_kfree(rvu->dev, aq);
 849 }
 850
 851 int rvu_aq_alloc(struct rvu *rvu, struct admin_queue **ad_queue,
 852                  int qsize, int inst_size, int res_size)
 853 {
 854         struct admin_queue *aq;
 855         int err;
 856
 857         *ad_queue = devm_kzalloc(rvu->dev, sizeof(*aq), GFP_KERNEL);
 858         if (!*ad_queue)
 859                 return -ENOMEM;
 860         aq = *ad_queue;
 861
 862         /* Alloc memory for instructions i.e AQ */
 863         err = qmem_alloc(rvu->dev, &aq->inst, qsize, inst_size);
 864         if (err) {
 865                 devm_kfree(rvu->dev, aq);
 866                 return err;
 867         }
 868
 869         /* Alloc memory for results */
 870         err = qmem_alloc(rvu->dev, &aq->res, qsize, res_size);
 871         if (err) {
 872                 rvu_aq_free(rvu, aq);
 873                 return err;
 874         }
 875
 876         spin_lock_init(&aq->lock);
 877         return 0;
 878 }
 879
 880 static int rvu_mbox_handler_ready(struct rvu *rvu, struct msg_req *req,
 881                                   struct ready_msg_rsp *rsp)
 882 {
 883         return 0;
 884 }
 885
 886 /* Get current count of a RVU block's LF/slots
 887  * provisioned to a given RVU func.
 888  */
 889 static u16 rvu_get_rsrc_mapcount(struct rvu_pfvf *pfvf, int blktype)
 890 {
 891         switch (blktype) {
 892         case BLKTYPE_NPA:
 893                 return pfvf->npalf ? 1 : 0;
 894         case BLKTYPE_NIX:
 895                 return pfvf->nixlf ? 1 : 0;
 896         case BLKTYPE_SSO:
 897                 return pfvf->sso;
 898         case BLKTYPE_SSOW:
 899                 return pfvf->ssow;
 900         case BLKTYPE_TIM:
 901                 return pfvf->timlfs;
 902         case BLKTYPE_CPT:
 903                 return pfvf->cptlfs;
 904         }
 905         return 0;
 906 }
 907
 908 bool is_pffunc_map_valid(struct rvu *rvu, u16 pcifunc, int blktype)
 909 {
 910         struct rvu_pfvf *pfvf;
 911
 912         if (!is_pf_func_valid(rvu, pcifunc))
 913                 return false;
 914
 915         pfvf = rvu_get_pfvf(rvu, pcifunc);
 916
 917         /* Check if this PFFUNC has a LF of type blktype attached */
 918         if (!rvu_get_rsrc_mapcount(pfvf, blktype))
 919                 return false;
 920
 921         return true;
 922 }
 923
 924 static int rvu_lookup_rsrc(struct rvu *rvu, struct rvu_block *block,
 925                            int pcifunc, int slot)
 926 {
 927         u64 val;
 928
 929         val = ((u64)pcifunc << 24) | (slot << 16) | (1ULL << 13);
 930         rvu_write64(rvu, block->addr, block->lookup_reg, val);
 931         /* Wait for the lookup to finish */
 932         /* TODO: put some timeout here */
 933         while (rvu_read64(rvu, block->addr, block->lookup_reg) & (1ULL << 13))
 934                 ;
 935
 936         val = rvu_read64(rvu, block->addr, block->lookup_reg);
 937
 938         /* Check LF valid bit */
 939         if (!(val & (1ULL << 12)))
 940                 return -1;
 941
 942         return (val & 0xFFF);
 943 }
 944
 945 static void rvu_detach_block(struct rvu *rvu, int pcifunc, int blktype)
 946 {
 947         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
 948         struct rvu_hwinfo *hw = rvu->hw;
 949         struct rvu_block *block;
 950         int slot, lf, num_lfs;
 951         int blkaddr;
 952
 953         blkaddr = rvu_get_blkaddr(rvu, blktype, pcifunc);
 954         if (blkaddr < 0)
 955                 return;
 956
 957         block = &hw->block[blkaddr];
 958
 959         num_lfs = rvu_get_rsrc_mapcount(pfvf, block->type);
 960         if (!num_lfs)
 961                 return;
 962
 963         for (slot = 0; slot < num_lfs; slot++) {
 964                 lf = rvu_lookup_rsrc(rvu, block, pcifunc, slot);
 965                 if (lf < 0) /* This should never happen */
 966                         continue;
 967
 968                 /* Disable the LF */
 969                 rvu_write64(rvu, blkaddr, block->lfcfg_reg |
 970                             (lf << block->lfshift), 0x00ULL);
 971
 972                 /* Update SW maintained mapping info as well */
 973                 rvu_update_rsrc_map(rvu, pfvf, block,
 974                                     pcifunc, lf, false);
 975
 976                 /* Free the resource */
 977                 rvu_free_rsrc(&block->lf, lf);
 978
 979                 /* Clear MSIX vector offset for this LF */
 980                 rvu_clear_msix_offset(rvu, pfvf, block, lf);
 981         }
 982 }
 983
 984 static int rvu_detach_rsrcs(struct rvu *rvu, struct rsrc_detach *detach,
 985                             u16 pcifunc)
 986 {
 987         struct rvu_hwinfo *hw = rvu->hw;
 988         bool detach_all = true;
 989         struct rvu_block *block;
 990         int blkid;
 991
 992         mutex_lock(&rvu->rsrc_lock);
 993
 994         /* Check for partial resource detach */
 995         if (detach && detach->partial)
 996                 detach_all = false;
 997
 998         /* Check for RVU block's LFs attached to this func,
 999          * if so, detach them.
1000          */
1001         for (blkid = 0; blkid < BLK_COUNT; blkid++) {
1002                 block = &hw->block[blkid];
1003                 if (!block->lf.bmap)
1004                         continue;
1005                 if (!detach_all && detach) {
1006                         if (blkid == BLKADDR_NPA && !detach->npalf)
1007                                 continue;
1008                         else if ((blkid == BLKADDR_NIX0) && !detach->nixlf)
1009                                 continue;
1010                         else if ((blkid == BLKADDR_SSO) && !detach->sso)
1011                                 continue;
1012                         else if ((blkid == BLKADDR_SSOW) && !detach->ssow)
1013                                 continue;
1014                         else if ((blkid == BLKADDR_TIM) && !detach->timlfs)
1015                                 continue;
1016                         else if ((blkid == BLKADDR_CPT0) && !detach->cptlfs)
1017                                 continue;
1018                 }
1019                 rvu_detach_block(rvu, pcifunc, block->type);
1020         }
1021
1022         mutex_unlock(&rvu->rsrc_lock);
1023         return 0;
1024 }
1025
1026 static int rvu_mbox_handler_detach_resources(struct rvu *rvu,
1027                                              struct rsrc_detach *detach,
1028                                              struct msg_rsp *rsp)
1029 {
1030         return rvu_detach_rsrcs(rvu, detach, detach->hdr.pcifunc);
1031 }
1032
1033 static void rvu_attach_block(struct rvu *rvu, int pcifunc,
1034                              int blktype, int num_lfs)
1035 {
1036         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1037         struct rvu_hwinfo *hw = rvu->hw;
1038         struct rvu_block *block;
1039         int slot, lf;
1040         int blkaddr;
1041         u64 cfg;
1042
1043         if (!num_lfs)
1044                 return;
1045
1046         blkaddr = rvu_get_blkaddr(rvu, blktype, 0);
1047         if (blkaddr < 0)
1048                 return;
1049
1050         block = &hw->block[blkaddr];
1051         if (!block->lf.bmap)
1052                 return;
1053
1054         for (slot = 0; slot < num_lfs; slot++) {
1055                 /* Allocate the resource */
1056                 lf = rvu_alloc_rsrc(&block->lf);
1057                 if (lf < 0)
1058                         return;
1059
1060                 cfg = (1ULL << 63) | (pcifunc << 8) | slot;
1061                 rvu_write64(rvu, blkaddr, block->lfcfg_reg |
1062                             (lf << block->lfshift), cfg);
1063                 rvu_update_rsrc_map(rvu, pfvf, block,
1064                                     pcifunc, lf, true);
1065
1066                 /* Set start MSIX vector for this LF within this PF/VF */
1067                 rvu_set_msix_offset(rvu, pfvf, block, lf);
1068         }
1069 }
1070
1071 static int rvu_check_rsrc_availability(struct rvu *rvu,
1072                                        struct rsrc_attach *req, u16 pcifunc)
1073 {
1074         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1075         struct rvu_hwinfo *hw = rvu->hw;
1076         struct rvu_block *block;
1077         int free_lfs, mappedlfs;
1078
1079         /* Only one NPA LF can be attached */
1080         if (req->npalf && !rvu_get_rsrc_mapcount(pfvf, BLKTYPE_NPA)) {
1081                 block = &hw->block[BLKADDR_NPA];
1082                 free_lfs = rvu_rsrc_free_count(&block->lf);
1083                 if (!free_lfs)
1084                         goto fail;
1085         } else if (req->npalf) {
1086                 dev_err(&rvu->pdev->dev,
1087                         "Func 0x%x: Invalid req, already has NPA\n",
1088                          pcifunc);
1089                 return -EINVAL;
1090         }
1091
1092         /* Only one NIX LF can be attached */
1093         if (req->nixlf && !rvu_get_rsrc_mapcount(pfvf, BLKTYPE_NIX)) {
1094                 block = &hw->block[BLKADDR_NIX0];
1095                 free_lfs = rvu_rsrc_free_count(&block->lf);
1096                 if (!free_lfs)
1097                         goto fail;
1098         } else if (req->nixlf) {
1099                 dev_err(&rvu->pdev->dev,
1100                         "Func 0x%x: Invalid req, already has NIX\n",
1101                         pcifunc);
1102                 return -EINVAL;
1103         }
1104
1105         if (req->sso) {
1106                 block = &hw->block[BLKADDR_SSO];
1107                 /* Is request within limits ? */
1108                 if (req->sso > block->lf.max) {
1109                         dev_err(&rvu->pdev->dev,
1110                                 "Func 0x%x: Invalid SSO req, %d > max %d\n",
1111                                  pcifunc, req->sso, block->lf.max);
1112                         return -EINVAL;
1113                 }
1114                 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1115                 free_lfs = rvu_rsrc_free_count(&block->lf);
1116                 /* Check if additional resources are available */
1117                 if (req->sso > mappedlfs &&
1118                     ((req->sso - mappedlfs) > free_lfs))
1119                         goto fail;
1120         }
1121
1122         if (req->ssow) {
1123                 block = &hw->block[BLKADDR_SSOW];
1124                 if (req->ssow > block->lf.max) {
1125                         dev_err(&rvu->pdev->dev,
1126                                 "Func 0x%x: Invalid SSOW req, %d > max %d\n",
1127                                  pcifunc, req->sso, block->lf.max);
1128                         return -EINVAL;
1129                 }
1130                 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1131                 free_lfs = rvu_rsrc_free_count(&block->lf);
1132                 if (req->ssow > mappedlfs &&
1133                     ((req->ssow - mappedlfs) > free_lfs))
1134                         goto fail;
1135         }
1136
1137         if (req->timlfs) {
1138                 block = &hw->block[BLKADDR_TIM];
1139                 if (req->timlfs > block->lf.max) {
1140                         dev_err(&rvu->pdev->dev,
1141                                 "Func 0x%x: Invalid TIMLF req, %d > max %d\n",
1142                                  pcifunc, req->timlfs, block->lf.max);
1143                         return -EINVAL;
1144                 }
1145                 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1146                 free_lfs = rvu_rsrc_free_count(&block->lf);
1147                 if (req->timlfs > mappedlfs &&
1148                     ((req->timlfs - mappedlfs) > free_lfs))
1149                         goto fail;
1150         }
1151
1152         if (req->cptlfs) {
1153                 block = &hw->block[BLKADDR_CPT0];
1154                 if (req->cptlfs > block->lf.max) {
1155                         dev_err(&rvu->pdev->dev,
1156                                 "Func 0x%x: Invalid CPTLF req, %d > max %d\n",
1157                                  pcifunc, req->cptlfs, block->lf.max);
1158                         return -EINVAL;
1159                 }
1160                 mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->type);
1161                 free_lfs = rvu_rsrc_free_count(&block->lf);
1162                 if (req->cptlfs > mappedlfs &&
1163                     ((req->cptlfs - mappedlfs) > free_lfs))
1164                         goto fail;
1165         }
1166
1167         return 0;
1168
1169 fail:
1170         dev_info(rvu->dev, "Request for %s failed\n", block->name);
1171         return -ENOSPC;
1172 }
1173
1174 static int rvu_mbox_handler_attach_resources(struct rvu *rvu,
1175                                              struct rsrc_attach *attach,
1176                                              struct msg_rsp *rsp)
1177 {
1178         u16 pcifunc = attach->hdr.pcifunc;
1179         int err;
1180
1181         /* If first request, detach all existing attached resources */
1182         if (!attach->modify)
1183                 rvu_detach_rsrcs(rvu, NULL, pcifunc);
1184
1185         mutex_lock(&rvu->rsrc_lock);
1186
1187         /* Check if the request can be accommodated */
1188         err = rvu_check_rsrc_availability(rvu, attach, pcifunc);
1189         if (err)
1190                 goto exit;
1191
1192         /* Now attach the requested resources */
1193         if (attach->npalf)
1194                 rvu_attach_block(rvu, pcifunc, BLKTYPE_NPA, 1);
1195
1196         if (attach->nixlf)
1197                 rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1);
1198
1199         if (attach->sso) {
1200                 /* RVU func doesn't know which exact LF or slot is attached
1201                  * to it, it always sees as slot 0,1,2. So for a 'modify'
1202                  * request, simply detach all existing attached LFs/slots
1203                  * and attach a fresh.
1204                  */
1205                 if (attach->modify)
1206                         rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO);
1207                 rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO, attach->sso);
1208         }
1209
1210         if (attach->ssow) {
1211                 if (attach->modify)
1212                         rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW);
1213                 rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW, attach->ssow);
1214         }
1215
1216         if (attach->timlfs) {
1217                 if (attach->modify)
1218                         rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM);
1219                 rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM, attach->timlfs);
1220         }
1221
1222         if (attach->cptlfs) {
1223                 if (attach->modify)
1224                         rvu_detach_block(rvu, pcifunc, BLKTYPE_CPT);
1225                 rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT, attach->cptlfs);
1226         }
1227
1228 exit:
1229         mutex_unlock(&rvu->rsrc_lock);
1230         return err;
1231 }
1232
1233 static u16 rvu_get_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1234                                int blkaddr, int lf)
1235 {
1236         u16 vec;
1237
1238         if (lf < 0)
1239                 return MSIX_VECTOR_INVALID;
1240
1241         for (vec = 0; vec < pfvf->msix.max; vec++) {
1242                 if (pfvf->msix_lfmap[vec] == MSIX_BLKLF(blkaddr, lf))
1243                         return vec;
1244         }
1245         return MSIX_VECTOR_INVALID;
1246 }
1247
1248 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1249                                 struct rvu_block *block, int lf)
1250 {
1251         u16 nvecs, vec, offset;
1252         u64 cfg;
1253
1254         cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |
1255                          (lf << block->lfshift));
1256         nvecs = (cfg >> 12) & 0xFF;
1257
1258         /* Check and alloc MSIX vectors, must be contiguous */
1259         if (!rvu_rsrc_check_contig(&pfvf->msix, nvecs))
1260                 return;
1261
1262         offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);
1263
1264         /* Config MSIX offset in LF */
1265         rvu_write64(rvu, block->addr, block->msixcfg_reg |
1266                     (lf << block->lfshift), (cfg & ~0x7FFULL) | offset);
1267
1268         /* Update the bitmap as well */
1269         for (vec = 0; vec < nvecs; vec++)
1270                 pfvf->msix_lfmap[offset + vec] = MSIX_BLKLF(block->addr, lf);
1271 }
1272
1273 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
1274                                   struct rvu_block *block, int lf)
1275 {
1276         u16 nvecs, vec, offset;
1277         u64 cfg;
1278
1279         cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |
1280                          (lf << block->lfshift));
1281         nvecs = (cfg >> 12) & 0xFF;
1282
1283         /* Clear MSIX offset in LF */
1284         rvu_write64(rvu, block->addr, block->msixcfg_reg |
1285                     (lf << block->lfshift), cfg & ~0x7FFULL);
1286
1287         offset = rvu_get_msix_offset(rvu, pfvf, block->addr, lf);
1288
1289         /* Update the mapping */
1290         for (vec = 0; vec < nvecs; vec++)
1291                 pfvf->msix_lfmap[offset + vec] = 0;
1292
1293         /* Free the same in MSIX bitmap */
1294         rvu_free_rsrc_contig(&pfvf->msix, nvecs, offset);
1295 }
1296
1297 static int rvu_mbox_handler_msix_offset(struct rvu *rvu, struct msg_req *req,
1298                                         struct msix_offset_rsp *rsp)
1299 {
1300         struct rvu_hwinfo *hw = rvu->hw;
1301         u16 pcifunc = req->hdr.pcifunc;
1302         struct rvu_pfvf *pfvf;
1303         int lf, slot;
1304
1305         pfvf = rvu_get_pfvf(rvu, pcifunc);
1306         if (!pfvf->msix.bmap)
1307                 return 0;
1308
1309         /* Set MSIX offsets for each block's LFs attached to this PF/VF */
1310         lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NPA], pcifunc, 0);
1311         rsp->npa_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NPA, lf);
1312
1313         lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NIX0], pcifunc, 0);
1314         rsp->nix_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NIX0, lf);
1315
1316         rsp->sso = pfvf->sso;
1317         for (slot = 0; slot < rsp->sso; slot++) {
1318                 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSO], pcifunc, slot);
1319                 rsp->sso_msixoff[slot] =
1320                         rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSO, lf);
1321         }
1322
1323         rsp->ssow = pfvf->ssow;
1324         for (slot = 0; slot < rsp->ssow; slot++) {
1325                 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSOW], pcifunc, slot);
1326                 rsp->ssow_msixoff[slot] =
1327                         rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSOW, lf);
1328         }
1329
1330         rsp->timlfs = pfvf->timlfs;
1331         for (slot = 0; slot < rsp->timlfs; slot++) {
1332                 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_TIM], pcifunc, slot);
1333                 rsp->timlf_msixoff[slot] =
1334                         rvu_get_msix_offset(rvu, pfvf, BLKADDR_TIM, lf);
1335         }
1336
1337         rsp->cptlfs = pfvf->cptlfs;
1338         for (slot = 0; slot < rsp->cptlfs; slot++) {
1339                 lf = rvu_get_lf(rvu, &hw->block[BLKADDR_CPT0], pcifunc, slot);
1340                 rsp->cptlf_msixoff[slot] =
1341                         rvu_get_msix_offset(rvu, pfvf, BLKADDR_CPT0, lf);
1342         }
1343         return 0;
1344 }
1345
1346 static int rvu_mbox_handler_vf_flr(struct rvu *rvu, struct msg_req *req,
1347                                    struct msg_rsp *rsp)
1348 {
1349         u16 pcifunc = req->hdr.pcifunc;
1350         u16 vf, numvfs;
1351         u64 cfg;
1352
1353         vf = pcifunc & RVU_PFVF_FUNC_MASK;
1354         cfg = rvu_read64(rvu, BLKADDR_RVUM,
1355                          RVU_PRIV_PFX_CFG(rvu_get_pf(pcifunc)));
1356         numvfs = (cfg >> 12) & 0xFF;
1357
1358         if (vf && vf <= numvfs)
1359                 __rvu_flr_handler(rvu, pcifunc);
1360         else
1361                 return RVU_INVALID_VF_ID;
1362
1363         return 0;
1364 }
1365
1366 static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid,
1367                                 struct mbox_msghdr *req)
1368 {
1369         struct rvu *rvu = pci_get_drvdata(mbox->pdev);
1370
1371         /* Check if valid, if not reply with a invalid msg */
1372         if (req->sig != OTX2_MBOX_REQ_SIG)
1373                 goto bad_message;
1374
1375         switch (req->id) {
1376 #define M(_name, _id, _fn_name, _req_type, _rsp_type)                   \
1377         case _id: {                                                     \
1378                 struct _rsp_type *rsp;                                  \
1379                 int err;                                                \
1380                                                                         \
1381                 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg(          \
1382                         mbox, devid,                                    \
1383                         sizeof(struct _rsp_type));                      \
1384                 /* some handlers should complete even if reply */       \
1385                 /* could not be allocated */                            \
1386                 if (!rsp &&                                             \
1387                     _id != MBOX_MSG_DETACH_RESOURCES &&                 \
1388                     _id != MBOX_MSG_NIX_TXSCH_FREE &&                   \
1389                     _id != MBOX_MSG_VF_FLR)                             \
1390                         return -ENOMEM;                                 \
1391                 if (rsp) {                                              \
1392                         rsp->hdr.id = _id;                              \
1393                         rsp->hdr.sig = OTX2_MBOX_RSP_SIG;               \
1394                         rsp->hdr.pcifunc = req->pcifunc;                \
1395                         rsp->hdr.rc = 0;                                \
1396                 }                                                       \
1397                                                                         \
1398                 err = rvu_mbox_handler_ ## _fn_name(rvu,                \
1399                                                     (struct _req_type *)req, \
1400                                                     rsp);               \
1401                 if (rsp && err)                                         \
1402                         rsp->hdr.rc = err;                              \
1403                                                                         \
1404                 return rsp ? err : -ENOMEM;                             \
1405         }
1406 MBOX_MESSAGES
1407 #undef M
1408
1409 bad_message:
1410         default:
1411                 otx2_reply_invalid_msg(mbox, devid, req->pcifunc, req->id);
1412                 return -ENODEV;
1413         }
1414 }
1415
1416 static void __rvu_mbox_handler(struct rvu_work *mwork, int type)
1417 {
1418         struct rvu *rvu = mwork->rvu;
1419         int offset, err, id, devid;
1420         struct otx2_mbox_dev *mdev;
1421         struct mbox_hdr *req_hdr;
1422         struct mbox_msghdr *msg;
1423         struct mbox_wq_info *mw;
1424         struct otx2_mbox *mbox;
1425
1426         switch (type) {
1427         case TYPE_AFPF:
1428                 mw = &rvu->afpf_wq_info;
1429                 break;
1430         case TYPE_AFVF:
1431                 mw = &rvu->afvf_wq_info;
1432                 break;
1433         default:
1434                 return;
1435         }
1436
1437         devid = mwork - mw->mbox_wrk;
1438         mbox = &mw->mbox;
1439         mdev = &mbox->dev[devid];
1440
1441         /* Process received mbox messages */
1442         req_hdr = mdev->mbase + mbox->rx_start;
1443         if (req_hdr->num_msgs == 0)
1444                 return;
1445
1446         offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);
1447
1448         for (id = 0; id < req_hdr->num_msgs; id++) {
1449                 msg = mdev->mbase + offset;
1450
1451                 /* Set which PF/VF sent this message based on mbox IRQ */
1452                 switch (type) {
1453                 case TYPE_AFPF:
1454                         msg->pcifunc &=
1455                                 ~(RVU_PFVF_PF_MASK << RVU_PFVF_PF_SHIFT);
1456                         msg->pcifunc |= (devid << RVU_PFVF_PF_SHIFT);
1457                         break;
1458                 case TYPE_AFVF:
1459                         msg->pcifunc &=
1460                                 ~(RVU_PFVF_FUNC_MASK << RVU_PFVF_FUNC_SHIFT);
1461                         msg->pcifunc |= (devid << RVU_PFVF_FUNC_SHIFT) + 1;
1462                         break;
1463                 }
1464
1465                 err = rvu_process_mbox_msg(mbox, devid, msg);
1466                 if (!err) {
1467                         offset = mbox->rx_start + msg->next_msgoff;
1468                         continue;
1469                 }
1470
1471                 if (msg->pcifunc & RVU_PFVF_FUNC_MASK)
1472                         dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d:VF%d\n",
1473                                  err, otx2_mbox_id2name(msg->id),
1474                                  msg->id, devid,
1475                                  (msg->pcifunc & RVU_PFVF_FUNC_MASK) - 1);
1476                 else
1477                         dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d\n",
1478                                  err, otx2_mbox_id2name(msg->id),
1479                                  msg->id, devid);
1480         }
1481
1482         /* Send mbox responses to VF/PF */
1483         otx2_mbox_msg_send(mbox, devid);
1484 }
1485
1486 static inline void rvu_afpf_mbox_handler(struct work_struct *work)
1487 {
1488         struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1489
1490         __rvu_mbox_handler(mwork, TYPE_AFPF);
1491 }
1492
1493 static inline void rvu_afvf_mbox_handler(struct work_struct *work)
1494 {
1495         struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1496
1497         __rvu_mbox_handler(mwork, TYPE_AFVF);
1498 }
1499
1500 static void __rvu_mbox_up_handler(struct rvu_work *mwork, int type)
1501 {
1502         struct rvu *rvu = mwork->rvu;
1503         struct otx2_mbox_dev *mdev;
1504         struct mbox_hdr *rsp_hdr;
1505         struct mbox_msghdr *msg;
1506         struct mbox_wq_info *mw;
1507         struct otx2_mbox *mbox;
1508         int offset, id, devid;
1509
1510         switch (type) {
1511         case TYPE_AFPF:
1512                 mw = &rvu->afpf_wq_info;
1513                 break;
1514         case TYPE_AFVF:
1515                 mw = &rvu->afvf_wq_info;
1516                 break;
1517         default:
1518                 return;
1519         }
1520
1521         devid = mwork - mw->mbox_wrk_up;
1522         mbox = &mw->mbox_up;
1523         mdev = &mbox->dev[devid];
1524
1525         rsp_hdr = mdev->mbase + mbox->rx_start;
1526         if (rsp_hdr->num_msgs == 0) {
1527                 dev_warn(rvu->dev, "mbox up handler: num_msgs = 0\n");
1528                 return;
1529         }
1530
1531         offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
1532
1533         for (id = 0; id < rsp_hdr->num_msgs; id++) {
1534                 msg = mdev->mbase + offset;
1535
1536                 if (msg->id >= MBOX_MSG_MAX) {
1537                         dev_err(rvu->dev,
1538                                 "Mbox msg with unknown ID 0x%x\n", msg->id);
1539                         goto end;
1540                 }
1541
1542                 if (msg->sig != OTX2_MBOX_RSP_SIG) {
1543                         dev_err(rvu->dev,
1544                                 "Mbox msg with wrong signature %x, ID 0x%x\n",
1545                                 msg->sig, msg->id);
1546                         goto end;
1547                 }
1548
1549                 switch (msg->id) {
1550                 case MBOX_MSG_CGX_LINK_EVENT:
1551                         break;
1552                 default:
1553                         if (msg->rc)
1554                                 dev_err(rvu->dev,
1555                                         "Mbox msg response has err %d, ID 0x%x\n",
1556                                         msg->rc, msg->id);
1557                         break;
1558                 }
1559 end:
1560                 offset = mbox->rx_start + msg->next_msgoff;
1561                 mdev->msgs_acked++;
1562         }
1563
1564         otx2_mbox_reset(mbox, devid);
1565 }
1566
1567 static inline void rvu_afpf_mbox_up_handler(struct work_struct *work)
1568 {
1569         struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1570
1571         __rvu_mbox_up_handler(mwork, TYPE_AFPF);
1572 }
1573
1574 static inline void rvu_afvf_mbox_up_handler(struct work_struct *work)
1575 {
1576         struct rvu_work *mwork = container_of(work, struct rvu_work, work);
1577
1578         __rvu_mbox_up_handler(mwork, TYPE_AFVF);
1579 }
1580
1581 static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,
1582                          int type, int num,
1583                          void (mbox_handler)(struct work_struct *),
1584                          void (mbox_up_handler)(struct work_struct *))
1585 {
1586         void __iomem *hwbase = NULL, *reg_base;
1587         int err, i, dir, dir_up;
1588         struct rvu_work *mwork;
1589         const char *name;
1590         u64 bar4_addr;
1591
1592         switch (type) {
1593         case TYPE_AFPF:
1594                 name = "rvu_afpf_mailbox";
1595                 bar4_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PF_BAR4_ADDR);
1596                 dir = MBOX_DIR_AFPF;
1597                 dir_up = MBOX_DIR_AFPF_UP;
1598                 reg_base = rvu->afreg_base;
1599                 break;
1600         case TYPE_AFVF:
1601                 name = "rvu_afvf_mailbox";
1602                 bar4_addr = rvupf_read64(rvu, RVU_PF_VF_BAR4_ADDR);
1603                 dir = MBOX_DIR_PFVF;
1604                 dir_up = MBOX_DIR_PFVF_UP;
1605                 reg_base = rvu->pfreg_base;
1606                 break;
1607         default:
1608                 return -EINVAL;
1609         }
1610
1611         mw->mbox_wq = alloc_workqueue(name,
1612                                       WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM,
1613                                       num);
1614         if (!mw->mbox_wq)
1615                 return -ENOMEM;
1616
1617         mw->mbox_wrk = devm_kcalloc(rvu->dev, num,
1618                                     sizeof(struct rvu_work), GFP_KERNEL);
1619         if (!mw->mbox_wrk) {
1620                 err = -ENOMEM;
1621                 goto exit;
1622         }
1623
1624         mw->mbox_wrk_up = devm_kcalloc(rvu->dev, num,
1625                                        sizeof(struct rvu_work), GFP_KERNEL);
1626         if (!mw->mbox_wrk_up) {
1627                 err = -ENOMEM;
1628                 goto exit;
1629         }
1630
1631         /* Mailbox is a reserved memory (in RAM) region shared between
1632          * RVU devices, shouldn't be mapped as device memory to allow
1633          * unaligned accesses.
1634          */
1635         hwbase = ioremap_wc(bar4_addr, MBOX_SIZE * num);
1636         if (!hwbase) {
1637                 dev_err(rvu->dev, "Unable to map mailbox region\n");
1638                 err = -ENOMEM;
1639                 goto exit;
1640         }
1641
1642         err = otx2_mbox_init(&mw->mbox, hwbase, rvu->pdev, reg_base, dir, num);
1643         if (err)
1644                 goto exit;
1645
1646         err = otx2_mbox_init(&mw->mbox_up, hwbase, rvu->pdev,
1647                              reg_base, dir_up, num);
1648         if (err)
1649                 goto exit;
1650
1651         for (i = 0; i < num; i++) {
1652                 mwork = &mw->mbox_wrk[i];
1653                 mwork->rvu = rvu;
1654                 INIT_WORK(&mwork->work, mbox_handler);
1655
1656                 mwork = &mw->mbox_wrk_up[i];
1657                 mwork->rvu = rvu;
1658                 INIT_WORK(&mwork->work, mbox_up_handler);
1659         }
1660
1661         return 0;
1662 exit:
1663         if (hwbase)
1664                 iounmap((void __iomem *)hwbase);
1665         destroy_workqueue(mw->mbox_wq);
1666         return err;
1667 }
1668
1669 static void rvu_mbox_destroy(struct mbox_wq_info *mw)
1670 {
1671         if (mw->mbox_wq) {
1672                 flush_workqueue(mw->mbox_wq);
1673                 destroy_workqueue(mw->mbox_wq);
1674                 mw->mbox_wq = NULL;
1675         }
1676
1677         if (mw->mbox.hwbase)
1678                 iounmap((void __iomem *)mw->mbox.hwbase);
1679
1680         otx2_mbox_destroy(&mw->mbox);
1681         otx2_mbox_destroy(&mw->mbox_up);
1682 }
1683
1684 static void rvu_queue_work(struct mbox_wq_info *mw, int first,
1685                            int mdevs, u64 intr)
1686 {
1687         struct otx2_mbox_dev *mdev;
1688         struct otx2_mbox *mbox;
1689         struct mbox_hdr *hdr;
1690         int i;
1691
1692         for (i = first; i < mdevs; i++) {
1693                 /* start from 0 */
1694                 if (!(intr & BIT_ULL(i - first)))
1695                         continue;
1696
1697                 mbox = &mw->mbox;
1698                 mdev = &mbox->dev[i];
1699                 hdr = mdev->mbase + mbox->rx_start;
1700                 if (hdr->num_msgs)
1701                         queue_work(mw->mbox_wq, &mw->mbox_wrk[i].work);
1702
1703                 mbox = &mw->mbox_up;
1704                 mdev = &mbox->dev[i];
1705                 hdr = mdev->mbase + mbox->rx_start;
1706                 if (hdr->num_msgs)
1707                         queue_work(mw->mbox_wq, &mw->mbox_wrk_up[i].work);
1708         }
1709 }
1710
1711 static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)
1712 {
1713         struct rvu *rvu = (struct rvu *)rvu_irq;
1714         int vfs = rvu->vfs;
1715         u64 intr;
1716
1717         intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT);
1718         /* Clear interrupts */
1719         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT, intr);
1720
1721         /* Sync with mbox memory region */
1722         rmb();
1723
1724         rvu_queue_work(&rvu->afpf_wq_info, 0, rvu->hw->total_pfs, intr);
1725
1726         /* Handle VF interrupts */
1727         if (vfs > 64) {
1728                 intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(1));
1729                 rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), intr);
1730
1731                 rvu_queue_work(&rvu->afvf_wq_info, 64, vfs, intr);
1732                 vfs -= 64;
1733         }
1734
1735         intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(0));
1736         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), intr);
1737
1738         rvu_queue_work(&rvu->afvf_wq_info, 0, vfs, intr);
1739
1740         return IRQ_HANDLED;
1741 }
1742
1743 static void rvu_enable_mbox_intr(struct rvu *rvu)
1744 {
1745         struct rvu_hwinfo *hw = rvu->hw;
1746
1747         /* Clear spurious irqs, if any */
1748         rvu_write64(rvu, BLKADDR_RVUM,
1749                     RVU_AF_PFAF_MBOX_INT, INTR_MASK(hw->total_pfs));
1750
1751         /* Enable mailbox interrupt for all PFs except PF0 i.e AF itself */
1752         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1S,
1753                     INTR_MASK(hw->total_pfs) & ~1ULL);
1754 }
1755
1756 static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr)
1757 {
1758         struct rvu_block *block;
1759         int slot, lf, num_lfs;
1760         int err;
1761
1762         block = &rvu->hw->block[blkaddr];
1763         num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
1764                                         block->type);
1765         if (!num_lfs)
1766                 return;
1767         for (slot = 0; slot < num_lfs; slot++) {
1768                 lf = rvu_get_lf(rvu, block, pcifunc, slot);
1769                 if (lf < 0)
1770                         continue;
1771
1772                 /* Cleanup LF and reset it */
1773                 if (block->addr == BLKADDR_NIX0)
1774                         rvu_nix_lf_teardown(rvu, pcifunc, block->addr, lf);
1775                 else if (block->addr == BLKADDR_NPA)
1776                         rvu_npa_lf_teardown(rvu, pcifunc, lf);
1777
1778                 err = rvu_lf_reset(rvu, block, lf);
1779                 if (err) {
1780                         dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
1781                                 block->addr, lf);
1782                 }
1783         }
1784 }
1785
1786 static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc)
1787 {
1788         mutex_lock(&rvu->flr_lock);
1789         /* Reset order should reflect inter-block dependencies:
1790          * 1. Reset any packet/work sources (NIX, CPT, TIM)
1791          * 2. Flush and reset SSO/SSOW
1792          * 3. Cleanup pools (NPA)
1793          */
1794         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NIX0);
1795         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_CPT0);
1796         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_TIM);
1797         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSOW);
1798         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSO);
1799         rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NPA);
1800         rvu_detach_rsrcs(rvu, NULL, pcifunc);
1801         mutex_unlock(&rvu->flr_lock);
1802 }
1803
1804 static void rvu_afvf_flr_handler(struct rvu *rvu, int vf)
1805 {
1806         int reg = 0;
1807
1808         /* pcifunc = 0(PF0) | (vf + 1) */
1809         __rvu_flr_handler(rvu, vf + 1);
1810
1811         if (vf >= 64) {
1812                 reg = 1;
1813                 vf = vf - 64;
1814         }
1815
1816         /* Signal FLR finish and enable IRQ */
1817         rvupf_write64(rvu, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
1818         rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
1819 }
1820
1821 static void rvu_flr_handler(struct work_struct *work)
1822 {
1823         struct rvu_work *flrwork = container_of(work, struct rvu_work, work);
1824         struct rvu *rvu = flrwork->rvu;
1825         u16 pcifunc, numvfs, vf;
1826         u64 cfg;
1827         int pf;
1828
1829         pf = flrwork - rvu->flr_wrk;
1830         if (pf >= rvu->hw->total_pfs) {
1831                 rvu_afvf_flr_handler(rvu, pf - rvu->hw->total_pfs);
1832                 return;
1833         }
1834
1835         cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
1836         numvfs = (cfg >> 12) & 0xFF;
1837         pcifunc  = pf << RVU_PFVF_PF_SHIFT;
1838
1839         for (vf = 0; vf < numvfs; vf++)
1840                 __rvu_flr_handler(rvu, (pcifunc | (vf + 1)));
1841
1842         __rvu_flr_handler(rvu, pcifunc);
1843
1844         /* Signal FLR finish */
1845         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND, BIT_ULL(pf));
1846
1847         /* Enable interrupt */
1848         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S,  BIT_ULL(pf));
1849 }
1850
1851 static void rvu_afvf_queue_flr_work(struct rvu *rvu, int start_vf, int numvfs)
1852 {
1853         int dev, vf, reg = 0;
1854         u64 intr;
1855
1856         if (start_vf >= 64)
1857                 reg = 1;
1858
1859         intr = rvupf_read64(rvu, RVU_PF_VFFLR_INTX(reg));
1860         if (!intr)
1861                 return;
1862
1863         for (vf = 0; vf < numvfs; vf++) {
1864                 if (!(intr & BIT_ULL(vf)))
1865                         continue;
1866                 dev = vf + start_vf + rvu->hw->total_pfs;
1867                 queue_work(rvu->flr_wq, &rvu->flr_wrk[dev].work);
1868                 /* Clear and disable the interrupt */
1869                 rvupf_write64(rvu, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
1870                 rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(reg), BIT_ULL(vf));
1871         }
1872 }
1873
1874 static irqreturn_t rvu_flr_intr_handler(int irq, void *rvu_irq)
1875 {
1876         struct rvu *rvu = (struct rvu *)rvu_irq;
1877         u64 intr;
1878         u8  pf;
1879
1880         intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT);
1881         if (!intr)
1882                 goto afvf_flr;
1883
1884         for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
1885                 if (intr & (1ULL << pf)) {
1886                         /* PF is already dead do only AF related operations */
1887                         queue_work(rvu->flr_wq, &rvu->flr_wrk[pf].work);
1888                         /* clear interrupt */
1889                         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT,
1890                                     BIT_ULL(pf));
1891                         /* Disable the interrupt */
1892                         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,
1893                                     BIT_ULL(pf));
1894                 }
1895         }
1896
1897 afvf_flr:
1898         rvu_afvf_queue_flr_work(rvu, 0, 64);
1899         if (rvu->vfs > 64)
1900                 rvu_afvf_queue_flr_work(rvu, 64, rvu->vfs - 64);
1901
1902         return IRQ_HANDLED;
1903 }
1904
1905 static void rvu_me_handle_vfset(struct rvu *rvu, int idx, u64 intr)
1906 {
1907         int vf;
1908
1909         /* Nothing to be done here other than clearing the
1910          * TRPEND bit.
1911          */
1912         for (vf = 0; vf < 64; vf++) {
1913                 if (intr & (1ULL << vf)) {
1914                         /* clear the trpend due to ME(master enable) */
1915                         rvupf_write64(rvu, RVU_PF_VFTRPENDX(idx), BIT_ULL(vf));
1916                         /* clear interrupt */
1917                         rvupf_write64(rvu, RVU_PF_VFME_INTX(idx), BIT_ULL(vf));
1918                 }
1919         }
1920 }
1921
1922 /* Handles ME interrupts from VFs of AF */
1923 static irqreturn_t rvu_me_vf_intr_handler(int irq, void *rvu_irq)
1924 {
1925         struct rvu *rvu = (struct rvu *)rvu_irq;
1926         int vfset;
1927         u64 intr;
1928
1929         intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);
1930
1931         for (vfset = 0; vfset <= 1; vfset++) {
1932                 intr = rvupf_read64(rvu, RVU_PF_VFME_INTX(vfset));
1933                 if (intr)
1934                         rvu_me_handle_vfset(rvu, vfset, intr);
1935         }
1936
1937         return IRQ_HANDLED;
1938 }
1939
1940 /* Handles ME interrupts from PFs */
1941 static irqreturn_t rvu_me_pf_intr_handler(int irq, void *rvu_irq)
1942 {
1943         struct rvu *rvu = (struct rvu *)rvu_irq;
1944         u64 intr;
1945         u8  pf;
1946
1947         intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);
1948
1949         /* Nothing to be done here other than clearing the
1950          * TRPEND bit.
1951          */
1952         for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
1953                 if (intr & (1ULL << pf)) {
1954                         /* clear the trpend due to ME(master enable) */
1955                         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND,
1956                                     BIT_ULL(pf));
1957                         /* clear interrupt */
1958                         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT,
1959                                     BIT_ULL(pf));
1960                 }
1961         }
1962
1963         return IRQ_HANDLED;
1964 }
1965
1966 static void rvu_unregister_interrupts(struct rvu *rvu)
1967 {
1968         int irq;
1969
1970         /* Disable the Mbox interrupt */
1971         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1C,
1972                     INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
1973
1974         /* Disable the PF FLR interrupt */
1975         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,
1976                     INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
1977
1978         /* Disable the PF ME interrupt */
1979         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1C,
1980                     INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
1981
1982         for (irq = 0; irq < rvu->num_vec; irq++) {
1983                 if (rvu->irq_allocated[irq])
1984                         free_irq(pci_irq_vector(rvu->pdev, irq), rvu);
1985         }
1986
1987         pci_free_irq_vectors(rvu->pdev);
1988         rvu->num_vec = 0;
1989 }
1990
1991 static int rvu_afvf_msix_vectors_num_ok(struct rvu *rvu)
1992 {
1993         struct rvu_pfvf *pfvf = &rvu->pf[0];
1994         int offset;
1995
1996         pfvf = &rvu->pf[0];
1997         offset = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
1998
1999         /* Make sure there are enough MSIX vectors configured so that
2000          * VF interrupts can be handled. Offset equal to zero means
2001          * that PF vectors are not configured and overlapping AF vectors.
2002          */
2003         return (pfvf->msix.max >= RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT) &&
2004                offset;
2005 }
2006
2007 static int rvu_register_interrupts(struct rvu *rvu)
2008 {
2009         int ret, offset, pf_vec_start;
2010
2011         rvu->num_vec = pci_msix_vec_count(rvu->pdev);
2012
2013         rvu->irq_name = devm_kmalloc_array(rvu->dev, rvu->num_vec,
2014                                            NAME_SIZE, GFP_KERNEL);
2015         if (!rvu->irq_name)
2016                 return -ENOMEM;
2017
2018         rvu->irq_allocated = devm_kcalloc(rvu->dev, rvu->num_vec,
2019                                           sizeof(bool), GFP_KERNEL);
2020         if (!rvu->irq_allocated)
2021                 return -ENOMEM;
2022
2023         /* Enable MSI-X */
2024         ret = pci_alloc_irq_vectors(rvu->pdev, rvu->num_vec,
2025                                     rvu->num_vec, PCI_IRQ_MSIX);
2026         if (ret < 0) {
2027                 dev_err(rvu->dev,
2028                         "RVUAF: Request for %d msix vectors failed, ret %d\n",
2029                         rvu->num_vec, ret);
2030                 return ret;
2031         }
2032
2033         /* Register mailbox interrupt handler */
2034         sprintf(&rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], "RVUAF Mbox");
2035         ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_MBOX),
2036                           rvu_mbox_intr_handler, 0,
2037                           &rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], rvu);
2038         if (ret) {
2039                 dev_err(rvu->dev,
2040                         "RVUAF: IRQ registration failed for mbox irq\n");
2041                 goto fail;
2042         }
2043
2044         rvu->irq_allocated[RVU_AF_INT_VEC_MBOX] = true;
2045
2046         /* Enable mailbox interrupts from all PFs */
2047         rvu_enable_mbox_intr(rvu);
2048
2049         /* Register FLR interrupt handler */
2050         sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],
2051                 "RVUAF FLR");
2052         ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFFLR),
2053                           rvu_flr_intr_handler, 0,
2054                           &rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],
2055                           rvu);
2056         if (ret) {
2057                 dev_err(rvu->dev,
2058                         "RVUAF: IRQ registration failed for FLR\n");
2059                 goto fail;
2060         }
2061         rvu->irq_allocated[RVU_AF_INT_VEC_PFFLR] = true;
2062
2063         /* Enable FLR interrupt for all PFs*/
2064         rvu_write64(rvu, BLKADDR_RVUM,
2065                     RVU_AF_PFFLR_INT, INTR_MASK(rvu->hw->total_pfs));
2066
2067         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S,
2068                     INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2069
2070         /* Register ME interrupt handler */
2071         sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],
2072                 "RVUAF ME");
2073         ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFME),
2074                           rvu_me_pf_intr_handler, 0,
2075                           &rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],
2076                           rvu);
2077         if (ret) {
2078                 dev_err(rvu->dev,
2079                         "RVUAF: IRQ registration failed for ME\n");
2080         }
2081         rvu->irq_allocated[RVU_AF_INT_VEC_PFME] = true;
2082
2083         /* Enable ME interrupt for all PFs*/
2084         rvu_write64(rvu, BLKADDR_RVUM,
2085                     RVU_AF_PFME_INT, INTR_MASK(rvu->hw->total_pfs));
2086
2087         rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1S,
2088                     INTR_MASK(rvu->hw->total_pfs) & ~1ULL);
2089
2090         if (!rvu_afvf_msix_vectors_num_ok(rvu))
2091                 return 0;
2092
2093         /* Get PF MSIX vectors offset. */
2094         pf_vec_start = rvu_read64(rvu, BLKADDR_RVUM,
2095                                   RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;
2096
2097         /* Register MBOX0 interrupt. */
2098         offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX0;
2099         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox0");
2100         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2101                           rvu_mbox_intr_handler, 0,
2102                           &rvu->irq_name[offset * NAME_SIZE],
2103                           rvu);
2104         if (ret)
2105                 dev_err(rvu->dev,
2106                         "RVUAF: IRQ registration failed for Mbox0\n");
2107
2108         rvu->irq_allocated[offset] = true;
2109
2110         /* Register MBOX1 interrupt. MBOX1 IRQ number follows MBOX0 so
2111          * simply increment current offset by 1.
2112          */
2113         offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX1;
2114         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox1");
2115         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2116                           rvu_mbox_intr_handler, 0,
2117                           &rvu->irq_name[offset * NAME_SIZE],
2118                           rvu);
2119         if (ret)
2120                 dev_err(rvu->dev,
2121                         "RVUAF: IRQ registration failed for Mbox1\n");
2122
2123         rvu->irq_allocated[offset] = true;
2124
2125         /* Register FLR interrupt handler for AF's VFs */
2126         offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR0;
2127         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR0");
2128         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2129                           rvu_flr_intr_handler, 0,
2130                           &rvu->irq_name[offset * NAME_SIZE], rvu);
2131         if (ret) {
2132                 dev_err(rvu->dev,
2133                         "RVUAF: IRQ registration failed for RVUAFVF FLR0\n");
2134                 goto fail;
2135         }
2136         rvu->irq_allocated[offset] = true;
2137
2138         offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR1;
2139         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR1");
2140         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2141                           rvu_flr_intr_handler, 0,
2142                           &rvu->irq_name[offset * NAME_SIZE], rvu);
2143         if (ret) {
2144                 dev_err(rvu->dev,
2145                         "RVUAF: IRQ registration failed for RVUAFVF FLR1\n");
2146                 goto fail;
2147         }
2148         rvu->irq_allocated[offset] = true;
2149
2150         /* Register ME interrupt handler for AF's VFs */
2151         offset = pf_vec_start + RVU_PF_INT_VEC_VFME0;
2152         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME0");
2153         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2154                           rvu_me_vf_intr_handler, 0,
2155                           &rvu->irq_name[offset * NAME_SIZE], rvu);
2156         if (ret) {
2157                 dev_err(rvu->dev,
2158                         "RVUAF: IRQ registration failed for RVUAFVF ME0\n");
2159                 goto fail;
2160         }
2161         rvu->irq_allocated[offset] = true;
2162
2163         offset = pf_vec_start + RVU_PF_INT_VEC_VFME1;
2164         sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME1");
2165         ret = request_irq(pci_irq_vector(rvu->pdev, offset),
2166                           rvu_me_vf_intr_handler, 0,
2167                           &rvu->irq_name[offset * NAME_SIZE], rvu);
2168         if (ret) {
2169                 dev_err(rvu->dev,
2170                         "RVUAF: IRQ registration failed for RVUAFVF ME1\n");
2171                 goto fail;
2172         }
2173         rvu->irq_allocated[offset] = true;
2174         return 0;
2175
2176 fail:
2177         rvu_unregister_interrupts(rvu);
2178         return ret;
2179 }
2180
2181 static void rvu_flr_wq_destroy(struct rvu *rvu)
2182 {
2183         if (rvu->flr_wq) {
2184                 flush_workqueue(rvu->flr_wq);
2185                 destroy_workqueue(rvu->flr_wq);
2186                 rvu->flr_wq = NULL;
2187         }
2188 }
2189
2190 static int rvu_flr_init(struct rvu *rvu)
2191 {
2192         int dev, num_devs;
2193         u64 cfg;
2194         int pf;
2195
2196         /* Enable FLR for all PFs*/
2197         for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
2198                 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
2199                 rvu_write64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf),
2200                             cfg | BIT_ULL(22));
2201         }
2202
2203         rvu->flr_wq = alloc_workqueue("rvu_afpf_flr",
2204                                       WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM,
2205                                        1);
2206         if (!rvu->flr_wq)
2207                 return -ENOMEM;
2208
2209         num_devs = rvu->hw->total_pfs + pci_sriov_get_totalvfs(rvu->pdev);
2210         rvu->flr_wrk = devm_kcalloc(rvu->dev, num_devs,
2211                                     sizeof(struct rvu_work), GFP_KERNEL);
2212         if (!rvu->flr_wrk) {
2213                 destroy_workqueue(rvu->flr_wq);
2214                 return -ENOMEM;
2215         }
2216
2217         for (dev = 0; dev < num_devs; dev++) {
2218                 rvu->flr_wrk[dev].rvu = rvu;
2219                 INIT_WORK(&rvu->flr_wrk[dev].work, rvu_flr_handler);
2220         }
2221
2222         mutex_init(&rvu->flr_lock);
2223
2224         return 0;
2225 }
2226
2227 static void rvu_disable_afvf_intr(struct rvu *rvu)
2228 {
2229         int vfs = rvu->vfs;
2230
2231         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), INTR_MASK(vfs));
2232         rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));
2233         rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));
2234         if (vfs <= 64)
2235                 return;
2236
2237         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1),
2238                       INTR_MASK(vfs - 64));
2239         rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
2240         rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
2241 }
2242
2243 static void rvu_enable_afvf_intr(struct rvu *rvu)
2244 {
2245         int vfs = rvu->vfs;
2246
2247         /* Clear any pending interrupts and enable AF VF interrupts for
2248          * the first 64 VFs.
2249          */
2250         /* Mbox */
2251         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), INTR_MASK(vfs));
2252         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(vfs));
2253
2254         /* FLR */
2255         rvupf_write64(rvu, RVU_PF_VFFLR_INTX(0), INTR_MASK(vfs));
2256         rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(vfs));
2257         rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(vfs));
2258
2259         /* Same for remaining VFs, if any. */
2260         if (vfs <= 64)
2261                 return;
2262
2263         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), INTR_MASK(vfs - 64));
2264         rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
2265                       INTR_MASK(vfs - 64));
2266
2267         rvupf_write64(rvu, RVU_PF_VFFLR_INTX(1), INTR_MASK(vfs - 64));
2268         rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));
2269         rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));
2270 }
2271
2272 #define PCI_DEVID_OCTEONTX2_LBK 0xA061
2273
2274 static int lbk_get_num_chans(void)
2275 {
2276         struct pci_dev *pdev;
2277         void __iomem *base;
2278         int ret = -EIO;
2279
2280         pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_LBK,
2281                               NULL);
2282         if (!pdev)
2283                 goto err;
2284
2285         base = pci_ioremap_bar(pdev, 0);
2286         if (!base)
2287                 goto err_put;
2288
2289         /* Read number of available LBK channels from LBK(0)_CONST register. */
2290         ret = (readq(base + 0x10) >> 32) & 0xffff;
2291         iounmap(base);
2292 err_put:
2293         pci_dev_put(pdev);
2294 err:
2295         return ret;
2296 }
2297
2298 static int rvu_enable_sriov(struct rvu *rvu)
2299 {
2300         struct pci_dev *pdev = rvu->pdev;
2301         int err, chans, vfs;
2302
2303         if (!rvu_afvf_msix_vectors_num_ok(rvu)) {
2304                 dev_warn(&pdev->dev,
2305                          "Skipping SRIOV enablement since not enough IRQs are available\n");
2306                 return 0;
2307         }
2308
2309         chans = lbk_get_num_chans();
2310         if (chans < 0)
2311                 return chans;
2312
2313         vfs = pci_sriov_get_totalvfs(pdev);
2314
2315         /* Limit VFs in case we have more VFs than LBK channels available. */
2316         if (vfs > chans)
2317                 vfs = chans;
2318
2319         /* AF's VFs work in pairs and talk over consecutive loopback channels.
2320          * Thus we want to enable maximum even number of VFs. In case
2321          * odd number of VFs are available then the last VF on the list
2322          * remains disabled.
2323          */
2324         if (vfs & 0x1) {
2325                 dev_warn(&pdev->dev,
2326                          "Number of VFs should be even. Enabling %d out of %d.\n",
2327                          vfs - 1, vfs);
2328                 vfs--;
2329         }
2330
2331         if (!vfs)
2332                 return 0;
2333
2334         /* Save VFs number for reference in VF interrupts handlers.
2335          * Since interrupts might start arriving during SRIOV enablement
2336          * ordinary API cannot be used to get number of enabled VFs.
2337          */
2338         rvu->vfs = vfs;
2339
2340         err = rvu_mbox_init(rvu, &rvu->afvf_wq_info, TYPE_AFVF, vfs,
2341                             rvu_afvf_mbox_handler, rvu_afvf_mbox_up_handler);
2342         if (err)
2343                 return err;
2344
2345         rvu_enable_afvf_intr(rvu);
2346         /* Make sure IRQs are enabled before SRIOV. */
2347         mb();
2348
2349         err = pci_enable_sriov(pdev, vfs);
2350         if (err) {
2351                 rvu_disable_afvf_intr(rvu);
2352                 rvu_mbox_destroy(&rvu->afvf_wq_info);
2353                 return err;
2354         }
2355
2356         return 0;
2357 }
2358
2359 static void rvu_disable_sriov(struct rvu *rvu)
2360 {
2361         rvu_disable_afvf_intr(rvu);
2362         rvu_mbox_destroy(&rvu->afvf_wq_info);
2363         pci_disable_sriov(rvu->pdev);
2364 }
2365
2366 static void rvu_update_module_params(struct rvu *rvu)
2367 {
2368         const char *default_pfl_name = "default";
2369
2370         strscpy(rvu->mkex_pfl_name,
2371                 mkex_profile ? mkex_profile : default_pfl_name, MKEX_NAME_LEN);
2372 }
2373
2374 static int rvu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2375 {
2376         struct device *dev = &pdev->dev;
2377         struct rvu *rvu;
2378         int    err;
2379
2380         rvu = devm_kzalloc(dev, sizeof(*rvu), GFP_KERNEL);
2381         if (!rvu)
2382                 return -ENOMEM;
2383
2384         rvu->hw = devm_kzalloc(dev, sizeof(struct rvu_hwinfo), GFP_KERNEL);
2385         if (!rvu->hw) {
2386                 devm_kfree(dev, rvu);
2387                 return -ENOMEM;
2388         }
2389
2390         pci_set_drvdata(pdev, rvu);
2391         rvu->pdev = pdev;
2392         rvu->dev = &pdev->dev;
2393
2394         err = pci_enable_device(pdev);
2395         if (err) {
2396                 dev_err(dev, "Failed to enable PCI device\n");
2397                 goto err_freemem;
2398         }
2399
2400         err = pci_request_regions(pdev, DRV_NAME);
2401         if (err) {
2402                 dev_err(dev, "PCI request regions failed 0x%x\n", err);
2403                 goto err_disable_device;
2404         }
2405
2406         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
2407         if (err) {
2408                 dev_err(dev, "Unable to set DMA mask\n");
2409                 goto err_release_regions;
2410         }
2411
2412         err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
2413         if (err) {
2414                 dev_err(dev, "Unable to set consistent DMA mask\n");
2415                 goto err_release_regions;
2416         }
2417
2418         /* Map Admin function CSRs */
2419         rvu->afreg_base = pcim_iomap(pdev, PCI_AF_REG_BAR_NUM, 0);
2420         rvu->pfreg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0);
2421         if (!rvu->afreg_base || !rvu->pfreg_base) {
2422                 dev_err(dev, "Unable to map admin function CSRs, aborting\n");
2423                 err = -ENOMEM;
2424                 goto err_release_regions;
2425         }
2426
2427         /* Store module params in rvu structure */
2428         rvu_update_module_params(rvu);
2429
2430         /* Check which blocks the HW supports */
2431         rvu_check_block_implemented(rvu);
2432
2433         rvu_reset_all_blocks(rvu);
2434
2435         err = rvu_setup_hw_resources(rvu);
2436         if (err)
2437                 goto err_release_regions;
2438
2439         /* Init mailbox btw AF and PFs */
2440         err = rvu_mbox_init(rvu, &rvu->afpf_wq_info, TYPE_AFPF,
2441                             rvu->hw->total_pfs, rvu_afpf_mbox_handler,
2442                             rvu_afpf_mbox_up_handler);
2443         if (err)
2444                 goto err_hwsetup;
2445
2446         err = rvu_flr_init(rvu);
2447         if (err)
2448                 goto err_mbox;
2449
2450         err = rvu_register_interrupts(rvu);
2451         if (err)
2452                 goto err_flr;
2453
2454         /* Enable AF's VFs (if any) */
2455         err = rvu_enable_sriov(rvu);
2456         if (err)
2457                 goto err_irq;
2458
2459         /* Initialize debugfs */
2460         rvu_dbg_init(rvu);
2461
2462         return 0;
2463 err_irq:
2464         rvu_unregister_interrupts(rvu);
2465 err_flr:
2466         rvu_flr_wq_destroy(rvu);
2467 err_mbox:
2468         rvu_mbox_destroy(&rvu->afpf_wq_info);
2469 err_hwsetup:
2470         rvu_cgx_exit(rvu);
2471         rvu_reset_all_blocks(rvu);
2472         rvu_free_hw_resources(rvu);
2473 err_release_regions:
2474         pci_release_regions(pdev);
2475 err_disable_device:
2476         pci_disable_device(pdev);
2477 err_freemem:
2478         pci_set_drvdata(pdev, NULL);
2479         devm_kfree(&pdev->dev, rvu->hw);
2480         devm_kfree(dev, rvu);
2481         return err;
2482 }
2483
2484 static void rvu_remove(struct pci_dev *pdev)
2485 {
2486         struct rvu *rvu = pci_get_drvdata(pdev);
2487
2488         rvu_dbg_exit(rvu);
2489         rvu_unregister_interrupts(rvu);
2490         rvu_flr_wq_destroy(rvu);
2491         rvu_cgx_exit(rvu);
2492         rvu_mbox_destroy(&rvu->afpf_wq_info);
2493         rvu_disable_sriov(rvu);
2494         rvu_reset_all_blocks(rvu);
2495         rvu_free_hw_resources(rvu);
2496
2497         pci_release_regions(pdev);
2498         pci_disable_device(pdev);
2499         pci_set_drvdata(pdev, NULL);
2500
2501         devm_kfree(&pdev->dev, rvu->hw);
2502         devm_kfree(&pdev->dev, rvu);
2503 }
2504
2505 static struct pci_driver rvu_driver = {
2506         .name = DRV_NAME,
2507         .id_table = rvu_id_table,
2508         .probe = rvu_probe,
2509         .remove = rvu_remove,
2510 };
2511
2512 static int __init rvu_init_module(void)
2513 {
2514         int err;
2515
2516         pr_info("%s: %s\n", DRV_NAME, DRV_STRING);
2517
2518         err = pci_register_driver(&cgx_driver);
2519         if (err < 0)
2520                 return err;
2521
2522         err =  pci_register_driver(&rvu_driver);
2523         if (err < 0)
2524                 pci_unregister_driver(&cgx_driver);
2525
2526         return err;
2527 }
2528
2529 static void __exit rvu_cleanup_module(void)
2530 {
2531         pci_unregister_driver(&rvu_driver);
2532         pci_unregister_driver(&cgx_driver);
2533 }
2534
2535 module_init(rvu_init_module);
2536 module_exit(rvu_cleanup_module);