drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell RVU Admin Function driver
   3  *
   4  * Copyright (C) 2018 Marvell.
   5  *
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/pci.h>
  10
  11 #include "rvu_struct.h"
  12 #include "rvu_reg.h"
  13 #include "rvu.h"
  14 #include "npc.h"
  15 #include "mcs.h"
  16 #include "cgx.h"
  17 #include "lmac_common.h"
  18 #include "rvu_npc_hash.h"
  19
  20 static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc);
  21 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
  22                             int type, int chan_id);
  23 static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc,
  24                                int type, bool add);
  25 static int nix_setup_ipolicers(struct rvu *rvu,
  26                                struct nix_hw *nix_hw, int blkaddr);
  27 static void nix_ipolicer_freemem(struct rvu *rvu, struct nix_hw *nix_hw);
  28 static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req,
  29                                struct nix_hw *nix_hw, u16 pcifunc);
  30 static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc);
  31 static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw,
  32                                      u32 leaf_prof);
  33 static const char *nix_get_ctx_name(int ctype);
  34
  35 enum mc_tbl_sz {
  36         MC_TBL_SZ_256,
  37         MC_TBL_SZ_512,
  38         MC_TBL_SZ_1K,
  39         MC_TBL_SZ_2K,
  40         MC_TBL_SZ_4K,
  41         MC_TBL_SZ_8K,
  42         MC_TBL_SZ_16K,
  43         MC_TBL_SZ_32K,
  44         MC_TBL_SZ_64K,
  45 };
  46
  47 enum mc_buf_cnt {
  48         MC_BUF_CNT_8,
  49         MC_BUF_CNT_16,
  50         MC_BUF_CNT_32,
  51         MC_BUF_CNT_64,
  52         MC_BUF_CNT_128,
  53         MC_BUF_CNT_256,
  54         MC_BUF_CNT_512,
  55         MC_BUF_CNT_1024,
  56         MC_BUF_CNT_2048,
  57 };
  58
  59 enum nix_makr_fmt_indexes {
  60         NIX_MARK_CFG_IP_DSCP_RED,
  61         NIX_MARK_CFG_IP_DSCP_YELLOW,
  62         NIX_MARK_CFG_IP_DSCP_YELLOW_RED,
  63         NIX_MARK_CFG_IP_ECN_RED,
  64         NIX_MARK_CFG_IP_ECN_YELLOW,
  65         NIX_MARK_CFG_IP_ECN_YELLOW_RED,
  66         NIX_MARK_CFG_VLAN_DEI_RED,
  67         NIX_MARK_CFG_VLAN_DEI_YELLOW,
  68         NIX_MARK_CFG_VLAN_DEI_YELLOW_RED,
  69         NIX_MARK_CFG_MAX,
  70 };
  71
  72 /* For now considering MC resources needed for broadcast
  73  * pkt replication only. i.e 256 HWVFs + 12 PFs.
  74  */
  75 #define MC_TBL_SIZE     MC_TBL_SZ_2K
  76 #define MC_BUF_CNT      MC_BUF_CNT_1024
  77
  78 #define MC_TX_MAX       2048
  79
  80 struct mce {
  81         struct hlist_node       node;
  82         u32                     rq_rss_index;
  83         u16                     pcifunc;
  84         u16                     channel;
  85         u8                      dest_type;
  86         u8                      is_active;
  87         u8                      reserved[2];
  88 };
  89
  90 int rvu_get_next_nix_blkaddr(struct rvu *rvu, int blkaddr)
  91 {
  92         int i = 0;
  93
  94         /*If blkaddr is 0, return the first nix block address*/
  95         if (blkaddr == 0)
  96                 return rvu->nix_blkaddr[blkaddr];
  97
  98         while (i + 1 < MAX_NIX_BLKS) {
  99                 if (rvu->nix_blkaddr[i] == blkaddr)
 100                         return rvu->nix_blkaddr[i + 1];
 101                 i++;
 102         }
 103
 104         return 0;
 105 }
 106
 107 bool is_nixlf_attached(struct rvu *rvu, u16 pcifunc)
 108 {
 109         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
 110         int blkaddr;
 111
 112         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
 113         if (!pfvf->nixlf || blkaddr < 0)
 114                 return false;
 115         return true;
 116 }
 117
 118 int rvu_get_nixlf_count(struct rvu *rvu)
 119 {
 120         int blkaddr = 0, max = 0;
 121         struct rvu_block *block;
 122
 123         blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
 124         while (blkaddr) {
 125                 block = &rvu->hw->block[blkaddr];
 126                 max += block->lf.max;
 127                 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
 128         }
 129         return max;
 130 }
 131
 132 int nix_get_nixlf(struct rvu *rvu, u16 pcifunc, int *nixlf, int *nix_blkaddr)
 133 {
 134         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
 135         struct rvu_hwinfo *hw = rvu->hw;
 136         int blkaddr;
 137
 138         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
 139         if (!pfvf->nixlf || blkaddr < 0)
 140                 return NIX_AF_ERR_AF_LF_INVALID;
 141
 142         *nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
 143         if (*nixlf < 0)
 144                 return NIX_AF_ERR_AF_LF_INVALID;
 145
 146         if (nix_blkaddr)
 147                 *nix_blkaddr = blkaddr;
 148
 149         return 0;
 150 }
 151
 152 int nix_get_struct_ptrs(struct rvu *rvu, u16 pcifunc,
 153                         struct nix_hw **nix_hw, int *blkaddr)
 154 {
 155         struct rvu_pfvf *pfvf;
 156
 157         pfvf = rvu_get_pfvf(rvu, pcifunc);
 158         *blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
 159         if (!pfvf->nixlf || *blkaddr < 0)
 160                 return NIX_AF_ERR_AF_LF_INVALID;
 161
 162         *nix_hw = get_nix_hw(rvu->hw, *blkaddr);
 163         if (!*nix_hw)
 164                 return NIX_AF_ERR_INVALID_NIXBLK;
 165         return 0;
 166 }
 167
 168 static void nix_mce_list_init(struct nix_mce_list *list, int max)
 169 {
 170         INIT_HLIST_HEAD(&list->head);
 171         list->count = 0;
 172         list->max = max;
 173 }
 174
 175 static int nix_alloc_mce_list(struct nix_mcast *mcast, int count, u8 dir)
 176 {
 177         struct rsrc_bmap *mce_counter;
 178         int idx;
 179
 180         if (!mcast)
 181                 return -EINVAL;
 182
 183         mce_counter = &mcast->mce_counter[dir];
 184         if (!rvu_rsrc_check_contig(mce_counter, count))
 185                 return -ENOSPC;
 186
 187         idx = rvu_alloc_rsrc_contig(mce_counter, count);
 188         return idx;
 189 }
 190
 191 static void nix_free_mce_list(struct nix_mcast *mcast, int count, int start, u8 dir)
 192 {
 193         struct rsrc_bmap *mce_counter;
 194
 195         if (!mcast)
 196                 return;
 197
 198         mce_counter = &mcast->mce_counter[dir];
 199         rvu_free_rsrc_contig(mce_counter, count, start);
 200 }
 201
 202 struct nix_hw *get_nix_hw(struct rvu_hwinfo *hw, int blkaddr)
 203 {
 204         int nix_blkaddr = 0, i = 0;
 205         struct rvu *rvu = hw->rvu;
 206
 207         nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr);
 208         while (nix_blkaddr) {
 209                 if (blkaddr == nix_blkaddr && hw->nix)
 210                         return &hw->nix[i];
 211                 nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr);
 212                 i++;
 213         }
 214         return NULL;
 215 }
 216
 217 int nix_get_dwrr_mtu_reg(struct rvu_hwinfo *hw, int smq_link_type)
 218 {
 219         if (hw->cap.nix_multiple_dwrr_mtu)
 220                 return NIX_AF_DWRR_MTUX(smq_link_type);
 221
 222         if (smq_link_type == SMQ_LINK_TYPE_SDP)
 223                 return NIX_AF_DWRR_SDP_MTU;
 224
 225         /* Here it's same reg for RPM and LBK */
 226         return NIX_AF_DWRR_RPM_MTU;
 227 }
 228
 229 u32 convert_dwrr_mtu_to_bytes(u8 dwrr_mtu)
 230 {
 231         dwrr_mtu &= 0x1FULL;
 232
 233         /* MTU used for DWRR calculation is in power of 2 up until 64K bytes.
 234          * Value of 4 is reserved for MTU value of 9728 bytes.
 235          * Value of 5 is reserved for MTU value of 10240 bytes.
 236          */
 237         switch (dwrr_mtu) {
 238         case 4:
 239                 return 9728;
 240         case 5:
 241                 return 10240;
 242         default:
 243                 return BIT_ULL(dwrr_mtu);
 244         }
 245
 246         return 0;
 247 }
 248
 249 u32 convert_bytes_to_dwrr_mtu(u32 bytes)
 250 {
 251         /* MTU used for DWRR calculation is in power of 2 up until 64K bytes.
 252          * Value of 4 is reserved for MTU value of 9728 bytes.
 253          * Value of 5 is reserved for MTU value of 10240 bytes.
 254          */
 255         if (bytes > BIT_ULL(16))
 256                 return 0;
 257
 258         switch (bytes) {
 259         case 9728:
 260                 return 4;
 261         case 10240:
 262                 return 5;
 263         default:
 264                 return ilog2(bytes);
 265         }
 266
 267         return 0;
 268 }
 269
 270 static void nix_rx_sync(struct rvu *rvu, int blkaddr)
 271 {
 272         int err;
 273
 274         /* Sync all in flight RX packets to LLC/DRAM */
 275         rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0));
 276         err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true);
 277         if (err)
 278                 dev_err(rvu->dev, "SYNC1: NIX RX software sync failed\n");
 279
 280         /* SW_SYNC ensures all existing transactions are finished and pkts
 281          * are written to LLC/DRAM, queues should be teared down after
 282          * successful SW_SYNC. Due to a HW errata, in some rare scenarios
 283          * an existing transaction might end after SW_SYNC operation. To
 284          * ensure operation is fully done, do the SW_SYNC twice.
 285          */
 286         rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0));
 287         err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true);
 288         if (err)
 289                 dev_err(rvu->dev, "SYNC2: NIX RX software sync failed\n");
 290 }
 291
 292 static bool is_valid_txschq(struct rvu *rvu, int blkaddr,
 293                             int lvl, u16 pcifunc, u16 schq)
 294 {
 295         struct rvu_hwinfo *hw = rvu->hw;
 296         struct nix_txsch *txsch;
 297         struct nix_hw *nix_hw;
 298         u16 map_func;
 299
 300         nix_hw = get_nix_hw(rvu->hw, blkaddr);
 301         if (!nix_hw)
 302                 return false;
 303
 304         txsch = &nix_hw->txsch[lvl];
 305         /* Check out of bounds */
 306         if (schq >= txsch->schq.max)
 307                 return false;
 308
 309         mutex_lock(&rvu->rsrc_lock);
 310         map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]);
 311         mutex_unlock(&rvu->rsrc_lock);
 312
 313         /* TLs aggegating traffic are shared across PF and VFs */
 314         if (lvl >= hw->cap.nix_tx_aggr_lvl) {
 315                 if (rvu_get_pf(map_func) != rvu_get_pf(pcifunc))
 316                         return false;
 317                 else
 318                         return true;
 319         }
 320
 321         if (map_func != pcifunc)
 322                 return false;
 323
 324         return true;
 325 }
 326
 327 static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf,
 328                               struct nix_lf_alloc_rsp *rsp, bool loop)
 329 {
 330         struct rvu_pfvf *parent_pf, *pfvf = rvu_get_pfvf(rvu, pcifunc);
 331         u16 req_chan_base, req_chan_end, req_chan_cnt;
 332         struct rvu_hwinfo *hw = rvu->hw;
 333         struct sdp_node_info *sdp_info;
 334         int pkind, pf, vf, lbkid, vfid;
 335         u8 cgx_id, lmac_id;
 336         bool from_vf;
 337         int err;
 338
 339         pf = rvu_get_pf(pcifunc);
 340         if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK &&
 341             type != NIX_INTF_TYPE_SDP)
 342                 return 0;
 343
 344         switch (type) {
 345         case NIX_INTF_TYPE_CGX:
 346                 pfvf->cgx_lmac = rvu->pf2cgxlmac_map[pf];
 347                 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id);
 348
 349                 pkind = rvu_npc_get_pkind(rvu, pf);
 350                 if (pkind < 0) {
 351                         dev_err(rvu->dev,
 352                                 "PF_Func 0x%x: Invalid pkind\n", pcifunc);
 353                         return -EINVAL;
 354                 }
 355                 pfvf->rx_chan_base = rvu_nix_chan_cgx(rvu, cgx_id, lmac_id, 0);
 356                 pfvf->tx_chan_base = pfvf->rx_chan_base;
 357                 pfvf->rx_chan_cnt = 1;
 358                 pfvf->tx_chan_cnt = 1;
 359                 rsp->tx_link = cgx_id * hw->lmac_per_cgx + lmac_id;
 360
 361                 cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, pkind);
 362                 rvu_npc_set_pkind(rvu, pkind, pfvf);
 363
 364                 break;
 365         case NIX_INTF_TYPE_LBK:
 366                 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
 367
 368                 /* If NIX1 block is present on the silicon then NIXes are
 369                  * assigned alternatively for lbk interfaces. NIX0 should
 370                  * send packets on lbk link 1 channels and NIX1 should send
 371                  * on lbk link 0 channels for the communication between
 372                  * NIX0 and NIX1.
 373                  */
 374                 lbkid = 0;
 375                 if (rvu->hw->lbk_links > 1)
 376                         lbkid = vf & 0x1 ? 0 : 1;
 377
 378                 /* By default NIX0 is configured to send packet on lbk link 1
 379                  * (which corresponds to LBK1), same packet will receive on
 380                  * NIX1 over lbk link 0. If NIX1 sends packet on lbk link 0
 381                  * (which corresponds to LBK2) packet will receive on NIX0 lbk
 382                  * link 1.
 383                  * But if lbk links for NIX0 and NIX1 are negated, i.e NIX0
 384                  * transmits and receives on lbk link 0, whick corresponds
 385                  * to LBK1 block, back to back connectivity between NIX and
 386                  * LBK can be achieved (which is similar to 96xx)
 387                  *
 388                  *                      RX              TX
 389                  * NIX0 lbk link        1 (LBK2)        1 (LBK1)
 390                  * NIX0 lbk link        0 (LBK0)        0 (LBK0)
 391                  * NIX1 lbk link        0 (LBK1)        0 (LBK2)
 392                  * NIX1 lbk link        1 (LBK3)        1 (LBK3)
 393                  */
 394                 if (loop)
 395                         lbkid = !lbkid;
 396
 397                 /* Note that AF's VFs work in pairs and talk over consecutive
 398                  * loopback channels.Therefore if odd number of AF VFs are
 399                  * enabled then the last VF remains with no pair.
 400                  */
 401                 pfvf->rx_chan_base = rvu_nix_chan_lbk(rvu, lbkid, vf);
 402                 pfvf->tx_chan_base = vf & 0x1 ?
 403                                         rvu_nix_chan_lbk(rvu, lbkid, vf - 1) :
 404                                         rvu_nix_chan_lbk(rvu, lbkid, vf + 1);
 405                 pfvf->rx_chan_cnt = 1;
 406                 pfvf->tx_chan_cnt = 1;
 407                 rsp->tx_link = hw->cgx_links + lbkid;
 408                 pfvf->lbkid = lbkid;
 409                 rvu_npc_set_pkind(rvu, NPC_RX_LBK_PKIND, pfvf);
 410                 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf,
 411                                               pfvf->rx_chan_base,
 412                                               pfvf->rx_chan_cnt);
 413
 414                 break;
 415         case NIX_INTF_TYPE_SDP:
 416                 from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK);
 417                 parent_pf = &rvu->pf[rvu_get_pf(pcifunc)];
 418                 sdp_info = parent_pf->sdp_info;
 419                 if (!sdp_info) {
 420                         dev_err(rvu->dev, "Invalid sdp_info pointer\n");
 421                         return -EINVAL;
 422                 }
 423                 if (from_vf) {
 424                         req_chan_base = rvu_nix_chan_sdp(rvu, 0) + sdp_info->pf_srn +
 425                                 sdp_info->num_pf_rings;
 426                         vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;
 427                         for (vfid = 0; vfid < vf; vfid++)
 428                                 req_chan_base += sdp_info->vf_rings[vfid];
 429                         req_chan_cnt = sdp_info->vf_rings[vf];
 430                         req_chan_end = req_chan_base + req_chan_cnt - 1;
 431                         if (req_chan_base < rvu_nix_chan_sdp(rvu, 0) ||
 432                             req_chan_end > rvu_nix_chan_sdp(rvu, 255)) {
 433                                 dev_err(rvu->dev,
 434                                         "PF_Func 0x%x: Invalid channel base and count\n",
 435                                         pcifunc);
 436                                 return -EINVAL;
 437                         }
 438                 } else {
 439                         req_chan_base = rvu_nix_chan_sdp(rvu, 0) + sdp_info->pf_srn;
 440                         req_chan_cnt = sdp_info->num_pf_rings;
 441                 }
 442
 443                 pfvf->rx_chan_base = req_chan_base;
 444                 pfvf->rx_chan_cnt = req_chan_cnt;
 445                 pfvf->tx_chan_base = pfvf->rx_chan_base;
 446                 pfvf->tx_chan_cnt = pfvf->rx_chan_cnt;
 447
 448                 rsp->tx_link = hw->cgx_links + hw->lbk_links;
 449                 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf,
 450                                               pfvf->rx_chan_base,
 451                                               pfvf->rx_chan_cnt);
 452                 break;
 453         }
 454
 455         /* Add a UCAST forwarding rule in MCAM with this NIXLF attached
 456          * RVU PF/VF's MAC address.
 457          */
 458         rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf,
 459                                     pfvf->rx_chan_base, pfvf->mac_addr);
 460
 461         /* Add this PF_FUNC to bcast pkt replication list */
 462         err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, true);
 463         if (err) {
 464                 dev_err(rvu->dev,
 465                         "Bcast list, failed to enable PF_FUNC 0x%x\n",
 466                         pcifunc);
 467                 return err;
 468         }
 469         /* Install MCAM rule matching Ethernet broadcast mac address */
 470         rvu_npc_install_bcast_match_entry(rvu, pcifunc,
 471                                           nixlf, pfvf->rx_chan_base);
 472
 473         pfvf->maxlen = NIC_HW_MIN_FRS;
 474         pfvf->minlen = NIC_HW_MIN_FRS;
 475
 476         return 0;
 477 }
 478
 479 static void nix_interface_deinit(struct rvu *rvu, u16 pcifunc, u8 nixlf)
 480 {
 481         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
 482         int err;
 483
 484         pfvf->maxlen = 0;
 485         pfvf->minlen = 0;
 486
 487         /* Remove this PF_FUNC from bcast pkt replication list */
 488         err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, false);
 489         if (err) {
 490                 dev_err(rvu->dev,
 491                         "Bcast list, failed to disable PF_FUNC 0x%x\n",
 492                         pcifunc);
 493         }
 494
 495         /* Free and disable any MCAM entries used by this NIX LF */
 496         rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
 497
 498         /* Disable DMAC filters used */
 499         rvu_cgx_disable_dmac_entries(rvu, pcifunc);
 500 }
 501
 502 #define NIX_BPIDS_PER_LMAC      8
 503 #define NIX_BPIDS_PER_CPT       1
 504 static int nix_setup_bpids(struct rvu *rvu, struct nix_hw *hw, int blkaddr)
 505 {
 506         struct nix_bp *bp = &hw->bp;
 507         int err, max_bpids;
 508         u64 cfg;
 509
 510         cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
 511         max_bpids =  FIELD_GET(NIX_CONST_MAX_BPIDS, cfg);
 512
 513         /* Reserve the BPIds for CGX and SDP */
 514         bp->cgx_bpid_cnt = rvu->hw->cgx_links * NIX_BPIDS_PER_LMAC;
 515         bp->sdp_bpid_cnt = rvu->hw->sdp_links * FIELD_GET(NIX_CONST_SDP_CHANS, cfg);
 516         bp->free_pool_base = bp->cgx_bpid_cnt + bp->sdp_bpid_cnt +
 517                              NIX_BPIDS_PER_CPT;
 518         bp->bpids.max = max_bpids - bp->free_pool_base;
 519
 520         err = rvu_alloc_bitmap(&bp->bpids);
 521         if (err)
 522                 return err;
 523
 524         bp->fn_map = devm_kcalloc(rvu->dev, bp->bpids.max,
 525                                   sizeof(u16), GFP_KERNEL);
 526         if (!bp->fn_map)
 527                 return -ENOMEM;
 528
 529         bp->intf_map = devm_kcalloc(rvu->dev, bp->bpids.max,
 530                                     sizeof(u8), GFP_KERNEL);
 531         if (!bp->intf_map)
 532                 return -ENOMEM;
 533
 534         bp->ref_cnt = devm_kcalloc(rvu->dev, bp->bpids.max,
 535                                    sizeof(u8), GFP_KERNEL);
 536         if (!bp->ref_cnt)
 537                 return -ENOMEM;
 538
 539         return 0;
 540 }
 541
 542 void rvu_nix_flr_free_bpids(struct rvu *rvu, u16 pcifunc)
 543 {
 544         int blkaddr, bpid, err;
 545         struct nix_hw *nix_hw;
 546         struct nix_bp *bp;
 547
 548         if (!is_lbk_vf(rvu, pcifunc))
 549                 return;
 550
 551         err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
 552         if (err)
 553                 return;
 554
 555         bp = &nix_hw->bp;
 556
 557         mutex_lock(&rvu->rsrc_lock);
 558         for (bpid = 0; bpid < bp->bpids.max; bpid++) {
 559                 if (bp->fn_map[bpid] == pcifunc) {
 560                         bp->ref_cnt[bpid]--;
 561                         if (bp->ref_cnt[bpid])
 562                                 continue;
 563                         rvu_free_rsrc(&bp->bpids, bpid);
 564                         bp->fn_map[bpid] = 0;
 565                 }
 566         }
 567         mutex_unlock(&rvu->rsrc_lock);
 568 }
 569
 570 int rvu_mbox_handler_nix_bp_disable(struct rvu *rvu,
 571                                     struct nix_bp_cfg_req *req,
 572                                     struct msg_rsp *rsp)
 573 {
 574         u16 pcifunc = req->hdr.pcifunc;
 575         int blkaddr, pf, type, err;
 576         u16 chan_base, chan, bpid;
 577         struct rvu_pfvf *pfvf;
 578         struct nix_hw *nix_hw;
 579         struct nix_bp *bp;
 580         u64 cfg;
 581
 582         pf = rvu_get_pf(pcifunc);
 583         type = is_lbk_vf(rvu, pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
 584         if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK)
 585                 return 0;
 586
 587         pfvf = rvu_get_pfvf(rvu, pcifunc);
 588         err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
 589         if (err)
 590                 return err;
 591
 592         bp = &nix_hw->bp;
 593         chan_base = pfvf->rx_chan_base + req->chan_base;
 594         for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) {
 595                 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan));
 596                 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan),
 597                             cfg & ~BIT_ULL(16));
 598
 599                 if (type == NIX_INTF_TYPE_LBK) {
 600                         bpid = cfg & GENMASK(8, 0);
 601                         mutex_lock(&rvu->rsrc_lock);
 602                         rvu_free_rsrc(&bp->bpids, bpid - bp->free_pool_base);
 603                         for (bpid = 0; bpid < bp->bpids.max; bpid++) {
 604                                 if (bp->fn_map[bpid] == pcifunc) {
 605                                         bp->fn_map[bpid] = 0;
 606                                         bp->ref_cnt[bpid] = 0;
 607                                 }
 608                         }
 609                         mutex_unlock(&rvu->rsrc_lock);
 610                 }
 611         }
 612         return 0;
 613 }
 614
 615 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
 616                             int type, int chan_id)
 617 {
 618         int bpid, blkaddr, sdp_chan_base, err;
 619         struct rvu_hwinfo *hw = rvu->hw;
 620         struct rvu_pfvf *pfvf;
 621         struct nix_hw *nix_hw;
 622         u8 cgx_id, lmac_id;
 623         struct nix_bp *bp;
 624
 625         pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
 626
 627         err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr);
 628         if (err)
 629                 return err;
 630
 631         bp = &nix_hw->bp;
 632
 633         /* Backpressure IDs range division
 634          * CGX channles are mapped to (0 - 191) BPIDs
 635          * LBK channles are mapped to (192 - 255) BPIDs
 636          * SDP channles are mapped to (256 - 511) BPIDs
 637          *
 638          * Lmac channles and bpids mapped as follows
 639          * cgx(0)_lmac(0)_chan(0 - 15) = bpid(0 - 15)
 640          * cgx(0)_lmac(1)_chan(0 - 15) = bpid(16 - 31) ....
 641          * cgx(1)_lmac(0)_chan(0 - 15) = bpid(64 - 79) ....
 642          */
 643         switch (type) {
 644         case NIX_INTF_TYPE_CGX:
 645                 if ((req->chan_base + req->chan_cnt) > NIX_BPIDS_PER_LMAC)
 646                         return NIX_AF_ERR_INVALID_BPID_REQ;
 647                 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id);
 648                 /* Assign bpid based on cgx, lmac and chan id */
 649                 bpid = (cgx_id * hw->lmac_per_cgx * NIX_BPIDS_PER_LMAC) +
 650                         (lmac_id * NIX_BPIDS_PER_LMAC) + req->chan_base;
 651
 652                 if (req->bpid_per_chan)
 653                         bpid += chan_id;
 654                 if (bpid > bp->cgx_bpid_cnt)
 655                         return NIX_AF_ERR_INVALID_BPID;
 656                 break;
 657
 658         case NIX_INTF_TYPE_LBK:
 659                 /* Alloc bpid from the free pool */
 660                 mutex_lock(&rvu->rsrc_lock);
 661                 bpid = rvu_alloc_rsrc(&bp->bpids);
 662                 if (bpid < 0) {
 663                         mutex_unlock(&rvu->rsrc_lock);
 664                         return NIX_AF_ERR_INVALID_BPID;
 665                 }
 666                 bp->fn_map[bpid] = req->hdr.pcifunc;
 667                 bp->ref_cnt[bpid]++;
 668                 bpid += bp->free_pool_base;
 669                 mutex_unlock(&rvu->rsrc_lock);
 670                 break;
 671         case NIX_INTF_TYPE_SDP:
 672                 if ((req->chan_base + req->chan_cnt) > bp->sdp_bpid_cnt)
 673                         return NIX_AF_ERR_INVALID_BPID_REQ;
 674
 675                 /* Handle usecase of 2 SDP blocks */
 676                 if (!hw->cap.programmable_chans)
 677                         sdp_chan_base = pfvf->rx_chan_base - NIX_CHAN_SDP_CH_START;
 678                 else
 679                         sdp_chan_base = pfvf->rx_chan_base - hw->sdp_chan_base;
 680
 681                 bpid = bp->cgx_bpid_cnt + req->chan_base + sdp_chan_base;
 682                 if (req->bpid_per_chan)
 683                         bpid += chan_id;
 684
 685                 if (bpid > (bp->cgx_bpid_cnt + bp->sdp_bpid_cnt))
 686                         return NIX_AF_ERR_INVALID_BPID;
 687                 break;
 688         default:
 689                 return -EINVAL;
 690         }
 691         return bpid;
 692 }
 693
 694 int rvu_mbox_handler_nix_bp_enable(struct rvu *rvu,
 695                                    struct nix_bp_cfg_req *req,
 696                                    struct nix_bp_cfg_rsp *rsp)
 697 {
 698         int blkaddr, pf, type, chan_id = 0;
 699         u16 pcifunc = req->hdr.pcifunc;
 700         struct rvu_pfvf *pfvf;
 701         u16 chan_base, chan;
 702         s16 bpid, bpid_base;
 703         u64 cfg;
 704
 705         pf = rvu_get_pf(pcifunc);
 706         type = is_lbk_vf(rvu, pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
 707         if (is_sdp_pfvf(pcifunc))
 708                 type = NIX_INTF_TYPE_SDP;
 709
 710         /* Enable backpressure only for CGX mapped PFs and LBK/SDP interface */
 711         if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK &&
 712             type != NIX_INTF_TYPE_SDP)
 713                 return 0;
 714
 715         pfvf = rvu_get_pfvf(rvu, pcifunc);
 716         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
 717
 718         bpid_base = rvu_nix_get_bpid(rvu, req, type, chan_id);
 719         chan_base = pfvf->rx_chan_base + req->chan_base;
 720         bpid = bpid_base;
 721
 722         for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) {
 723                 if (bpid < 0) {
 724                         dev_warn(rvu->dev, "Fail to enable backpressure\n");
 725                         return -EINVAL;
 726                 }
 727
 728                 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan));
 729                 cfg &= ~GENMASK_ULL(8, 0);
 730                 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan),
 731                             cfg | (bpid & GENMASK_ULL(8, 0)) | BIT_ULL(16));
 732                 chan_id++;
 733                 bpid = rvu_nix_get_bpid(rvu, req, type, chan_id);
 734         }
 735
 736         for (chan = 0; chan < req->chan_cnt; chan++) {
 737                 /* Map channel and bpid assign to it */
 738                 rsp->chan_bpid[chan] = ((req->chan_base + chan) & 0x7F) << 10 |
 739                                         (bpid_base & 0x3FF);
 740                 if (req->bpid_per_chan)
 741                         bpid_base++;
 742         }
 743         rsp->chan_cnt = req->chan_cnt;
 744
 745         return 0;
 746 }
 747
 748 static void nix_setup_lso_tso_l3(struct rvu *rvu, int blkaddr,
 749                                  u64 format, bool v4, u64 *fidx)
 750 {
 751         struct nix_lso_format field = {0};
 752
 753         /* IP's Length field */
 754         field.layer = NIX_TXLAYER_OL3;
 755         /* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */
 756         field.offset = v4 ? 2 : 4;
 757         field.sizem1 = 1; /* i.e 2 bytes */
 758         field.alg = NIX_LSOALG_ADD_PAYLEN;
 759         rvu_write64(rvu, blkaddr,
 760                     NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
 761                     *(u64 *)&field);
 762
 763         /* No ID field in IPv6 header */
 764         if (!v4)
 765                 return;
 766
 767         /* IP's ID field */
 768         field.layer = NIX_TXLAYER_OL3;
 769         field.offset = 4;
 770         field.sizem1 = 1; /* i.e 2 bytes */
 771         field.alg = NIX_LSOALG_ADD_SEGNUM;
 772         rvu_write64(rvu, blkaddr,
 773                     NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
 774                     *(u64 *)&field);
 775 }
 776
 777 static void nix_setup_lso_tso_l4(struct rvu *rvu, int blkaddr,
 778                                  u64 format, u64 *fidx)
 779 {
 780         struct nix_lso_format field = {0};
 781
 782         /* TCP's sequence number field */
 783         field.layer = NIX_TXLAYER_OL4;
 784         field.offset = 4;
 785         field.sizem1 = 3; /* i.e 4 bytes */
 786         field.alg = NIX_LSOALG_ADD_OFFSET;
 787         rvu_write64(rvu, blkaddr,
 788                     NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
 789                     *(u64 *)&field);
 790
 791         /* TCP's flags field */
 792         field.layer = NIX_TXLAYER_OL4;
 793         field.offset = 12;
 794         field.sizem1 = 1; /* 2 bytes */
 795         field.alg = NIX_LSOALG_TCP_FLAGS;
 796         rvu_write64(rvu, blkaddr,
 797                     NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++),
 798                     *(u64 *)&field);
 799 }
 800
 801 static void nix_setup_lso(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
 802 {
 803         u64 cfg, idx, fidx = 0;
 804
 805         /* Get max HW supported format indices */
 806         cfg = (rvu_read64(rvu, blkaddr, NIX_AF_CONST1) >> 48) & 0xFF;
 807         nix_hw->lso.total = cfg;
 808
 809         /* Enable LSO */
 810         cfg = rvu_read64(rvu, blkaddr, NIX_AF_LSO_CFG);
 811         /* For TSO, set first and middle segment flags to
 812          * mask out PSH, RST & FIN flags in TCP packet
 813          */
 814         cfg &= ~((0xFFFFULL << 32) | (0xFFFFULL << 16));
 815         cfg |= (0xFFF2ULL << 32) | (0xFFF2ULL << 16);
 816         rvu_write64(rvu, blkaddr, NIX_AF_LSO_CFG, cfg | BIT_ULL(63));
 817
 818         /* Setup default static LSO formats
 819          *
 820          * Configure format fields for TCPv4 segmentation offload
 821          */
 822         idx = NIX_LSO_FORMAT_IDX_TSOV4;
 823         nix_setup_lso_tso_l3(rvu, blkaddr, idx, true, &fidx);
 824         nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx);
 825
 826         /* Set rest of the fields to NOP */
 827         for (; fidx < 8; fidx++) {
 828                 rvu_write64(rvu, blkaddr,
 829                             NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL);
 830         }
 831         nix_hw->lso.in_use++;
 832
 833         /* Configure format fields for TCPv6 segmentation offload */
 834         idx = NIX_LSO_FORMAT_IDX_TSOV6;
 835         fidx = 0;
 836         nix_setup_lso_tso_l3(rvu, blkaddr, idx, false, &fidx);
 837         nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx);
 838
 839         /* Set rest of the fields to NOP */
 840         for (; fidx < 8; fidx++) {
 841                 rvu_write64(rvu, blkaddr,
 842                             NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL);
 843         }
 844         nix_hw->lso.in_use++;
 845 }
 846
 847 static void nix_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf)
 848 {
 849         kfree(pfvf->rq_bmap);
 850         kfree(pfvf->sq_bmap);
 851         kfree(pfvf->cq_bmap);
 852         if (pfvf->rq_ctx)
 853                 qmem_free(rvu->dev, pfvf->rq_ctx);
 854         if (pfvf->sq_ctx)
 855                 qmem_free(rvu->dev, pfvf->sq_ctx);
 856         if (pfvf->cq_ctx)
 857                 qmem_free(rvu->dev, pfvf->cq_ctx);
 858         if (pfvf->rss_ctx)
 859                 qmem_free(rvu->dev, pfvf->rss_ctx);
 860         if (pfvf->nix_qints_ctx)
 861                 qmem_free(rvu->dev, pfvf->nix_qints_ctx);
 862         if (pfvf->cq_ints_ctx)
 863                 qmem_free(rvu->dev, pfvf->cq_ints_ctx);
 864
 865         pfvf->rq_bmap = NULL;
 866         pfvf->cq_bmap = NULL;
 867         pfvf->sq_bmap = NULL;
 868         pfvf->rq_ctx = NULL;
 869         pfvf->sq_ctx = NULL;
 870         pfvf->cq_ctx = NULL;
 871         pfvf->rss_ctx = NULL;
 872         pfvf->nix_qints_ctx = NULL;
 873         pfvf->cq_ints_ctx = NULL;
 874 }
 875
 876 static int nixlf_rss_ctx_init(struct rvu *rvu, int blkaddr,
 877                               struct rvu_pfvf *pfvf, int nixlf,
 878                               int rss_sz, int rss_grps, int hwctx_size,
 879                               u64 way_mask, bool tag_lsb_as_adder)
 880 {
 881         int err, grp, num_indices;
 882         u64 val;
 883
 884         /* RSS is not requested for this NIXLF */
 885         if (!rss_sz)
 886                 return 0;
 887         num_indices = rss_sz * rss_grps;
 888
 889         /* Alloc NIX RSS HW context memory and config the base */
 890         err = qmem_alloc(rvu->dev, &pfvf->rss_ctx, num_indices, hwctx_size);
 891         if (err)
 892                 return err;
 893
 894         rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_BASE(nixlf),
 895                     (u64)pfvf->rss_ctx->iova);
 896
 897         /* Config full RSS table size, enable RSS and caching */
 898         val = BIT_ULL(36) | BIT_ULL(4) | way_mask << 20 |
 899                         ilog2(num_indices / MAX_RSS_INDIR_TBL_SIZE);
 900
 901         if (tag_lsb_as_adder)
 902                 val |= BIT_ULL(5);
 903
 904         rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf), val);
 905         /* Config RSS group offset and sizes */
 906         for (grp = 0; grp < rss_grps; grp++)
 907                 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_GRPX(nixlf, grp),
 908                             ((ilog2(rss_sz) - 1) << 16) | (rss_sz * grp));
 909         return 0;
 910 }
 911
 912 static int nix_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block,
 913                                struct nix_aq_inst_s *inst)
 914 {
 915         struct admin_queue *aq = block->aq;
 916         struct nix_aq_res_s *result;
 917         int timeout = 1000;
 918         u64 reg, head;
 919         int ret;
 920
 921         result = (struct nix_aq_res_s *)aq->res->base;
 922
 923         /* Get current head pointer where to append this instruction */
 924         reg = rvu_read64(rvu, block->addr, NIX_AF_AQ_STATUS);
 925         head = (reg >> 4) & AQ_PTR_MASK;
 926
 927         memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)),
 928                (void *)inst, aq->inst->entry_sz);
 929         memset(result, 0, sizeof(*result));
 930         /* sync into memory */
 931         wmb();
 932
 933         /* Ring the doorbell and wait for result */
 934         rvu_write64(rvu, block->addr, NIX_AF_AQ_DOOR, 1);
 935         while (result->compcode == NIX_AQ_COMP_NOTDONE) {
 936                 cpu_relax();
 937                 udelay(1);
 938                 timeout--;
 939                 if (!timeout)
 940                         return -EBUSY;
 941         }
 942
 943         if (result->compcode != NIX_AQ_COMP_GOOD) {
 944                 /* TODO: Replace this with some error code */
 945                 if (result->compcode == NIX_AQ_COMP_CTX_FAULT ||
 946                     result->compcode == NIX_AQ_COMP_LOCKERR ||
 947                     result->compcode == NIX_AQ_COMP_CTX_POISON) {
 948                         ret = rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX0_RX);
 949                         ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX0_TX);
 950                         ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX1_RX);
 951                         ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX1_TX);
 952                         if (ret)
 953                                 dev_err(rvu->dev,
 954                                         "%s: Not able to unlock cachelines\n", __func__);
 955                 }
 956
 957                 return -EBUSY;
 958         }
 959
 960         return 0;
 961 }
 962
 963 static void nix_get_aq_req_smq(struct rvu *rvu, struct nix_aq_enq_req *req,
 964                                u16 *smq, u16 *smq_mask)
 965 {
 966         struct nix_cn10k_aq_enq_req *aq_req;
 967
 968         if (!is_rvu_otx2(rvu)) {
 969                 aq_req = (struct nix_cn10k_aq_enq_req *)req;
 970                 *smq = aq_req->sq.smq;
 971                 *smq_mask = aq_req->sq_mask.smq;
 972         } else {
 973                 *smq = req->sq.smq;
 974                 *smq_mask = req->sq_mask.smq;
 975         }
 976 }
 977
 978 static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw,
 979                                    struct nix_aq_enq_req *req,
 980                                    struct nix_aq_enq_rsp *rsp)
 981 {
 982         struct rvu_hwinfo *hw = rvu->hw;
 983         u16 pcifunc = req->hdr.pcifunc;
 984         int nixlf, blkaddr, rc = 0;
 985         struct nix_aq_inst_s inst;
 986         struct rvu_block *block;
 987         struct admin_queue *aq;
 988         struct rvu_pfvf *pfvf;
 989         u16 smq, smq_mask;
 990         void *ctx, *mask;
 991         bool ena;
 992         u64 cfg;
 993
 994         blkaddr = nix_hw->blkaddr;
 995         block = &hw->block[blkaddr];
 996         aq = block->aq;
 997         if (!aq) {
 998                 dev_warn(rvu->dev, "%s: NIX AQ not initialized\n", __func__);
 999                 return NIX_AF_ERR_AQ_ENQUEUE;
1000         }
1001
1002         pfvf = rvu_get_pfvf(rvu, pcifunc);
1003         nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
1004
1005         /* Skip NIXLF check for broadcast MCE entry and bandwidth profile
1006          * operations done by AF itself.
1007          */
1008         if (!((!rsp && req->ctype == NIX_AQ_CTYPE_MCE) ||
1009               (req->ctype == NIX_AQ_CTYPE_BANDPROF && !pcifunc))) {
1010                 if (!pfvf->nixlf || nixlf < 0)
1011                         return NIX_AF_ERR_AF_LF_INVALID;
1012         }
1013
1014         switch (req->ctype) {
1015         case NIX_AQ_CTYPE_RQ:
1016                 /* Check if index exceeds max no of queues */
1017                 if (!pfvf->rq_ctx || req->qidx >= pfvf->rq_ctx->qsize)
1018                         rc = NIX_AF_ERR_AQ_ENQUEUE;
1019                 break;
1020         case NIX_AQ_CTYPE_SQ:
1021                 if (!pfvf->sq_ctx || req->qidx >= pfvf->sq_ctx->qsize)
1022                         rc = NIX_AF_ERR_AQ_ENQUEUE;
1023                 break;
1024         case NIX_AQ_CTYPE_CQ:
1025                 if (!pfvf->cq_ctx || req->qidx >= pfvf->cq_ctx->qsize)
1026                         rc = NIX_AF_ERR_AQ_ENQUEUE;
1027                 break;
1028         case NIX_AQ_CTYPE_RSS:
1029                 /* Check if RSS is enabled and qidx is within range */
1030                 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf));
1031                 if (!(cfg & BIT_ULL(4)) || !pfvf->rss_ctx ||
1032                     (req->qidx >= (256UL << (cfg & 0xF))))
1033                         rc = NIX_AF_ERR_AQ_ENQUEUE;
1034                 break;
1035         case NIX_AQ_CTYPE_MCE:
1036                 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG);
1037
1038                 /* Check if index exceeds MCE list length */
1039                 if (!nix_hw->mcast.mce_ctx ||
1040                     (req->qidx >= (256UL << (cfg & 0xF))))
1041                         rc = NIX_AF_ERR_AQ_ENQUEUE;
1042
1043                 /* Adding multicast lists for requests from PF/VFs is not
1044                  * yet supported, so ignore this.
1045                  */
1046                 if (rsp)
1047                         rc = NIX_AF_ERR_AQ_ENQUEUE;
1048                 break;
1049         case NIX_AQ_CTYPE_BANDPROF:
1050                 if (nix_verify_bandprof((struct nix_cn10k_aq_enq_req *)req,
1051                                         nix_hw, pcifunc))
1052                         rc = NIX_AF_ERR_INVALID_BANDPROF;
1053                 break;
1054         default:
1055                 rc = NIX_AF_ERR_AQ_ENQUEUE;
1056         }
1057
1058         if (rc)
1059                 return rc;
1060
1061         nix_get_aq_req_smq(rvu, req, &smq, &smq_mask);
1062         /* Check if SQ pointed SMQ belongs to this PF/VF or not */
1063         if (req->ctype == NIX_AQ_CTYPE_SQ &&
1064             ((req->op == NIX_AQ_INSTOP_INIT && req->sq.ena) ||
1065              (req->op == NIX_AQ_INSTOP_WRITE &&
1066               req->sq_mask.ena && req->sq.ena && smq_mask))) {
1067                 if (!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_SMQ,
1068                                      pcifunc, smq))
1069                         return NIX_AF_ERR_AQ_ENQUEUE;
1070         }
1071
1072         memset(&inst, 0, sizeof(struct nix_aq_inst_s));
1073         inst.lf = nixlf;
1074         inst.cindex = req->qidx;
1075         inst.ctype = req->ctype;
1076         inst.op = req->op;
1077         /* Currently we are not supporting enqueuing multiple instructions,
1078          * so always choose first entry in result memory.
1079          */
1080         inst.res_addr = (u64)aq->res->iova;
1081
1082         /* Hardware uses same aq->res->base for updating result of
1083          * previous instruction hence wait here till it is done.
1084          */
1085         spin_lock(&aq->lock);
1086
1087         /* Clean result + context memory */
1088         memset(aq->res->base, 0, aq->res->entry_sz);
1089         /* Context needs to be written at RES_ADDR + 128 */
1090         ctx = aq->res->base + 128;
1091         /* Mask needs to be written at RES_ADDR + 256 */
1092         mask = aq->res->base + 256;
1093
1094         switch (req->op) {
1095         case NIX_AQ_INSTOP_WRITE:
1096                 if (req->ctype == NIX_AQ_CTYPE_RQ)
1097                         memcpy(mask, &req->rq_mask,
1098                                sizeof(struct nix_rq_ctx_s));
1099                 else if (req->ctype == NIX_AQ_CTYPE_SQ)
1100                         memcpy(mask, &req->sq_mask,
1101                                sizeof(struct nix_sq_ctx_s));
1102                 else if (req->ctype == NIX_AQ_CTYPE_CQ)
1103                         memcpy(mask, &req->cq_mask,
1104                                sizeof(struct nix_cq_ctx_s));
1105                 else if (req->ctype == NIX_AQ_CTYPE_RSS)
1106                         memcpy(mask, &req->rss_mask,
1107                                sizeof(struct nix_rsse_s));
1108                 else if (req->ctype == NIX_AQ_CTYPE_MCE)
1109                         memcpy(mask, &req->mce_mask,
1110                                sizeof(struct nix_rx_mce_s));
1111                 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF)
1112                         memcpy(mask, &req->prof_mask,
1113                                sizeof(struct nix_bandprof_s));
1114                 fallthrough;
1115         case NIX_AQ_INSTOP_INIT:
1116                 if (req->ctype == NIX_AQ_CTYPE_RQ)
1117                         memcpy(ctx, &req->rq, sizeof(struct nix_rq_ctx_s));
1118                 else if (req->ctype == NIX_AQ_CTYPE_SQ)
1119                         memcpy(ctx, &req->sq, sizeof(struct nix_sq_ctx_s));
1120                 else if (req->ctype == NIX_AQ_CTYPE_CQ)
1121                         memcpy(ctx, &req->cq, sizeof(struct nix_cq_ctx_s));
1122                 else if (req->ctype == NIX_AQ_CTYPE_RSS)
1123                         memcpy(ctx, &req->rss, sizeof(struct nix_rsse_s));
1124                 else if (req->ctype == NIX_AQ_CTYPE_MCE)
1125                         memcpy(ctx, &req->mce, sizeof(struct nix_rx_mce_s));
1126                 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF)
1127                         memcpy(ctx, &req->prof, sizeof(struct nix_bandprof_s));
1128                 break;
1129         case NIX_AQ_INSTOP_NOP:
1130         case NIX_AQ_INSTOP_READ:
1131         case NIX_AQ_INSTOP_LOCK:
1132         case NIX_AQ_INSTOP_UNLOCK:
1133                 break;
1134         default:
1135                 rc = NIX_AF_ERR_AQ_ENQUEUE;
1136                 spin_unlock(&aq->lock);
1137                 return rc;
1138         }
1139
1140         /* Submit the instruction to AQ */
1141         rc = nix_aq_enqueue_wait(rvu, block, &inst);
1142         if (rc) {
1143                 spin_unlock(&aq->lock);
1144                 return rc;
1145         }
1146
1147         /* Set RQ/SQ/CQ bitmap if respective queue hw context is enabled */
1148         if (req->op == NIX_AQ_INSTOP_INIT) {
1149                 if (req->ctype == NIX_AQ_CTYPE_RQ && req->rq.ena)
1150                         __set_bit(req->qidx, pfvf->rq_bmap);
1151                 if (req->ctype == NIX_AQ_CTYPE_SQ && req->sq.ena)
1152                         __set_bit(req->qidx, pfvf->sq_bmap);
1153                 if (req->ctype == NIX_AQ_CTYPE_CQ && req->cq.ena)
1154                         __set_bit(req->qidx, pfvf->cq_bmap);
1155         }
1156
1157         if (req->op == NIX_AQ_INSTOP_WRITE) {
1158                 if (req->ctype == NIX_AQ_CTYPE_RQ) {
1159                         ena = (req->rq.ena & req->rq_mask.ena) |
1160                                 (test_bit(req->qidx, pfvf->rq_bmap) &
1161                                 ~req->rq_mask.ena);
1162                         if (ena)
1163                                 __set_bit(req->qidx, pfvf->rq_bmap);
1164                         else
1165                                 __clear_bit(req->qidx, pfvf->rq_bmap);
1166                 }
1167                 if (req->ctype == NIX_AQ_CTYPE_SQ) {
1168                         ena = (req->rq.ena & req->sq_mask.ena) |
1169                                 (test_bit(req->qidx, pfvf->sq_bmap) &
1170                                 ~req->sq_mask.ena);
1171                         if (ena)
1172                                 __set_bit(req->qidx, pfvf->sq_bmap);
1173                         else
1174                                 __clear_bit(req->qidx, pfvf->sq_bmap);
1175                 }
1176                 if (req->ctype == NIX_AQ_CTYPE_CQ) {
1177                         ena = (req->rq.ena & req->cq_mask.ena) |
1178                                 (test_bit(req->qidx, pfvf->cq_bmap) &
1179                                 ~req->cq_mask.ena);
1180                         if (ena)
1181                                 __set_bit(req->qidx, pfvf->cq_bmap);
1182                         else
1183                                 __clear_bit(req->qidx, pfvf->cq_bmap);
1184                 }
1185         }
1186
1187         if (rsp) {
1188                 /* Copy read context into mailbox */
1189                 if (req->op == NIX_AQ_INSTOP_READ) {
1190                         if (req->ctype == NIX_AQ_CTYPE_RQ)
1191                                 memcpy(&rsp->rq, ctx,
1192                                        sizeof(struct nix_rq_ctx_s));
1193                         else if (req->ctype == NIX_AQ_CTYPE_SQ)
1194                                 memcpy(&rsp->sq, ctx,
1195                                        sizeof(struct nix_sq_ctx_s));
1196                         else if (req->ctype == NIX_AQ_CTYPE_CQ)
1197                                 memcpy(&rsp->cq, ctx,
1198                                        sizeof(struct nix_cq_ctx_s));
1199                         else if (req->ctype == NIX_AQ_CTYPE_RSS)
1200                                 memcpy(&rsp->rss, ctx,
1201                                        sizeof(struct nix_rsse_s));
1202                         else if (req->ctype == NIX_AQ_CTYPE_MCE)
1203                                 memcpy(&rsp->mce, ctx,
1204                                        sizeof(struct nix_rx_mce_s));
1205                         else if (req->ctype == NIX_AQ_CTYPE_BANDPROF)
1206                                 memcpy(&rsp->prof, ctx,
1207                                        sizeof(struct nix_bandprof_s));
1208                 }
1209         }
1210
1211         spin_unlock(&aq->lock);
1212         return 0;
1213 }
1214
1215 static int rvu_nix_verify_aq_ctx(struct rvu *rvu, struct nix_hw *nix_hw,
1216                                  struct nix_aq_enq_req *req, u8 ctype)
1217 {
1218         struct nix_cn10k_aq_enq_req aq_req;
1219         struct nix_cn10k_aq_enq_rsp aq_rsp;
1220         int rc, word;
1221
1222         if (req->ctype != NIX_AQ_CTYPE_CQ)
1223                 return 0;
1224
1225         rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp,
1226                                  req->hdr.pcifunc, ctype, req->qidx);
1227         if (rc) {
1228                 dev_err(rvu->dev,
1229                         "%s: Failed to fetch %s%d context of PFFUNC 0x%x\n",
1230                         __func__, nix_get_ctx_name(ctype), req->qidx,
1231                         req->hdr.pcifunc);
1232                 return rc;
1233         }
1234
1235         /* Make copy of original context & mask which are required
1236          * for resubmission
1237          */
1238         memcpy(&aq_req.cq_mask, &req->cq_mask, sizeof(struct nix_cq_ctx_s));
1239         memcpy(&aq_req.cq, &req->cq, sizeof(struct nix_cq_ctx_s));
1240
1241         /* exclude fields which HW can update */
1242         aq_req.cq_mask.cq_err       = 0;
1243         aq_req.cq_mask.wrptr        = 0;
1244         aq_req.cq_mask.tail         = 0;
1245         aq_req.cq_mask.head         = 0;
1246         aq_req.cq_mask.avg_level    = 0;
1247         aq_req.cq_mask.update_time  = 0;
1248         aq_req.cq_mask.substream    = 0;
1249
1250         /* Context mask (cq_mask) holds mask value of fields which
1251          * are changed in AQ WRITE operation.
1252          * for example cq.drop = 0xa;
1253          *             cq_mask.drop = 0xff;
1254          * Below logic performs '&' between cq and cq_mask so that non
1255          * updated fields are masked out for request and response
1256          * comparison
1257          */
1258         for (word = 0; word < sizeof(struct nix_cq_ctx_s) / sizeof(u64);
1259              word++) {
1260                 *(u64 *)((u8 *)&aq_rsp.cq + word * 8) &=
1261                         (*(u64 *)((u8 *)&aq_req.cq_mask + word * 8));
1262                 *(u64 *)((u8 *)&aq_req.cq + word * 8) &=
1263                         (*(u64 *)((u8 *)&aq_req.cq_mask + word * 8));
1264         }
1265
1266         if (memcmp(&aq_req.cq, &aq_rsp.cq, sizeof(struct nix_cq_ctx_s)))
1267                 return NIX_AF_ERR_AQ_CTX_RETRY_WRITE;
1268
1269         return 0;
1270 }
1271
1272 static int rvu_nix_aq_enq_inst(struct rvu *rvu, struct nix_aq_enq_req *req,
1273                                struct nix_aq_enq_rsp *rsp)
1274 {
1275         struct nix_hw *nix_hw;
1276         int err, retries = 5;
1277         int blkaddr;
1278
1279         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc);
1280         if (blkaddr < 0)
1281                 return NIX_AF_ERR_AF_LF_INVALID;
1282
1283         nix_hw =  get_nix_hw(rvu->hw, blkaddr);
1284         if (!nix_hw)
1285                 return NIX_AF_ERR_INVALID_NIXBLK;
1286
1287 retry:
1288         err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, req, rsp);
1289
1290         /* HW errata 'AQ Modification to CQ could be discarded on heavy traffic'
1291          * As a work around perfrom CQ context read after each AQ write. If AQ
1292          * read shows AQ write is not updated perform AQ write again.
1293          */
1294         if (!err && req->op == NIX_AQ_INSTOP_WRITE) {
1295                 err = rvu_nix_verify_aq_ctx(rvu, nix_hw, req, NIX_AQ_CTYPE_CQ);
1296                 if (err == NIX_AF_ERR_AQ_CTX_RETRY_WRITE) {
1297                         if (retries--)
1298                                 goto retry;
1299                         else
1300                                 return NIX_AF_ERR_CQ_CTX_WRITE_ERR;
1301                 }
1302         }
1303
1304         return err;
1305 }
1306
1307 static const char *nix_get_ctx_name(int ctype)
1308 {
1309         switch (ctype) {
1310         case NIX_AQ_CTYPE_CQ:
1311                 return "CQ";
1312         case NIX_AQ_CTYPE_SQ:
1313                 return "SQ";
1314         case NIX_AQ_CTYPE_RQ:
1315                 return "RQ";
1316         case NIX_AQ_CTYPE_RSS:
1317                 return "RSS";
1318         }
1319         return "";
1320 }
1321
1322 static int nix_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req)
1323 {
1324         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
1325         struct nix_aq_enq_req aq_req;
1326         unsigned long *bmap;
1327         int qidx, q_cnt = 0;
1328         int err = 0, rc;
1329
1330         if (!pfvf->cq_ctx || !pfvf->sq_ctx || !pfvf->rq_ctx)
1331                 return NIX_AF_ERR_AQ_ENQUEUE;
1332
1333         memset(&aq_req, 0, sizeof(struct nix_aq_enq_req));
1334         aq_req.hdr.pcifunc = req->hdr.pcifunc;
1335
1336         if (req->ctype == NIX_AQ_CTYPE_CQ) {
1337                 aq_req.cq.ena = 0;
1338                 aq_req.cq_mask.ena = 1;
1339                 aq_req.cq.bp_ena = 0;
1340                 aq_req.cq_mask.bp_ena = 1;
1341                 q_cnt = pfvf->cq_ctx->qsize;
1342                 bmap = pfvf->cq_bmap;
1343         }
1344         if (req->ctype == NIX_AQ_CTYPE_SQ) {
1345                 aq_req.sq.ena = 0;
1346                 aq_req.sq_mask.ena = 1;
1347                 q_cnt = pfvf->sq_ctx->qsize;
1348                 bmap = pfvf->sq_bmap;
1349         }
1350         if (req->ctype == NIX_AQ_CTYPE_RQ) {
1351                 aq_req.rq.ena = 0;
1352                 aq_req.rq_mask.ena = 1;
1353                 q_cnt = pfvf->rq_ctx->qsize;
1354                 bmap = pfvf->rq_bmap;
1355         }
1356
1357         aq_req.ctype = req->ctype;
1358         aq_req.op = NIX_AQ_INSTOP_WRITE;
1359
1360         for (qidx = 0; qidx < q_cnt; qidx++) {
1361                 if (!test_bit(qidx, bmap))
1362                         continue;
1363                 aq_req.qidx = qidx;
1364                 rc = rvu_nix_aq_enq_inst(rvu, &aq_req, NULL);
1365                 if (rc) {
1366                         err = rc;
1367                         dev_err(rvu->dev, "Failed to disable %s:%d context\n",
1368                                 nix_get_ctx_name(req->ctype), qidx);
1369                 }
1370         }
1371
1372         return err;
1373 }
1374
1375 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
1376 static int nix_lf_hwctx_lockdown(struct rvu *rvu, struct nix_aq_enq_req *req)
1377 {
1378         struct nix_aq_enq_req lock_ctx_req;
1379         int err;
1380
1381         if (req->op != NIX_AQ_INSTOP_INIT)
1382                 return 0;
1383
1384         if (req->ctype == NIX_AQ_CTYPE_MCE ||
1385             req->ctype == NIX_AQ_CTYPE_DYNO)
1386                 return 0;
1387
1388         memset(&lock_ctx_req, 0, sizeof(struct nix_aq_enq_req));
1389         lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc;
1390         lock_ctx_req.ctype = req->ctype;
1391         lock_ctx_req.op = NIX_AQ_INSTOP_LOCK;
1392         lock_ctx_req.qidx = req->qidx;
1393         err = rvu_nix_aq_enq_inst(rvu, &lock_ctx_req, NULL);
1394         if (err)
1395                 dev_err(rvu->dev,
1396                         "PFUNC 0x%x: Failed to lock NIX %s:%d context\n",
1397                         req->hdr.pcifunc,
1398                         nix_get_ctx_name(req->ctype), req->qidx);
1399         return err;
1400 }
1401
1402 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu,
1403                                 struct nix_aq_enq_req *req,
1404                                 struct nix_aq_enq_rsp *rsp)
1405 {
1406         int err;
1407
1408         err = rvu_nix_aq_enq_inst(rvu, req, rsp);
1409         if (!err)
1410                 err = nix_lf_hwctx_lockdown(rvu, req);
1411         return err;
1412 }
1413 #else
1414
1415 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu,
1416                                 struct nix_aq_enq_req *req,
1417                                 struct nix_aq_enq_rsp *rsp)
1418 {
1419         return rvu_nix_aq_enq_inst(rvu, req, rsp);
1420 }
1421 #endif
1422 /* CN10K mbox handler */
1423 int rvu_mbox_handler_nix_cn10k_aq_enq(struct rvu *rvu,
1424                                       struct nix_cn10k_aq_enq_req *req,
1425                                       struct nix_cn10k_aq_enq_rsp *rsp)
1426 {
1427         return rvu_nix_aq_enq_inst(rvu, (struct nix_aq_enq_req *)req,
1428                                   (struct nix_aq_enq_rsp *)rsp);
1429 }
1430
1431 int rvu_mbox_handler_nix_hwctx_disable(struct rvu *rvu,
1432                                        struct hwctx_disable_req *req,
1433                                        struct msg_rsp *rsp)
1434 {
1435         return nix_lf_hwctx_disable(rvu, req);
1436 }
1437
1438 int rvu_mbox_handler_nix_lf_alloc(struct rvu *rvu,
1439                                   struct nix_lf_alloc_req *req,
1440                                   struct nix_lf_alloc_rsp *rsp)
1441 {
1442         int nixlf, qints, hwctx_size, intf, err, rc = 0;
1443         struct rvu_hwinfo *hw = rvu->hw;
1444         u16 pcifunc = req->hdr.pcifunc;
1445         struct rvu_block *block;
1446         struct rvu_pfvf *pfvf;
1447         u64 cfg, ctx_cfg;
1448         int blkaddr;
1449
1450         if (!req->rq_cnt || !req->sq_cnt || !req->cq_cnt)
1451                 return NIX_AF_ERR_PARAM;
1452
1453         if (req->way_mask)
1454                 req->way_mask &= 0xFFFF;
1455
1456         pfvf = rvu_get_pfvf(rvu, pcifunc);
1457         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1458         if (!pfvf->nixlf || blkaddr < 0)
1459                 return NIX_AF_ERR_AF_LF_INVALID;
1460
1461         block = &hw->block[blkaddr];
1462         nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
1463         if (nixlf < 0)
1464                 return NIX_AF_ERR_AF_LF_INVALID;
1465
1466         /* Check if requested 'NIXLF <=> NPALF' mapping is valid */
1467         if (req->npa_func) {
1468                 /* If default, use 'this' NIXLF's PFFUNC */
1469                 if (req->npa_func == RVU_DEFAULT_PF_FUNC)
1470                         req->npa_func = pcifunc;
1471                 if (!is_pffunc_map_valid(rvu, req->npa_func, BLKTYPE_NPA))
1472                         return NIX_AF_INVAL_NPA_PF_FUNC;
1473         }
1474
1475         /* Check if requested 'NIXLF <=> SSOLF' mapping is valid */
1476         if (req->sso_func) {
1477                 /* If default, use 'this' NIXLF's PFFUNC */
1478                 if (req->sso_func == RVU_DEFAULT_PF_FUNC)
1479                         req->sso_func = pcifunc;
1480                 if (!is_pffunc_map_valid(rvu, req->sso_func, BLKTYPE_SSO))
1481                         return NIX_AF_INVAL_SSO_PF_FUNC;
1482         }
1483
1484         /* If RSS is being enabled, check if requested config is valid.
1485          * RSS table size should be power of two, otherwise
1486          * RSS_GRP::OFFSET + adder might go beyond that group or
1487          * won't be able to use entire table.
1488          */
1489         if (req->rss_sz && (req->rss_sz > MAX_RSS_INDIR_TBL_SIZE ||
1490                             !is_power_of_2(req->rss_sz)))
1491                 return NIX_AF_ERR_RSS_SIZE_INVALID;
1492
1493         if (req->rss_sz &&
1494             (!req->rss_grps || req->rss_grps > MAX_RSS_GROUPS))
1495                 return NIX_AF_ERR_RSS_GRPS_INVALID;
1496
1497         /* Reset this NIX LF */
1498         err = rvu_lf_reset(rvu, block, nixlf);
1499         if (err) {
1500                 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n",
1501                         block->addr - BLKADDR_NIX0, nixlf);
1502                 return NIX_AF_ERR_LF_RESET;
1503         }
1504
1505         ctx_cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST3);
1506
1507         /* Alloc NIX RQ HW context memory and config the base */
1508         hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF);
1509         err = qmem_alloc(rvu->dev, &pfvf->rq_ctx, req->rq_cnt, hwctx_size);
1510         if (err)
1511                 goto free_mem;
1512
1513         pfvf->rq_bmap = kcalloc(req->rq_cnt, sizeof(long), GFP_KERNEL);
1514         if (!pfvf->rq_bmap)
1515                 goto free_mem;
1516
1517         rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_BASE(nixlf),
1518                     (u64)pfvf->rq_ctx->iova);
1519
1520         /* Set caching and queue count in HW */
1521         cfg = BIT_ULL(36) | (req->rq_cnt - 1) | req->way_mask << 20;
1522         rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_CFG(nixlf), cfg);
1523
1524         /* Alloc NIX SQ HW context memory and config the base */
1525         hwctx_size = 1UL << (ctx_cfg & 0xF);
1526         err = qmem_alloc(rvu->dev, &pfvf->sq_ctx, req->sq_cnt, hwctx_size);
1527         if (err)
1528                 goto free_mem;
1529
1530         pfvf->sq_bmap = kcalloc(req->sq_cnt, sizeof(long), GFP_KERNEL);
1531         if (!pfvf->sq_bmap)
1532                 goto free_mem;
1533
1534         rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_BASE(nixlf),
1535                     (u64)pfvf->sq_ctx->iova);
1536
1537         cfg = BIT_ULL(36) | (req->sq_cnt - 1) | req->way_mask << 20;
1538         rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_CFG(nixlf), cfg);
1539
1540         /* Alloc NIX CQ HW context memory and config the base */
1541         hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF);
1542         err = qmem_alloc(rvu->dev, &pfvf->cq_ctx, req->cq_cnt, hwctx_size);
1543         if (err)
1544                 goto free_mem;
1545
1546         pfvf->cq_bmap = kcalloc(req->cq_cnt, sizeof(long), GFP_KERNEL);
1547         if (!pfvf->cq_bmap)
1548                 goto free_mem;
1549
1550         rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_BASE(nixlf),
1551                     (u64)pfvf->cq_ctx->iova);
1552
1553         cfg = BIT_ULL(36) | (req->cq_cnt - 1) | req->way_mask << 20;
1554         rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_CFG(nixlf), cfg);
1555
1556         /* Initialize receive side scaling (RSS) */
1557         hwctx_size = 1UL << ((ctx_cfg >> 12) & 0xF);
1558         err = nixlf_rss_ctx_init(rvu, blkaddr, pfvf, nixlf, req->rss_sz,
1559                                  req->rss_grps, hwctx_size, req->way_mask,
1560                                  !!(req->flags & NIX_LF_RSS_TAG_LSB_AS_ADDER));
1561         if (err)
1562                 goto free_mem;
1563
1564         /* Alloc memory for CQINT's HW contexts */
1565         cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
1566         qints = (cfg >> 24) & 0xFFF;
1567         hwctx_size = 1UL << ((ctx_cfg >> 24) & 0xF);
1568         err = qmem_alloc(rvu->dev, &pfvf->cq_ints_ctx, qints, hwctx_size);
1569         if (err)
1570                 goto free_mem;
1571
1572         rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_BASE(nixlf),
1573                     (u64)pfvf->cq_ints_ctx->iova);
1574
1575         rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_CFG(nixlf),
1576                     BIT_ULL(36) | req->way_mask << 20);
1577
1578         /* Alloc memory for QINT's HW contexts */
1579         cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
1580         qints = (cfg >> 12) & 0xFFF;
1581         hwctx_size = 1UL << ((ctx_cfg >> 20) & 0xF);
1582         err = qmem_alloc(rvu->dev, &pfvf->nix_qints_ctx, qints, hwctx_size);
1583         if (err)
1584                 goto free_mem;
1585
1586         rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_BASE(nixlf),
1587                     (u64)pfvf->nix_qints_ctx->iova);
1588         rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_CFG(nixlf),
1589                     BIT_ULL(36) | req->way_mask << 20);
1590
1591         /* Setup VLANX TPID's.
1592          * Use VLAN1 for 802.1Q
1593          * and VLAN0 for 802.1AD.
1594          */
1595         cfg = (0x8100ULL << 16) | 0x88A8ULL;
1596         rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg);
1597
1598         /* Enable LMTST for this NIX LF */
1599         rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG2(nixlf), BIT_ULL(0));
1600
1601         /* Set CQE/WQE size, NPA_PF_FUNC for SQBs and also SSO_PF_FUNC */
1602         if (req->npa_func)
1603                 cfg = req->npa_func;
1604         if (req->sso_func)
1605                 cfg |= (u64)req->sso_func << 16;
1606
1607         cfg |= (u64)req->xqe_sz << 33;
1608         rvu_write64(rvu, blkaddr, NIX_AF_LFX_CFG(nixlf), cfg);
1609
1610         /* Config Rx pkt length, csum checks and apad  enable / disable */
1611         rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), req->rx_cfg);
1612
1613         /* Configure pkind for TX parse config */
1614         cfg = NPC_TX_DEF_PKIND;
1615         rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), cfg);
1616
1617         intf = is_lbk_vf(rvu, pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX;
1618         if (is_sdp_pfvf(pcifunc))
1619                 intf = NIX_INTF_TYPE_SDP;
1620
1621         err = nix_interface_init(rvu, pcifunc, intf, nixlf, rsp,
1622                                  !!(req->flags & NIX_LF_LBK_BLK_SEL));
1623         if (err)
1624                 goto free_mem;
1625
1626         /* Disable NPC entries as NIXLF's contexts are not initialized yet */
1627         rvu_npc_disable_default_entries(rvu, pcifunc, nixlf);
1628
1629         /* Configure RX VTAG Type 7 (strip) for vf vlan */
1630         rvu_write64(rvu, blkaddr,
1631                     NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, NIX_AF_LFX_RX_VTAG_TYPE7),
1632                     VTAGSIZE_T4 | VTAG_STRIP);
1633
1634         goto exit;
1635
1636 free_mem:
1637         nix_ctx_free(rvu, pfvf);
1638         rc = -ENOMEM;
1639
1640 exit:
1641         /* Set macaddr of this PF/VF */
1642         ether_addr_copy(rsp->mac_addr, pfvf->mac_addr);
1643
1644         /* set SQB size info */
1645         cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQ_CONST);
1646         rsp->sqb_size = (cfg >> 34) & 0xFFFF;
1647         rsp->rx_chan_base = pfvf->rx_chan_base;
1648         rsp->tx_chan_base = pfvf->tx_chan_base;
1649         rsp->rx_chan_cnt = pfvf->rx_chan_cnt;
1650         rsp->tx_chan_cnt = pfvf->tx_chan_cnt;
1651         rsp->lso_tsov4_idx = NIX_LSO_FORMAT_IDX_TSOV4;
1652         rsp->lso_tsov6_idx = NIX_LSO_FORMAT_IDX_TSOV6;
1653         /* Get HW supported stat count */
1654         cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
1655         rsp->lf_rx_stats = ((cfg >> 32) & 0xFF);
1656         rsp->lf_tx_stats = ((cfg >> 24) & 0xFF);
1657         /* Get count of CQ IRQs and error IRQs supported per LF */
1658         cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);
1659         rsp->qints = ((cfg >> 12) & 0xFFF);
1660         rsp->cints = ((cfg >> 24) & 0xFFF);
1661         rsp->cgx_links = hw->cgx_links;
1662         rsp->lbk_links = hw->lbk_links;
1663         rsp->sdp_links = hw->sdp_links;
1664
1665         return rc;
1666 }
1667
1668 int rvu_mbox_handler_nix_lf_free(struct rvu *rvu, struct nix_lf_free_req *req,
1669                                  struct msg_rsp *rsp)
1670 {
1671         struct rvu_hwinfo *hw = rvu->hw;
1672         u16 pcifunc = req->hdr.pcifunc;
1673         struct rvu_block *block;
1674         int blkaddr, nixlf, err;
1675         struct rvu_pfvf *pfvf;
1676
1677         pfvf = rvu_get_pfvf(rvu, pcifunc);
1678         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1679         if (!pfvf->nixlf || blkaddr < 0)
1680                 return NIX_AF_ERR_AF_LF_INVALID;
1681
1682         block = &hw->block[blkaddr];
1683         nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
1684         if (nixlf < 0)
1685                 return NIX_AF_ERR_AF_LF_INVALID;
1686
1687         if (req->flags & NIX_LF_DISABLE_FLOWS)
1688                 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
1689         else
1690                 rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf);
1691
1692         /* Free any tx vtag def entries used by this NIX LF */
1693         if (!(req->flags & NIX_LF_DONT_FREE_TX_VTAG))
1694                 nix_free_tx_vtag_entries(rvu, pcifunc);
1695
1696         nix_interface_deinit(rvu, pcifunc, nixlf);
1697
1698         /* Reset this NIX LF */
1699         err = rvu_lf_reset(rvu, block, nixlf);
1700         if (err) {
1701                 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n",
1702                         block->addr - BLKADDR_NIX0, nixlf);
1703                 return NIX_AF_ERR_LF_RESET;
1704         }
1705
1706         nix_ctx_free(rvu, pfvf);
1707
1708         return 0;
1709 }
1710
1711 int rvu_mbox_handler_nix_mark_format_cfg(struct rvu *rvu,
1712                                          struct nix_mark_format_cfg  *req,
1713                                          struct nix_mark_format_cfg_rsp *rsp)
1714 {
1715         u16 pcifunc = req->hdr.pcifunc;
1716         struct nix_hw *nix_hw;
1717         struct rvu_pfvf *pfvf;
1718         int blkaddr, rc;
1719         u32 cfg;
1720
1721         pfvf = rvu_get_pfvf(rvu, pcifunc);
1722         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1723         if (!pfvf->nixlf || blkaddr < 0)
1724                 return NIX_AF_ERR_AF_LF_INVALID;
1725
1726         nix_hw = get_nix_hw(rvu->hw, blkaddr);
1727         if (!nix_hw)
1728                 return NIX_AF_ERR_INVALID_NIXBLK;
1729
1730         cfg = (((u32)req->offset & 0x7) << 16) |
1731               (((u32)req->y_mask & 0xF) << 12) |
1732               (((u32)req->y_val & 0xF) << 8) |
1733               (((u32)req->r_mask & 0xF) << 4) | ((u32)req->r_val & 0xF);
1734
1735         rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfg);
1736         if (rc < 0) {
1737                 dev_err(rvu->dev, "No mark_format_ctl for (pf:%d, vf:%d)",
1738                         rvu_get_pf(pcifunc), pcifunc & RVU_PFVF_FUNC_MASK);
1739                 return NIX_AF_ERR_MARK_CFG_FAIL;
1740         }
1741
1742         rsp->mark_format_idx = rc;
1743         return 0;
1744 }
1745
1746 /* Handle shaper update specially for few revisions */
1747 static bool
1748 handle_txschq_shaper_update(struct rvu *rvu, int blkaddr, int nixlf,
1749                             int lvl, u64 reg, u64 regval)
1750 {
1751         u64 regbase, oldval, sw_xoff = 0;
1752         u64 dbgval, md_debug0 = 0;
1753         unsigned long poll_tmo;
1754         bool rate_reg = 0;
1755         u32 schq;
1756
1757         regbase = reg & 0xFFFF;
1758         schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
1759
1760         /* Check for rate register */
1761         switch (lvl) {
1762         case NIX_TXSCH_LVL_TL1:
1763                 md_debug0 = NIX_AF_TL1X_MD_DEBUG0(schq);
1764                 sw_xoff = NIX_AF_TL1X_SW_XOFF(schq);
1765
1766                 rate_reg = !!(regbase == NIX_AF_TL1X_CIR(0));
1767                 break;
1768         case NIX_TXSCH_LVL_TL2:
1769                 md_debug0 = NIX_AF_TL2X_MD_DEBUG0(schq);
1770                 sw_xoff = NIX_AF_TL2X_SW_XOFF(schq);
1771
1772                 rate_reg = (regbase == NIX_AF_TL2X_CIR(0) ||
1773                             regbase == NIX_AF_TL2X_PIR(0));
1774                 break;
1775         case NIX_TXSCH_LVL_TL3:
1776                 md_debug0 = NIX_AF_TL3X_MD_DEBUG0(schq);
1777                 sw_xoff = NIX_AF_TL3X_SW_XOFF(schq);
1778
1779                 rate_reg = (regbase == NIX_AF_TL3X_CIR(0) ||
1780                             regbase == NIX_AF_TL3X_PIR(0));
1781                 break;
1782         case NIX_TXSCH_LVL_TL4:
1783                 md_debug0 = NIX_AF_TL4X_MD_DEBUG0(schq);
1784                 sw_xoff = NIX_AF_TL4X_SW_XOFF(schq);
1785
1786                 rate_reg = (regbase == NIX_AF_TL4X_CIR(0) ||
1787                             regbase == NIX_AF_TL4X_PIR(0));
1788                 break;
1789         case NIX_TXSCH_LVL_MDQ:
1790                 sw_xoff = NIX_AF_MDQX_SW_XOFF(schq);
1791                 rate_reg = (regbase == NIX_AF_MDQX_CIR(0) ||
1792                             regbase == NIX_AF_MDQX_PIR(0));
1793                 break;
1794         }
1795
1796         if (!rate_reg)
1797                 return false;
1798
1799         /* Nothing special to do when state is not toggled */
1800         oldval = rvu_read64(rvu, blkaddr, reg);
1801         if ((oldval & 0x1) == (regval & 0x1)) {
1802                 rvu_write64(rvu, blkaddr, reg, regval);
1803                 return true;
1804         }
1805
1806         /* PIR/CIR disable */
1807         if (!(regval & 0x1)) {
1808                 rvu_write64(rvu, blkaddr, sw_xoff, 1);
1809                 rvu_write64(rvu, blkaddr, reg, 0);
1810                 udelay(4);
1811                 rvu_write64(rvu, blkaddr, sw_xoff, 0);
1812                 return true;
1813         }
1814
1815         /* PIR/CIR enable */
1816         rvu_write64(rvu, blkaddr, sw_xoff, 1);
1817         if (md_debug0) {
1818                 poll_tmo = jiffies + usecs_to_jiffies(10000);
1819                 /* Wait until VLD(bit32) == 1 or C_CON(bit48) == 0 */
1820                 do {
1821                         if (time_after(jiffies, poll_tmo)) {
1822                                 dev_err(rvu->dev,
1823                                         "NIXLF%d: TLX%u(lvl %u) CIR/PIR enable failed\n",
1824                                         nixlf, schq, lvl);
1825                                 goto exit;
1826                         }
1827                         usleep_range(1, 5);
1828                         dbgval = rvu_read64(rvu, blkaddr, md_debug0);
1829                 } while (!(dbgval & BIT_ULL(32)) && (dbgval & BIT_ULL(48)));
1830         }
1831         rvu_write64(rvu, blkaddr, reg, regval);
1832 exit:
1833         rvu_write64(rvu, blkaddr, sw_xoff, 0);
1834         return true;
1835 }
1836
1837 static void nix_reset_tx_schedule(struct rvu *rvu, int blkaddr,
1838                                   int lvl, int schq)
1839 {
1840         u64 tlx_parent = 0, tlx_schedule = 0;
1841
1842         switch (lvl) {
1843         case NIX_TXSCH_LVL_TL2:
1844                 tlx_parent   = NIX_AF_TL2X_PARENT(schq);
1845                 tlx_schedule = NIX_AF_TL2X_SCHEDULE(schq);
1846                 break;
1847         case NIX_TXSCH_LVL_TL3:
1848                 tlx_parent   = NIX_AF_TL3X_PARENT(schq);
1849                 tlx_schedule = NIX_AF_TL3X_SCHEDULE(schq);
1850                 break;
1851         case NIX_TXSCH_LVL_TL4:
1852                 tlx_parent   = NIX_AF_TL4X_PARENT(schq);
1853                 tlx_schedule = NIX_AF_TL4X_SCHEDULE(schq);
1854                 break;
1855         case NIX_TXSCH_LVL_MDQ:
1856                 /* no need to reset SMQ_CFG as HW clears this CSR
1857                  * on SMQ flush
1858                  */
1859                 tlx_parent   = NIX_AF_MDQX_PARENT(schq);
1860                 tlx_schedule = NIX_AF_MDQX_SCHEDULE(schq);
1861                 break;
1862         default:
1863                 return;
1864         }
1865
1866         if (tlx_parent)
1867                 rvu_write64(rvu, blkaddr, tlx_parent, 0x0);
1868
1869         if (tlx_schedule)
1870                 rvu_write64(rvu, blkaddr, tlx_schedule, 0x0);
1871 }
1872
1873 /* Disable shaping of pkts by a scheduler queue
1874  * at a given scheduler level.
1875  */
1876 static void nix_reset_tx_shaping(struct rvu *rvu, int blkaddr,
1877                                  int nixlf, int lvl, int schq)
1878 {
1879         struct rvu_hwinfo *hw = rvu->hw;
1880         u64  cir_reg = 0, pir_reg = 0;
1881         u64  cfg;
1882
1883         switch (lvl) {
1884         case NIX_TXSCH_LVL_TL1:
1885                 cir_reg = NIX_AF_TL1X_CIR(schq);
1886                 pir_reg = 0; /* PIR not available at TL1 */
1887                 break;
1888         case NIX_TXSCH_LVL_TL2:
1889                 cir_reg = NIX_AF_TL2X_CIR(schq);
1890                 pir_reg = NIX_AF_TL2X_PIR(schq);
1891                 break;
1892         case NIX_TXSCH_LVL_TL3:
1893                 cir_reg = NIX_AF_TL3X_CIR(schq);
1894                 pir_reg = NIX_AF_TL3X_PIR(schq);
1895                 break;
1896         case NIX_TXSCH_LVL_TL4:
1897                 cir_reg = NIX_AF_TL4X_CIR(schq);
1898                 pir_reg = NIX_AF_TL4X_PIR(schq);
1899                 break;
1900         case NIX_TXSCH_LVL_MDQ:
1901                 cir_reg = NIX_AF_MDQX_CIR(schq);
1902                 pir_reg = NIX_AF_MDQX_PIR(schq);
1903                 break;
1904         }
1905
1906         /* Shaper state toggle needs wait/poll */
1907         if (hw->cap.nix_shaper_toggle_wait) {
1908                 if (cir_reg)
1909                         handle_txschq_shaper_update(rvu, blkaddr, nixlf,
1910                                                     lvl, cir_reg, 0);
1911                 if (pir_reg)
1912                         handle_txschq_shaper_update(rvu, blkaddr, nixlf,
1913                                                     lvl, pir_reg, 0);
1914                 return;
1915         }
1916
1917         if (!cir_reg)
1918                 return;
1919         cfg = rvu_read64(rvu, blkaddr, cir_reg);
1920         rvu_write64(rvu, blkaddr, cir_reg, cfg & ~BIT_ULL(0));
1921
1922         if (!pir_reg)
1923                 return;
1924         cfg = rvu_read64(rvu, blkaddr, pir_reg);
1925         rvu_write64(rvu, blkaddr, pir_reg, cfg & ~BIT_ULL(0));
1926 }
1927
1928 static void nix_reset_tx_linkcfg(struct rvu *rvu, int blkaddr,
1929                                  int lvl, int schq)
1930 {
1931         struct rvu_hwinfo *hw = rvu->hw;
1932         int link_level;
1933         int link;
1934
1935         if (lvl >= hw->cap.nix_tx_aggr_lvl)
1936                 return;
1937
1938         /* Reset TL4's SDP link config */
1939         if (lvl == NIX_TXSCH_LVL_TL4)
1940                 rvu_write64(rvu, blkaddr, NIX_AF_TL4X_SDP_LINK_CFG(schq), 0x00);
1941
1942         link_level = rvu_read64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ?
1943                         NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2;
1944         if (lvl != link_level)
1945                 return;
1946
1947         /* Reset TL2's CGX or LBK link config */
1948         for (link = 0; link < (hw->cgx_links + hw->lbk_links); link++)
1949                 rvu_write64(rvu, blkaddr,
1950                             NIX_AF_TL3_TL2X_LINKX_CFG(schq, link), 0x00);
1951 }
1952
1953 static void nix_clear_tx_xoff(struct rvu *rvu, int blkaddr,
1954                               int lvl, int schq)
1955 {
1956         struct rvu_hwinfo *hw = rvu->hw;
1957         u64 reg;
1958
1959         /* Skip this if shaping is not supported */
1960         if (!hw->cap.nix_shaping)
1961                 return;
1962
1963         /* Clear level specific SW_XOFF */
1964         switch (lvl) {
1965         case NIX_TXSCH_LVL_TL1:
1966                 reg = NIX_AF_TL1X_SW_XOFF(schq);
1967                 break;
1968         case NIX_TXSCH_LVL_TL2:
1969                 reg = NIX_AF_TL2X_SW_XOFF(schq);
1970                 break;
1971         case NIX_TXSCH_LVL_TL3:
1972                 reg = NIX_AF_TL3X_SW_XOFF(schq);
1973                 break;
1974         case NIX_TXSCH_LVL_TL4:
1975                 reg = NIX_AF_TL4X_SW_XOFF(schq);
1976                 break;
1977         case NIX_TXSCH_LVL_MDQ:
1978                 reg = NIX_AF_MDQX_SW_XOFF(schq);
1979                 break;
1980         default:
1981                 return;
1982         }
1983
1984         rvu_write64(rvu, blkaddr, reg, 0x0);
1985 }
1986
1987 static int nix_get_tx_link(struct rvu *rvu, u16 pcifunc)
1988 {
1989         struct rvu_hwinfo *hw = rvu->hw;
1990         int pf = rvu_get_pf(pcifunc);
1991         u8 cgx_id = 0, lmac_id = 0;
1992
1993         if (is_lbk_vf(rvu, pcifunc)) {/* LBK links */
1994                 return hw->cgx_links;
1995         } else if (is_pf_cgxmapped(rvu, pf)) {
1996                 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
1997                 return (cgx_id * hw->lmac_per_cgx) + lmac_id;
1998         }
1999
2000         /* SDP link */
2001         return hw->cgx_links + hw->lbk_links;
2002 }
2003
2004 static void nix_get_txschq_range(struct rvu *rvu, u16 pcifunc,
2005                                  int link, int *start, int *end)
2006 {
2007         struct rvu_hwinfo *hw = rvu->hw;
2008         int pf = rvu_get_pf(pcifunc);
2009
2010         if (is_lbk_vf(rvu, pcifunc)) { /* LBK links */
2011                 *start = hw->cap.nix_txsch_per_cgx_lmac * link;
2012                 *end = *start + hw->cap.nix_txsch_per_lbk_lmac;
2013         } else if (is_pf_cgxmapped(rvu, pf)) { /* CGX links */
2014                 *start = hw->cap.nix_txsch_per_cgx_lmac * link;
2015                 *end = *start + hw->cap.nix_txsch_per_cgx_lmac;
2016         } else { /* SDP link */
2017                 *start = (hw->cap.nix_txsch_per_cgx_lmac * hw->cgx_links) +
2018                         (hw->cap.nix_txsch_per_lbk_lmac * hw->lbk_links);
2019                 *end = *start + hw->cap.nix_txsch_per_sdp_lmac;
2020         }
2021 }
2022
2023 static int nix_check_txschq_alloc_req(struct rvu *rvu, int lvl, u16 pcifunc,
2024                                       struct nix_hw *nix_hw,
2025                                       struct nix_txsch_alloc_req *req)
2026 {
2027         struct rvu_hwinfo *hw = rvu->hw;
2028         int schq, req_schq, free_cnt;
2029         struct nix_txsch *txsch;
2030         int link, start, end;
2031
2032         txsch = &nix_hw->txsch[lvl];
2033         req_schq = req->schq_contig[lvl] + req->schq[lvl];
2034
2035         if (!req_schq)
2036                 return 0;
2037
2038         link = nix_get_tx_link(rvu, pcifunc);
2039
2040         /* For traffic aggregating scheduler level, one queue is enough */
2041         if (lvl >= hw->cap.nix_tx_aggr_lvl) {
2042                 if (req_schq != 1)
2043                         return NIX_AF_ERR_TLX_ALLOC_FAIL;
2044                 return 0;
2045         }
2046
2047         /* Get free SCHQ count and check if request can be accomodated */
2048         if (hw->cap.nix_fixed_txschq_mapping) {
2049                 nix_get_txschq_range(rvu, pcifunc, link, &start, &end);
2050                 schq = start + (pcifunc & RVU_PFVF_FUNC_MASK);
2051                 if (end <= txsch->schq.max && schq < end &&
2052                     !test_bit(schq, txsch->schq.bmap))
2053                         free_cnt = 1;
2054                 else
2055                         free_cnt = 0;
2056         } else {
2057                 free_cnt = rvu_rsrc_free_count(&txsch->schq);
2058         }
2059
2060         if (free_cnt < req_schq || req->schq[lvl] > MAX_TXSCHQ_PER_FUNC ||
2061             req->schq_contig[lvl] > MAX_TXSCHQ_PER_FUNC)
2062                 return NIX_AF_ERR_TLX_ALLOC_FAIL;
2063
2064         /* If contiguous queues are needed, check for availability */
2065         if (!hw->cap.nix_fixed_txschq_mapping && req->schq_contig[lvl] &&
2066             !rvu_rsrc_check_contig(&txsch->schq, req->schq_contig[lvl]))
2067                 return NIX_AF_ERR_TLX_ALLOC_FAIL;
2068
2069         return 0;
2070 }
2071
2072 static void nix_txsch_alloc(struct rvu *rvu, struct nix_txsch *txsch,
2073                             struct nix_txsch_alloc_rsp *rsp,
2074                             int lvl, int start, int end)
2075 {
2076         struct rvu_hwinfo *hw = rvu->hw;
2077         u16 pcifunc = rsp->hdr.pcifunc;
2078         int idx, schq;
2079
2080         /* For traffic aggregating levels, queue alloc is based
2081          * on transmit link to which PF_FUNC is mapped to.
2082          */
2083         if (lvl >= hw->cap.nix_tx_aggr_lvl) {
2084                 /* A single TL queue is allocated */
2085                 if (rsp->schq_contig[lvl]) {
2086                         rsp->schq_contig[lvl] = 1;
2087                         rsp->schq_contig_list[lvl][0] = start;
2088                 }
2089
2090                 /* Both contig and non-contig reqs doesn't make sense here */
2091                 if (rsp->schq_contig[lvl])
2092                         rsp->schq[lvl] = 0;
2093
2094                 if (rsp->schq[lvl]) {
2095                         rsp->schq[lvl] = 1;
2096                         rsp->schq_list[lvl][0] = start;
2097                 }
2098                 return;
2099         }
2100
2101         /* Adjust the queue request count if HW supports
2102          * only one queue per level configuration.
2103          */
2104         if (hw->cap.nix_fixed_txschq_mapping) {
2105                 idx = pcifunc & RVU_PFVF_FUNC_MASK;
2106                 schq = start + idx;
2107                 if (idx >= (end - start) || test_bit(schq, txsch->schq.bmap)) {
2108                         rsp->schq_contig[lvl] = 0;
2109                         rsp->schq[lvl] = 0;
2110                         return;
2111                 }
2112
2113                 if (rsp->schq_contig[lvl]) {
2114                         rsp->schq_contig[lvl] = 1;
2115                         set_bit(schq, txsch->schq.bmap);
2116                         rsp->schq_contig_list[lvl][0] = schq;
2117                         rsp->schq[lvl] = 0;
2118                 } else if (rsp->schq[lvl]) {
2119                         rsp->schq[lvl] = 1;
2120                         set_bit(schq, txsch->schq.bmap);
2121                         rsp->schq_list[lvl][0] = schq;
2122                 }
2123                 return;
2124         }
2125
2126         /* Allocate contiguous queue indices requesty first */
2127         if (rsp->schq_contig[lvl]) {
2128                 schq = bitmap_find_next_zero_area(txsch->schq.bmap,
2129                                                   txsch->schq.max, start,
2130                                                   rsp->schq_contig[lvl], 0);
2131                 if (schq >= end)
2132                         rsp->schq_contig[lvl] = 0;
2133                 for (idx = 0; idx < rsp->schq_contig[lvl]; idx++) {
2134                         set_bit(schq, txsch->schq.bmap);
2135                         rsp->schq_contig_list[lvl][idx] = schq;
2136                         schq++;
2137                 }
2138         }
2139
2140         /* Allocate non-contiguous queue indices */
2141         if (rsp->schq[lvl]) {
2142                 idx = 0;
2143                 for (schq = start; schq < end; schq++) {
2144                         if (!test_bit(schq, txsch->schq.bmap)) {
2145                                 set_bit(schq, txsch->schq.bmap);
2146                                 rsp->schq_list[lvl][idx++] = schq;
2147                         }
2148                         if (idx == rsp->schq[lvl])
2149                                 break;
2150                 }
2151                 /* Update how many were allocated */
2152                 rsp->schq[lvl] = idx;
2153         }
2154 }
2155
2156 int rvu_mbox_handler_nix_txsch_alloc(struct rvu *rvu,
2157                                      struct nix_txsch_alloc_req *req,
2158                                      struct nix_txsch_alloc_rsp *rsp)
2159 {
2160         struct rvu_hwinfo *hw = rvu->hw;
2161         u16 pcifunc = req->hdr.pcifunc;
2162         int link, blkaddr, rc = 0;
2163         int lvl, idx, start, end;
2164         struct nix_txsch *txsch;
2165         struct nix_hw *nix_hw;
2166         u32 *pfvf_map;
2167         int nixlf;
2168         u16 schq;
2169
2170         rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
2171         if (rc)
2172                 return rc;
2173
2174         nix_hw = get_nix_hw(rvu->hw, blkaddr);
2175         if (!nix_hw)
2176                 return NIX_AF_ERR_INVALID_NIXBLK;
2177
2178         mutex_lock(&rvu->rsrc_lock);
2179
2180         /* Check if request is valid as per HW capabilities
2181          * and can be accomodated.
2182          */
2183         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
2184                 rc = nix_check_txschq_alloc_req(rvu, lvl, pcifunc, nix_hw, req);
2185                 if (rc)
2186                         goto err;
2187         }
2188
2189         /* Allocate requested Tx scheduler queues */
2190         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
2191                 txsch = &nix_hw->txsch[lvl];
2192                 pfvf_map = txsch->pfvf_map;
2193
2194                 if (!req->schq[lvl] && !req->schq_contig[lvl])
2195                         continue;
2196
2197                 rsp->schq[lvl] = req->schq[lvl];
2198                 rsp->schq_contig[lvl] = req->schq_contig[lvl];
2199
2200                 link = nix_get_tx_link(rvu, pcifunc);
2201
2202                 if (lvl >= hw->cap.nix_tx_aggr_lvl) {
2203                         start = link;
2204                         end = link;
2205                 } else if (hw->cap.nix_fixed_txschq_mapping) {
2206                         nix_get_txschq_range(rvu, pcifunc, link, &start, &end);
2207                 } else {
2208                         start = 0;
2209                         end = txsch->schq.max;
2210                 }
2211
2212                 nix_txsch_alloc(rvu, txsch, rsp, lvl, start, end);
2213
2214                 /* Reset queue config */
2215                 for (idx = 0; idx < req->schq_contig[lvl]; idx++) {
2216                         schq = rsp->schq_contig_list[lvl][idx];
2217                         if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) &
2218                             NIX_TXSCHQ_CFG_DONE))
2219                                 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0);
2220                         nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
2221                         nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq);
2222                         nix_reset_tx_schedule(rvu, blkaddr, lvl, schq);
2223                 }
2224
2225                 for (idx = 0; idx < req->schq[lvl]; idx++) {
2226                         schq = rsp->schq_list[lvl][idx];
2227                         if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) &
2228                             NIX_TXSCHQ_CFG_DONE))
2229                                 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0);
2230                         nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
2231                         nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq);
2232                         nix_reset_tx_schedule(rvu, blkaddr, lvl, schq);
2233                 }
2234         }
2235
2236         rsp->aggr_level = hw->cap.nix_tx_aggr_lvl;
2237         rsp->aggr_lvl_rr_prio = TXSCH_TL1_DFLT_RR_PRIO;
2238         rsp->link_cfg_lvl = rvu_read64(rvu, blkaddr,
2239                                        NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ?
2240                                        NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2;
2241         goto exit;
2242 err:
2243         rc = NIX_AF_ERR_TLX_ALLOC_FAIL;
2244 exit:
2245         mutex_unlock(&rvu->rsrc_lock);
2246         return rc;
2247 }
2248
2249 static void nix_smq_flush_fill_ctx(struct rvu *rvu, int blkaddr, int smq,
2250                                    struct nix_smq_flush_ctx *smq_flush_ctx)
2251 {
2252         struct nix_smq_tree_ctx *smq_tree_ctx;
2253         u64 parent_off, regval;
2254         u16 schq;
2255         int lvl;
2256
2257         smq_flush_ctx->smq = smq;
2258
2259         schq = smq;
2260         for (lvl = NIX_TXSCH_LVL_SMQ; lvl <= NIX_TXSCH_LVL_TL1; lvl++) {
2261                 smq_tree_ctx = &smq_flush_ctx->smq_tree_ctx[lvl];
2262                 if (lvl == NIX_TXSCH_LVL_TL1) {
2263                         smq_flush_ctx->tl1_schq = schq;
2264                         smq_tree_ctx->cir_off = NIX_AF_TL1X_CIR(schq);
2265                         smq_tree_ctx->pir_off = 0;
2266                         smq_tree_ctx->pir_val = 0;
2267                         parent_off = 0;
2268                 } else if (lvl == NIX_TXSCH_LVL_TL2) {
2269                         smq_flush_ctx->tl2_schq = schq;
2270                         smq_tree_ctx->cir_off = NIX_AF_TL2X_CIR(schq);
2271                         smq_tree_ctx->pir_off = NIX_AF_TL2X_PIR(schq);
2272                         parent_off = NIX_AF_TL2X_PARENT(schq);
2273                 } else if (lvl == NIX_TXSCH_LVL_TL3) {
2274                         smq_tree_ctx->cir_off = NIX_AF_TL3X_CIR(schq);
2275                         smq_tree_ctx->pir_off = NIX_AF_TL3X_PIR(schq);
2276                         parent_off = NIX_AF_TL3X_PARENT(schq);
2277                 } else if (lvl == NIX_TXSCH_LVL_TL4) {
2278                         smq_tree_ctx->cir_off = NIX_AF_TL4X_CIR(schq);
2279                         smq_tree_ctx->pir_off = NIX_AF_TL4X_PIR(schq);
2280                         parent_off = NIX_AF_TL4X_PARENT(schq);
2281                 } else if (lvl == NIX_TXSCH_LVL_MDQ) {
2282                         smq_tree_ctx->cir_off = NIX_AF_MDQX_CIR(schq);
2283                         smq_tree_ctx->pir_off = NIX_AF_MDQX_PIR(schq);
2284                         parent_off = NIX_AF_MDQX_PARENT(schq);
2285                 }
2286                 /* save cir/pir register values */
2287                 smq_tree_ctx->cir_val = rvu_read64(rvu, blkaddr, smq_tree_ctx->cir_off);
2288                 if (smq_tree_ctx->pir_off)
2289                         smq_tree_ctx->pir_val = rvu_read64(rvu, blkaddr, smq_tree_ctx->pir_off);
2290
2291                 /* get parent txsch node */
2292                 if (parent_off) {
2293                         regval = rvu_read64(rvu, blkaddr, parent_off);
2294                         schq = (regval >> 16) & 0x1FF;
2295                 }
2296         }
2297 }
2298
2299 static void nix_smq_flush_enadis_xoff(struct rvu *rvu, int blkaddr,
2300                                       struct nix_smq_flush_ctx *smq_flush_ctx, bool enable)
2301 {
2302         struct nix_txsch *txsch;
2303         struct nix_hw *nix_hw;
2304         u64 regoff;
2305         int tl2;
2306
2307         nix_hw = get_nix_hw(rvu->hw, blkaddr);
2308         if (!nix_hw)
2309                 return;
2310
2311         /* loop through all TL2s with matching PF_FUNC */
2312         txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL2];
2313         for (tl2 = 0; tl2 < txsch->schq.max; tl2++) {
2314                 /* skip the smq(flush) TL2 */
2315                 if (tl2 == smq_flush_ctx->tl2_schq)
2316                         continue;
2317                 /* skip unused TL2s */
2318                 if (TXSCH_MAP_FLAGS(txsch->pfvf_map[tl2]) & NIX_TXSCHQ_FREE)
2319                         continue;
2320                 /* skip if PF_FUNC doesn't match */
2321                 if ((TXSCH_MAP_FUNC(txsch->pfvf_map[tl2]) & ~RVU_PFVF_FUNC_MASK) !=
2322                     (TXSCH_MAP_FUNC(txsch->pfvf_map[smq_flush_ctx->tl2_schq] &
2323                                     ~RVU_PFVF_FUNC_MASK)))
2324                         continue;
2325                 /* enable/disable XOFF */
2326                 regoff = NIX_AF_TL2X_SW_XOFF(tl2);
2327                 if (enable)
2328                         rvu_write64(rvu, blkaddr, regoff, 0x1);
2329                 else
2330                         rvu_write64(rvu, blkaddr, regoff, 0x0);
2331         }
2332 }
2333
2334 static void nix_smq_flush_enadis_rate(struct rvu *rvu, int blkaddr,
2335                                       struct nix_smq_flush_ctx *smq_flush_ctx, bool enable)
2336 {
2337         u64 cir_off, pir_off, cir_val, pir_val;
2338         struct nix_smq_tree_ctx *smq_tree_ctx;
2339         int lvl;
2340
2341         for (lvl = NIX_TXSCH_LVL_SMQ; lvl <= NIX_TXSCH_LVL_TL1; lvl++) {
2342                 smq_tree_ctx = &smq_flush_ctx->smq_tree_ctx[lvl];
2343                 cir_off = smq_tree_ctx->cir_off;
2344                 cir_val = smq_tree_ctx->cir_val;
2345                 pir_off = smq_tree_ctx->pir_off;
2346                 pir_val = smq_tree_ctx->pir_val;
2347
2348                 if (enable) {
2349                         rvu_write64(rvu, blkaddr, cir_off, cir_val);
2350                         if (lvl != NIX_TXSCH_LVL_TL1)
2351                                 rvu_write64(rvu, blkaddr, pir_off, pir_val);
2352                 } else {
2353                         rvu_write64(rvu, blkaddr, cir_off, 0x0);
2354                         if (lvl != NIX_TXSCH_LVL_TL1)
2355                                 rvu_write64(rvu, blkaddr, pir_off, 0x0);
2356                 }
2357         }
2358 }
2359
2360 static int nix_smq_flush(struct rvu *rvu, int blkaddr,
2361                          int smq, u16 pcifunc, int nixlf)
2362 {
2363         struct nix_smq_flush_ctx *smq_flush_ctx;
2364         int pf = rvu_get_pf(pcifunc);
2365         u8 cgx_id = 0, lmac_id = 0;
2366         int err, restore_tx_en = 0;
2367         u64 cfg;
2368
2369         if (!is_rvu_otx2(rvu)) {
2370                 /* Skip SMQ flush if pkt count is zero */
2371                 cfg = rvu_read64(rvu, blkaddr, NIX_AF_MDQX_IN_MD_COUNT(smq));
2372                 if (!cfg)
2373                         return 0;
2374         }
2375
2376         /* enable cgx tx if disabled */
2377         if (is_pf_cgxmapped(rvu, pf)) {
2378                 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
2379                 restore_tx_en = !rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu),
2380                                                    lmac_id, true);
2381         }
2382
2383         /* XOFF all TL2s whose parent TL1 matches SMQ tree TL1 */
2384         smq_flush_ctx = kzalloc(sizeof(*smq_flush_ctx), GFP_KERNEL);
2385         if (!smq_flush_ctx)
2386                 return -ENOMEM;
2387         nix_smq_flush_fill_ctx(rvu, blkaddr, smq, smq_flush_ctx);
2388         nix_smq_flush_enadis_xoff(rvu, blkaddr, smq_flush_ctx, true);
2389         nix_smq_flush_enadis_rate(rvu, blkaddr, smq_flush_ctx, false);
2390
2391         cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq));
2392         /* Do SMQ flush and set enqueue xoff */
2393         cfg |= BIT_ULL(50) | BIT_ULL(49);
2394         rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq), cfg);
2395
2396         /* Disable backpressure from physical link,
2397          * otherwise SMQ flush may stall.
2398          */
2399         rvu_cgx_enadis_rx_bp(rvu, pf, false);
2400
2401         /* Wait for flush to complete */
2402         err = rvu_poll_reg(rvu, blkaddr,
2403                            NIX_AF_SMQX_CFG(smq), BIT_ULL(49), true);
2404         if (err)
2405                 dev_info(rvu->dev,
2406                          "NIXLF%d: SMQ%d flush failed, txlink might be busy\n",
2407                          nixlf, smq);
2408
2409         /* clear XOFF on TL2s */
2410         nix_smq_flush_enadis_rate(rvu, blkaddr, smq_flush_ctx, true);
2411         nix_smq_flush_enadis_xoff(rvu, blkaddr, smq_flush_ctx, false);
2412         kfree(smq_flush_ctx);
2413
2414         rvu_cgx_enadis_rx_bp(rvu, pf, true);
2415         /* restore cgx tx state */
2416         if (restore_tx_en)
2417                 rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false);
2418         return err;
2419 }
2420
2421 static int nix_txschq_free(struct rvu *rvu, u16 pcifunc)
2422 {
2423         int blkaddr, nixlf, lvl, schq, err;
2424         struct rvu_hwinfo *hw = rvu->hw;
2425         struct nix_txsch *txsch;
2426         struct nix_hw *nix_hw;
2427         u16 map_func;
2428
2429         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2430         if (blkaddr < 0)
2431                 return NIX_AF_ERR_AF_LF_INVALID;
2432
2433         nix_hw = get_nix_hw(rvu->hw, blkaddr);
2434         if (!nix_hw)
2435                 return NIX_AF_ERR_INVALID_NIXBLK;
2436
2437         nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
2438         if (nixlf < 0)
2439                 return NIX_AF_ERR_AF_LF_INVALID;
2440
2441         /* Disable TL2/3 queue links and all XOFF's before SMQ flush*/
2442         mutex_lock(&rvu->rsrc_lock);
2443         for (lvl = NIX_TXSCH_LVL_MDQ; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
2444                 txsch = &nix_hw->txsch[lvl];
2445
2446                 if (lvl >= hw->cap.nix_tx_aggr_lvl)
2447                         continue;
2448
2449                 for (schq = 0; schq < txsch->schq.max; schq++) {
2450                         if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
2451                                 continue;
2452                         nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
2453                         nix_clear_tx_xoff(rvu, blkaddr, lvl, schq);
2454                         nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq);
2455                 }
2456         }
2457         nix_clear_tx_xoff(rvu, blkaddr, NIX_TXSCH_LVL_TL1,
2458                           nix_get_tx_link(rvu, pcifunc));
2459
2460         /* On PF cleanup, clear cfg done flag as
2461          * PF would have changed default config.
2462          */
2463         if (!(pcifunc & RVU_PFVF_FUNC_MASK)) {
2464                 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL1];
2465                 schq = nix_get_tx_link(rvu, pcifunc);
2466                 /* Do not clear pcifunc in txsch->pfvf_map[schq] because
2467                  * VF might be using this TL1 queue
2468                  */
2469                 map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]);
2470                 txsch->pfvf_map[schq] = TXSCH_SET_FLAG(map_func, 0x0);
2471         }
2472
2473         /* Flush SMQs */
2474         txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];
2475         for (schq = 0; schq < txsch->schq.max; schq++) {
2476                 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
2477                         continue;
2478                 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf);
2479         }
2480
2481         /* Now free scheduler queues to free pool */
2482         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
2483                  /* TLs above aggregation level are shared across all PF
2484                   * and it's VFs, hence skip freeing them.
2485                   */
2486                 if (lvl >= hw->cap.nix_tx_aggr_lvl)
2487                         continue;
2488
2489                 txsch = &nix_hw->txsch[lvl];
2490                 for (schq = 0; schq < txsch->schq.max; schq++) {
2491                         if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
2492                                 continue;
2493                         nix_reset_tx_schedule(rvu, blkaddr, lvl, schq);
2494                         rvu_free_rsrc(&txsch->schq, schq);
2495                         txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
2496                 }
2497         }
2498         mutex_unlock(&rvu->rsrc_lock);
2499
2500         /* Sync cached info for this LF in NDC-TX to LLC/DRAM */
2501         rvu_write64(rvu, blkaddr, NIX_AF_NDC_TX_SYNC, BIT_ULL(12) | nixlf);
2502         err = rvu_poll_reg(rvu, blkaddr, NIX_AF_NDC_TX_SYNC, BIT_ULL(12), true);
2503         if (err)
2504                 dev_err(rvu->dev, "NDC-TX sync failed for NIXLF %d\n", nixlf);
2505
2506         return 0;
2507 }
2508
2509 static int nix_txschq_free_one(struct rvu *rvu,
2510                                struct nix_txsch_free_req *req)
2511 {
2512         struct rvu_hwinfo *hw = rvu->hw;
2513         u16 pcifunc = req->hdr.pcifunc;
2514         int lvl, schq, nixlf, blkaddr;
2515         struct nix_txsch *txsch;
2516         struct nix_hw *nix_hw;
2517         u32 *pfvf_map;
2518         int rc;
2519
2520         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2521         if (blkaddr < 0)
2522                 return NIX_AF_ERR_AF_LF_INVALID;
2523
2524         nix_hw = get_nix_hw(rvu->hw, blkaddr);
2525         if (!nix_hw)
2526                 return NIX_AF_ERR_INVALID_NIXBLK;
2527
2528         nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
2529         if (nixlf < 0)
2530                 return NIX_AF_ERR_AF_LF_INVALID;
2531
2532         lvl = req->schq_lvl;
2533         schq = req->schq;
2534         txsch = &nix_hw->txsch[lvl];
2535
2536         if (lvl >= hw->cap.nix_tx_aggr_lvl || schq >= txsch->schq.max)
2537                 return 0;
2538
2539         pfvf_map = txsch->pfvf_map;
2540         mutex_lock(&rvu->rsrc_lock);
2541
2542         if (TXSCH_MAP_FUNC(pfvf_map[schq]) != pcifunc) {
2543                 rc = NIX_AF_ERR_TLX_INVALID;
2544                 goto err;
2545         }
2546
2547         /* Clear SW_XOFF of this resource only.
2548          * For SMQ level, all path XOFF's
2549          * need to be made clear by user
2550          */
2551         nix_clear_tx_xoff(rvu, blkaddr, lvl, schq);
2552
2553         nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq);
2554         nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq);
2555
2556         /* Flush if it is a SMQ. Onus of disabling
2557          * TL2/3 queue links before SMQ flush is on user
2558          */
2559         if (lvl == NIX_TXSCH_LVL_SMQ &&
2560             nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf)) {
2561                 rc = NIX_AF_SMQ_FLUSH_FAILED;
2562                 goto err;
2563         }
2564
2565         nix_reset_tx_schedule(rvu, blkaddr, lvl, schq);
2566
2567         /* Free the resource */
2568         rvu_free_rsrc(&txsch->schq, schq);
2569         txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
2570         mutex_unlock(&rvu->rsrc_lock);
2571         return 0;
2572 err:
2573         mutex_unlock(&rvu->rsrc_lock);
2574         return rc;
2575 }
2576
2577 int rvu_mbox_handler_nix_txsch_free(struct rvu *rvu,
2578                                     struct nix_txsch_free_req *req,
2579                                     struct msg_rsp *rsp)
2580 {
2581         if (req->flags & TXSCHQ_FREE_ALL)
2582                 return nix_txschq_free(rvu, req->hdr.pcifunc);
2583         else
2584                 return nix_txschq_free_one(rvu, req);
2585 }
2586
2587 static bool is_txschq_hierarchy_valid(struct rvu *rvu, u16 pcifunc, int blkaddr,
2588                                       int lvl, u64 reg, u64 regval)
2589 {
2590         u64 regbase = reg & 0xFFFF;
2591         u16 schq, parent;
2592
2593         if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, lvl, reg))
2594                 return false;
2595
2596         schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
2597         /* Check if this schq belongs to this PF/VF or not */
2598         if (!is_valid_txschq(rvu, blkaddr, lvl, pcifunc, schq))
2599                 return false;
2600
2601         parent = (regval >> 16) & 0x1FF;
2602         /* Validate MDQ's TL4 parent */
2603         if (regbase == NIX_AF_MDQX_PARENT(0) &&
2604             !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL4, pcifunc, parent))
2605                 return false;
2606
2607         /* Validate TL4's TL3 parent */
2608         if (regbase == NIX_AF_TL4X_PARENT(0) &&
2609             !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL3, pcifunc, parent))
2610                 return false;
2611
2612         /* Validate TL3's TL2 parent */
2613         if (regbase == NIX_AF_TL3X_PARENT(0) &&
2614             !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL2, pcifunc, parent))
2615                 return false;
2616
2617         /* Validate TL2's TL1 parent */
2618         if (regbase == NIX_AF_TL2X_PARENT(0) &&
2619             !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL1, pcifunc, parent))
2620                 return false;
2621
2622         return true;
2623 }
2624
2625 static bool is_txschq_shaping_valid(struct rvu_hwinfo *hw, int lvl, u64 reg)
2626 {
2627         u64 regbase;
2628
2629         if (hw->cap.nix_shaping)
2630                 return true;
2631
2632         /* If shaping and coloring is not supported, then
2633          * *_CIR and *_PIR registers should not be configured.
2634          */
2635         regbase = reg & 0xFFFF;
2636
2637         switch (lvl) {
2638         case NIX_TXSCH_LVL_TL1:
2639                 if (regbase == NIX_AF_TL1X_CIR(0))
2640                         return false;
2641                 break;
2642         case NIX_TXSCH_LVL_TL2:
2643                 if (regbase == NIX_AF_TL2X_CIR(0) ||
2644                     regbase == NIX_AF_TL2X_PIR(0))
2645                         return false;
2646                 break;
2647         case NIX_TXSCH_LVL_TL3:
2648                 if (regbase == NIX_AF_TL3X_CIR(0) ||
2649                     regbase == NIX_AF_TL3X_PIR(0))
2650                         return false;
2651                 break;
2652         case NIX_TXSCH_LVL_TL4:
2653                 if (regbase == NIX_AF_TL4X_CIR(0) ||
2654                     regbase == NIX_AF_TL4X_PIR(0))
2655                         return false;
2656                 break;
2657         case NIX_TXSCH_LVL_MDQ:
2658                 if (regbase == NIX_AF_MDQX_CIR(0) ||
2659                     regbase == NIX_AF_MDQX_PIR(0))
2660                         return false;
2661                 break;
2662         }
2663         return true;
2664 }
2665
2666 static void nix_tl1_default_cfg(struct rvu *rvu, struct nix_hw *nix_hw,
2667                                 u16 pcifunc, int blkaddr)
2668 {
2669         u32 *pfvf_map;
2670         int schq;
2671
2672         schq = nix_get_tx_link(rvu, pcifunc);
2673         pfvf_map = nix_hw->txsch[NIX_TXSCH_LVL_TL1].pfvf_map;
2674         /* Skip if PF has already done the config */
2675         if (TXSCH_MAP_FLAGS(pfvf_map[schq]) & NIX_TXSCHQ_CFG_DONE)
2676                 return;
2677         rvu_write64(rvu, blkaddr, NIX_AF_TL1X_TOPOLOGY(schq),
2678                     (TXSCH_TL1_DFLT_RR_PRIO << 1));
2679
2680         /* On OcteonTx2 the config was in bytes and newer silcons
2681          * it's changed to weight.
2682          */
2683         if (!rvu->hw->cap.nix_common_dwrr_mtu)
2684                 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq),
2685                             TXSCH_TL1_DFLT_RR_QTM);
2686         else
2687                 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq),
2688                             CN10K_MAX_DWRR_WEIGHT);
2689
2690         rvu_write64(rvu, blkaddr, NIX_AF_TL1X_CIR(schq), 0x00);
2691         pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq], NIX_TXSCHQ_CFG_DONE);
2692 }
2693
2694 /* Register offset - [15:0]
2695  * Scheduler Queue number - [25:16]
2696  */
2697 #define NIX_TX_SCHQ_MASK        GENMASK_ULL(25, 0)
2698
2699 static int nix_txschq_cfg_read(struct rvu *rvu, struct nix_hw *nix_hw,
2700                                int blkaddr, struct nix_txschq_config *req,
2701                                struct nix_txschq_config *rsp)
2702 {
2703         u16 pcifunc = req->hdr.pcifunc;
2704         int idx, schq;
2705         u64 reg;
2706
2707         for (idx = 0; idx < req->num_regs; idx++) {
2708                 reg = req->reg[idx];
2709                 reg &= NIX_TX_SCHQ_MASK;
2710                 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
2711                 if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, req->lvl, reg) ||
2712                     !is_valid_txschq(rvu, blkaddr, req->lvl, pcifunc, schq))
2713                         return NIX_AF_INVAL_TXSCHQ_CFG;
2714                 rsp->regval[idx] = rvu_read64(rvu, blkaddr, reg);
2715         }
2716         rsp->lvl = req->lvl;
2717         rsp->num_regs = req->num_regs;
2718         return 0;
2719 }
2720
2721 void rvu_nix_tx_tl2_cfg(struct rvu *rvu, int blkaddr, u16 pcifunc,
2722                         struct nix_txsch *txsch, bool enable)
2723 {
2724         struct rvu_hwinfo *hw = rvu->hw;
2725         int lbk_link_start, lbk_links;
2726         u8 pf = rvu_get_pf(pcifunc);
2727         int schq;
2728         u64 cfg;
2729
2730         if (!is_pf_cgxmapped(rvu, pf))
2731                 return;
2732
2733         cfg = enable ? (BIT_ULL(12) | RVU_SWITCH_LBK_CHAN) : 0;
2734         lbk_link_start = hw->cgx_links;
2735
2736         for (schq = 0; schq < txsch->schq.max; schq++) {
2737                 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc)
2738                         continue;
2739                 /* Enable all LBK links with channel 63 by default so that
2740                  * packets can be sent to LBK with a NPC TX MCAM rule
2741                  */
2742                 lbk_links = hw->lbk_links;
2743                 while (lbk_links--)
2744                         rvu_write64(rvu, blkaddr,
2745                                     NIX_AF_TL3_TL2X_LINKX_CFG(schq,
2746                                                               lbk_link_start +
2747                                                               lbk_links), cfg);
2748         }
2749 }
2750
2751 int rvu_mbox_handler_nix_txschq_cfg(struct rvu *rvu,
2752                                     struct nix_txschq_config *req,
2753                                     struct nix_txschq_config *rsp)
2754 {
2755         u64 reg, val, regval, schq_regbase, val_mask;
2756         struct rvu_hwinfo *hw = rvu->hw;
2757         u16 pcifunc = req->hdr.pcifunc;
2758         struct nix_txsch *txsch;
2759         struct nix_hw *nix_hw;
2760         int blkaddr, idx, err;
2761         int nixlf, schq;
2762         u32 *pfvf_map;
2763
2764         if (req->lvl >= NIX_TXSCH_LVL_CNT ||
2765             req->num_regs > MAX_REGS_PER_MBOX_MSG)
2766                 return NIX_AF_INVAL_TXSCHQ_CFG;
2767
2768         err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
2769         if (err)
2770                 return err;
2771
2772         nix_hw = get_nix_hw(rvu->hw, blkaddr);
2773         if (!nix_hw)
2774                 return NIX_AF_ERR_INVALID_NIXBLK;
2775
2776         if (req->read)
2777                 return nix_txschq_cfg_read(rvu, nix_hw, blkaddr, req, rsp);
2778
2779         txsch = &nix_hw->txsch[req->lvl];
2780         pfvf_map = txsch->pfvf_map;
2781
2782         if (req->lvl >= hw->cap.nix_tx_aggr_lvl &&
2783             pcifunc & RVU_PFVF_FUNC_MASK) {
2784                 mutex_lock(&rvu->rsrc_lock);
2785                 if (req->lvl == NIX_TXSCH_LVL_TL1)
2786                         nix_tl1_default_cfg(rvu, nix_hw, pcifunc, blkaddr);
2787                 mutex_unlock(&rvu->rsrc_lock);
2788                 return 0;
2789         }
2790
2791         for (idx = 0; idx < req->num_regs; idx++) {
2792                 reg = req->reg[idx];
2793                 reg &= NIX_TX_SCHQ_MASK;
2794                 regval = req->regval[idx];
2795                 schq_regbase = reg & 0xFFFF;
2796                 val_mask = req->regval_mask[idx];
2797
2798                 if (!is_txschq_hierarchy_valid(rvu, pcifunc, blkaddr,
2799                                                txsch->lvl, reg, regval))
2800                         return NIX_AF_INVAL_TXSCHQ_CFG;
2801
2802                 /* Check if shaping and coloring is supported */
2803                 if (!is_txschq_shaping_valid(hw, req->lvl, reg))
2804                         continue;
2805
2806                 val = rvu_read64(rvu, blkaddr, reg);
2807                 regval = (val & val_mask) | (regval & ~val_mask);
2808
2809                 /* Handle shaping state toggle specially */
2810                 if (hw->cap.nix_shaper_toggle_wait &&
2811                     handle_txschq_shaper_update(rvu, blkaddr, nixlf,
2812                                                 req->lvl, reg, regval))
2813                         continue;
2814
2815                 /* Replace PF/VF visible NIXLF slot with HW NIXLF id */
2816                 if (schq_regbase == NIX_AF_SMQX_CFG(0)) {
2817                         nixlf = rvu_get_lf(rvu, &hw->block[blkaddr],
2818                                            pcifunc, 0);
2819                         regval &= ~(0x7FULL << 24);
2820                         regval |= ((u64)nixlf << 24);
2821                 }
2822
2823                 /* Clear 'BP_ENA' config, if it's not allowed */
2824                 if (!hw->cap.nix_tx_link_bp) {
2825                         if (schq_regbase == NIX_AF_TL4X_SDP_LINK_CFG(0) ||
2826                             (schq_regbase & 0xFF00) ==
2827                             NIX_AF_TL3_TL2X_LINKX_CFG(0, 0))
2828                                 regval &= ~BIT_ULL(13);
2829                 }
2830
2831                 /* Mark config as done for TL1 by PF */
2832                 if (schq_regbase >= NIX_AF_TL1X_SCHEDULE(0) &&
2833                     schq_regbase <= NIX_AF_TL1X_GREEN_BYTES(0)) {
2834                         schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
2835                         mutex_lock(&rvu->rsrc_lock);
2836                         pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq],
2837                                                         NIX_TXSCHQ_CFG_DONE);
2838                         mutex_unlock(&rvu->rsrc_lock);
2839                 }
2840
2841                 /* SMQ flush is special hence split register writes such
2842                  * that flush first and write rest of the bits later.
2843                  */
2844                 if (schq_regbase == NIX_AF_SMQX_CFG(0) &&
2845                     (regval & BIT_ULL(49))) {
2846                         schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT);
2847                         nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf);
2848                         regval &= ~BIT_ULL(49);
2849                 }
2850                 rvu_write64(rvu, blkaddr, reg, regval);
2851         }
2852
2853         return 0;
2854 }
2855
2856 static int nix_rx_vtag_cfg(struct rvu *rvu, int nixlf, int blkaddr,
2857                            struct nix_vtag_config *req)
2858 {
2859         u64 regval = req->vtag_size;
2860
2861         if (req->rx.vtag_type > NIX_AF_LFX_RX_VTAG_TYPE7 ||
2862             req->vtag_size > VTAGSIZE_T8)
2863                 return -EINVAL;
2864
2865         /* RX VTAG Type 7 reserved for vf vlan */
2866         if (req->rx.vtag_type == NIX_AF_LFX_RX_VTAG_TYPE7)
2867                 return NIX_AF_ERR_RX_VTAG_INUSE;
2868
2869         if (req->rx.capture_vtag)
2870                 regval |= BIT_ULL(5);
2871         if (req->rx.strip_vtag)
2872                 regval |= BIT_ULL(4);
2873
2874         rvu_write64(rvu, blkaddr,
2875                     NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, req->rx.vtag_type), regval);
2876         return 0;
2877 }
2878
2879 static int nix_tx_vtag_free(struct rvu *rvu, int blkaddr,
2880                             u16 pcifunc, int index)
2881 {
2882         struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
2883         struct nix_txvlan *vlan;
2884
2885         if (!nix_hw)
2886                 return NIX_AF_ERR_INVALID_NIXBLK;
2887
2888         vlan = &nix_hw->txvlan;
2889         if (vlan->entry2pfvf_map[index] != pcifunc)
2890                 return NIX_AF_ERR_PARAM;
2891
2892         rvu_write64(rvu, blkaddr,
2893                     NIX_AF_TX_VTAG_DEFX_DATA(index), 0x0ull);
2894         rvu_write64(rvu, blkaddr,
2895                     NIX_AF_TX_VTAG_DEFX_CTL(index), 0x0ull);
2896
2897         vlan->entry2pfvf_map[index] = 0;
2898         rvu_free_rsrc(&vlan->rsrc, index);
2899
2900         return 0;
2901 }
2902
2903 static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc)
2904 {
2905         struct nix_txvlan *vlan;
2906         struct nix_hw *nix_hw;
2907         int index, blkaddr;
2908
2909         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
2910         if (blkaddr < 0)
2911                 return;
2912
2913         nix_hw = get_nix_hw(rvu->hw, blkaddr);
2914         if (!nix_hw)
2915                 return;
2916
2917         vlan = &nix_hw->txvlan;
2918
2919         mutex_lock(&vlan->rsrc_lock);
2920         /* Scan all the entries and free the ones mapped to 'pcifunc' */
2921         for (index = 0; index < vlan->rsrc.max; index++) {
2922                 if (vlan->entry2pfvf_map[index] == pcifunc)
2923                         nix_tx_vtag_free(rvu, blkaddr, pcifunc, index);
2924         }
2925         mutex_unlock(&vlan->rsrc_lock);
2926 }
2927
2928 static int nix_tx_vtag_alloc(struct rvu *rvu, int blkaddr,
2929                              u64 vtag, u8 size)
2930 {
2931         struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
2932         struct nix_txvlan *vlan;
2933         u64 regval;
2934         int index;
2935
2936         if (!nix_hw)
2937                 return NIX_AF_ERR_INVALID_NIXBLK;
2938
2939         vlan = &nix_hw->txvlan;
2940
2941         mutex_lock(&vlan->rsrc_lock);
2942
2943         index = rvu_alloc_rsrc(&vlan->rsrc);
2944         if (index < 0) {
2945                 mutex_unlock(&vlan->rsrc_lock);
2946                 return index;
2947         }
2948
2949         mutex_unlock(&vlan->rsrc_lock);
2950
2951         regval = size ? vtag : vtag << 32;
2952
2953         rvu_write64(rvu, blkaddr,
2954                     NIX_AF_TX_VTAG_DEFX_DATA(index), regval);
2955         rvu_write64(rvu, blkaddr,
2956                     NIX_AF_TX_VTAG_DEFX_CTL(index), size);
2957
2958         return index;
2959 }
2960
2961 static int nix_tx_vtag_decfg(struct rvu *rvu, int blkaddr,
2962                              struct nix_vtag_config *req)
2963 {
2964         struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
2965         u16 pcifunc = req->hdr.pcifunc;
2966         int idx0 = req->tx.vtag0_idx;
2967         int idx1 = req->tx.vtag1_idx;
2968         struct nix_txvlan *vlan;
2969         int err = 0;
2970
2971         if (!nix_hw)
2972                 return NIX_AF_ERR_INVALID_NIXBLK;
2973
2974         vlan = &nix_hw->txvlan;
2975         if (req->tx.free_vtag0 && req->tx.free_vtag1)
2976                 if (vlan->entry2pfvf_map[idx0] != pcifunc ||
2977                     vlan->entry2pfvf_map[idx1] != pcifunc)
2978                         return NIX_AF_ERR_PARAM;
2979
2980         mutex_lock(&vlan->rsrc_lock);
2981
2982         if (req->tx.free_vtag0) {
2983                 err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx0);
2984                 if (err)
2985                         goto exit;
2986         }
2987
2988         if (req->tx.free_vtag1)
2989                 err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx1);
2990
2991 exit:
2992         mutex_unlock(&vlan->rsrc_lock);
2993         return err;
2994 }
2995
2996 static int nix_tx_vtag_cfg(struct rvu *rvu, int blkaddr,
2997                            struct nix_vtag_config *req,
2998                            struct nix_vtag_config_rsp *rsp)
2999 {
3000         struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr);
3001         struct nix_txvlan *vlan;
3002         u16 pcifunc = req->hdr.pcifunc;
3003
3004         if (!nix_hw)
3005                 return NIX_AF_ERR_INVALID_NIXBLK;
3006
3007         vlan = &nix_hw->txvlan;
3008         if (req->tx.cfg_vtag0) {
3009                 rsp->vtag0_idx =
3010                         nix_tx_vtag_alloc(rvu, blkaddr,
3011                                           req->tx.vtag0, req->vtag_size);
3012
3013                 if (rsp->vtag0_idx < 0)
3014                         return NIX_AF_ERR_TX_VTAG_NOSPC;
3015
3016                 vlan->entry2pfvf_map[rsp->vtag0_idx] = pcifunc;
3017         }
3018
3019         if (req->tx.cfg_vtag1) {
3020                 rsp->vtag1_idx =
3021                         nix_tx_vtag_alloc(rvu, blkaddr,
3022                                           req->tx.vtag1, req->vtag_size);
3023
3024                 if (rsp->vtag1_idx < 0)
3025                         goto err_free;
3026
3027                 vlan->entry2pfvf_map[rsp->vtag1_idx] = pcifunc;
3028         }
3029
3030         return 0;
3031
3032 err_free:
3033         if (req->tx.cfg_vtag0)
3034                 nix_tx_vtag_free(rvu, blkaddr, pcifunc, rsp->vtag0_idx);
3035
3036         return NIX_AF_ERR_TX_VTAG_NOSPC;
3037 }
3038
3039 int rvu_mbox_handler_nix_vtag_cfg(struct rvu *rvu,
3040                                   struct nix_vtag_config *req,
3041                                   struct nix_vtag_config_rsp *rsp)
3042 {
3043         u16 pcifunc = req->hdr.pcifunc;
3044         int blkaddr, nixlf, err;
3045
3046         err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
3047         if (err)
3048                 return err;
3049
3050         if (req->cfg_type) {
3051                 /* rx vtag configuration */
3052                 err = nix_rx_vtag_cfg(rvu, nixlf, blkaddr, req);
3053                 if (err)
3054                         return NIX_AF_ERR_PARAM;
3055         } else {
3056                 /* tx vtag configuration */
3057                 if ((req->tx.cfg_vtag0 || req->tx.cfg_vtag1) &&
3058                     (req->tx.free_vtag0 || req->tx.free_vtag1))
3059                         return NIX_AF_ERR_PARAM;
3060
3061                 if (req->tx.cfg_vtag0 || req->tx.cfg_vtag1)
3062                         return nix_tx_vtag_cfg(rvu, blkaddr, req, rsp);
3063
3064                 if (req->tx.free_vtag0 || req->tx.free_vtag1)
3065                         return nix_tx_vtag_decfg(rvu, blkaddr, req);
3066         }
3067
3068         return 0;
3069 }
3070
3071 static int nix_blk_setup_mce(struct rvu *rvu, struct nix_hw *nix_hw,
3072                              int mce, u8 op, u16 pcifunc, int next,
3073                              int index, u8 mce_op, bool eol)
3074 {
3075         struct nix_aq_enq_req aq_req;
3076         int err;
3077
3078         aq_req.hdr.pcifunc = 0;
3079         aq_req.ctype = NIX_AQ_CTYPE_MCE;
3080         aq_req.op = op;
3081         aq_req.qidx = mce;
3082
3083         /* Use RSS with RSS index 0 */
3084         aq_req.mce.op = mce_op;
3085         aq_req.mce.index = index;
3086         aq_req.mce.eol = eol;
3087         aq_req.mce.pf_func = pcifunc;
3088         aq_req.mce.next = next;
3089
3090         /* All fields valid */
3091         *(u64 *)(&aq_req.mce_mask) = ~0ULL;
3092
3093         err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, &aq_req, NULL);
3094         if (err) {
3095                 dev_err(rvu->dev, "Failed to setup Bcast MCE for PF%d:VF%d\n",
3096                         rvu_get_pf(pcifunc), pcifunc & RVU_PFVF_FUNC_MASK);
3097                 return err;
3098         }
3099         return 0;
3100 }
3101
3102 static void nix_delete_mcast_mce_list(struct nix_mce_list *mce_list)
3103 {
3104         struct hlist_node *tmp;
3105         struct mce *mce;
3106
3107         /* Scan through the current list */
3108         hlist_for_each_entry_safe(mce, tmp, &mce_list->head, node) {
3109                 hlist_del(&mce->node);
3110                 kfree(mce);
3111         }
3112
3113         mce_list->count = 0;
3114         mce_list->max = 0;
3115 }
3116
3117 static int nix_get_last_mce_list_index(struct nix_mcast_grp_elem *elem)
3118 {
3119         return elem->mce_start_index + elem->mcast_mce_list.count - 1;
3120 }
3121
3122 static int nix_update_ingress_mce_list_hw(struct rvu *rvu,
3123                                           struct nix_hw *nix_hw,
3124                                           struct nix_mcast_grp_elem *elem)
3125 {
3126         int idx, last_idx, next_idx, err;
3127         struct nix_mce_list *mce_list;
3128         struct mce *mce, *prev_mce;
3129
3130         mce_list = &elem->mcast_mce_list;
3131         idx = elem->mce_start_index;
3132         last_idx = nix_get_last_mce_list_index(elem);
3133         hlist_for_each_entry(mce, &mce_list->head, node) {
3134                 if (idx > last_idx)
3135                         break;
3136
3137                 if (!mce->is_active) {
3138                         if (idx == elem->mce_start_index) {
3139                                 idx++;
3140                                 prev_mce = mce;
3141                                 elem->mce_start_index = idx;
3142                                 continue;
3143                         } else if (idx == last_idx) {
3144                                 err = nix_blk_setup_mce(rvu, nix_hw, idx - 1, NIX_AQ_INSTOP_WRITE,
3145                                                         prev_mce->pcifunc, next_idx,
3146                                                         prev_mce->rq_rss_index,
3147                                                         prev_mce->dest_type,
3148                                                         false);
3149                                 if (err)
3150                                         return err;
3151
3152                                 break;
3153                         }
3154                 }
3155
3156                 next_idx = idx + 1;
3157                 /* EOL should be set in last MCE */
3158                 err = nix_blk_setup_mce(rvu, nix_hw, idx, NIX_AQ_INSTOP_WRITE,
3159                                         mce->pcifunc, next_idx,
3160                                         mce->rq_rss_index, mce->dest_type,
3161                                         (next_idx > last_idx) ? true : false);
3162                 if (err)
3163                         return err;
3164
3165                 idx++;
3166                 prev_mce = mce;
3167         }
3168
3169         return 0;
3170 }
3171
3172 static void nix_update_egress_mce_list_hw(struct rvu *rvu,
3173                                           struct nix_hw *nix_hw,
3174                                           struct nix_mcast_grp_elem *elem)
3175 {
3176         struct nix_mce_list *mce_list;
3177         int idx, last_idx, next_idx;
3178         struct mce *mce, *prev_mce;
3179         u64 regval;
3180         u8 eol;
3181
3182         mce_list = &elem->mcast_mce_list;
3183         idx = elem->mce_start_index;
3184         last_idx = nix_get_last_mce_list_index(elem);
3185         hlist_for_each_entry(mce, &mce_list->head, node) {
3186                 if (idx > last_idx)
3187                         break;
3188
3189                 if (!mce->is_active) {
3190                         if (idx == elem->mce_start_index) {
3191                                 idx++;
3192                                 prev_mce = mce;
3193                                 elem->mce_start_index = idx;
3194                                 continue;
3195                         } else if (idx == last_idx) {
3196                                 regval = (next_idx << 16) | (1 << 12) | prev_mce->channel;
3197                                 rvu_write64(rvu, nix_hw->blkaddr,
3198                                             NIX_AF_TX_MCASTX(idx - 1),
3199                                             regval);
3200                                 break;
3201                         }
3202                 }
3203
3204                 eol = 0;
3205                 next_idx = idx + 1;
3206                 /* EOL should be set in last MCE */
3207                 if (next_idx > last_idx)
3208                         eol = 1;
3209
3210                 regval = (next_idx << 16) | (eol << 12) | mce->channel;
3211                 rvu_write64(rvu, nix_hw->blkaddr,
3212                             NIX_AF_TX_MCASTX(idx),
3213                             regval);
3214                 idx++;
3215                 prev_mce = mce;
3216         }
3217 }
3218
3219 static int nix_del_mce_list_entry(struct rvu *rvu,
3220                                   struct nix_hw *nix_hw,
3221                                   struct nix_mcast_grp_elem *elem,
3222                                   struct nix_mcast_grp_update_req *req)
3223 {
3224         u32 num_entry = req->num_mce_entry;
3225         struct nix_mce_list *mce_list;
3226         struct mce *mce;
3227         bool is_found;
3228         int i;
3229
3230         mce_list = &elem->mcast_mce_list;
3231         for (i = 0; i < num_entry; i++) {
3232                 is_found = false;
3233                 hlist_for_each_entry(mce, &mce_list->head, node) {
3234                         /* If already exists, then delete */
3235                         if (mce->pcifunc == req->pcifunc[i]) {
3236                                 hlist_del(&mce->node);
3237                                 kfree(mce);
3238                                 mce_list->count--;
3239                                 is_found = true;
3240                                 break;
3241                         }
3242                 }
3243
3244                 if (!is_found)
3245                         return NIX_AF_ERR_INVALID_MCAST_DEL_REQ;
3246         }
3247
3248         mce_list->max = mce_list->count;
3249         /* Dump the updated list to HW */
3250         if (elem->dir == NIX_MCAST_INGRESS)
3251                 return nix_update_ingress_mce_list_hw(rvu, nix_hw, elem);
3252
3253         nix_update_egress_mce_list_hw(rvu, nix_hw, elem);
3254         return 0;
3255 }
3256
3257 static int nix_add_mce_list_entry(struct rvu *rvu,
3258                                   struct nix_hw *nix_hw,
3259                                   struct nix_mcast_grp_elem *elem,
3260                                   struct nix_mcast_grp_update_req *req)
3261 {
3262         u32 num_entry = req->num_mce_entry;
3263         struct nix_mce_list *mce_list;
3264         struct hlist_node *tmp;
3265         struct mce *mce;
3266         int i;
3267
3268         mce_list = &elem->mcast_mce_list;
3269         for (i = 0; i < num_entry; i++) {
3270                 mce = kzalloc(sizeof(*mce), GFP_KERNEL);
3271                 if (!mce)
3272                         goto free_mce;
3273
3274                 mce->pcifunc = req->pcifunc[i];
3275                 mce->channel = req->channel[i];
3276                 mce->rq_rss_index = req->rq_rss_index[i];
3277                 mce->dest_type = req->dest_type[i];
3278                 mce->is_active = 1;
3279                 hlist_add_head(&mce->node, &mce_list->head);
3280                 mce_list->count++;
3281         }
3282
3283         mce_list->max += num_entry;
3284
3285         /* Dump the updated list to HW */
3286         if (elem->dir == NIX_MCAST_INGRESS)
3287                 return nix_update_ingress_mce_list_hw(rvu, nix_hw, elem);
3288
3289         nix_update_egress_mce_list_hw(rvu, nix_hw, elem);
3290         return 0;
3291
3292 free_mce:
3293         hlist_for_each_entry_safe(mce, tmp, &mce_list->head, node) {
3294                 hlist_del(&mce->node);
3295                 kfree(mce);
3296                 mce_list->count--;
3297         }
3298
3299         return -ENOMEM;
3300 }
3301
3302 static int nix_update_mce_list_entry(struct nix_mce_list *mce_list,
3303                                      u16 pcifunc, bool add)
3304 {
3305         struct mce *mce, *tail = NULL;
3306         bool delete = false;
3307
3308         /* Scan through the current list */
3309         hlist_for_each_entry(mce, &mce_list->head, node) {
3310                 /* If already exists, then delete */
3311                 if (mce->pcifunc == pcifunc && !add) {
3312                         delete = true;
3313                         break;
3314                 } else if (mce->pcifunc == pcifunc && add) {
3315                         /* entry already exists */
3316                         return 0;
3317                 }
3318                 tail = mce;
3319         }
3320
3321         if (delete) {
3322                 hlist_del(&mce->node);
3323                 kfree(mce);
3324                 mce_list->count--;
3325                 return 0;
3326         }
3327
3328         if (!add)
3329                 return 0;
3330
3331         /* Add a new one to the list, at the tail */
3332         mce = kzalloc(sizeof(*mce), GFP_KERNEL);
3333         if (!mce)
3334                 return -ENOMEM;
3335         mce->pcifunc = pcifunc;
3336         if (!tail)
3337                 hlist_add_head(&mce->node, &mce_list->head);
3338         else
3339                 hlist_add_behind(&mce->node, &tail->node);
3340         mce_list->count++;
3341         return 0;
3342 }
3343
3344 int nix_update_mce_list(struct rvu *rvu, u16 pcifunc,
3345                         struct nix_mce_list *mce_list,
3346                         int mce_idx, int mcam_index, bool add)
3347 {
3348         int err = 0, idx, next_idx, last_idx, blkaddr, npc_blkaddr;
3349         struct npc_mcam *mcam = &rvu->hw->mcam;
3350         struct nix_mcast *mcast;
3351         struct nix_hw *nix_hw;
3352         struct mce *mce;
3353
3354         if (!mce_list)
3355                 return -EINVAL;
3356
3357         /* Get this PF/VF func's MCE index */
3358         idx = mce_idx + (pcifunc & RVU_PFVF_FUNC_MASK);
3359
3360         if (idx > (mce_idx + mce_list->max)) {
3361                 dev_err(rvu->dev,
3362                         "%s: Idx %d > max MCE idx %d, for PF%d bcast list\n",
3363                         __func__, idx, mce_list->max,
3364                         pcifunc >> RVU_PFVF_PF_SHIFT);
3365                 return -EINVAL;
3366         }
3367
3368         err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
3369         if (err)
3370                 return err;
3371
3372         mcast = &nix_hw->mcast;
3373         mutex_lock(&mcast->mce_lock);
3374
3375         err = nix_update_mce_list_entry(mce_list, pcifunc, add);
3376         if (err)
3377                 goto end;
3378
3379         /* Disable MCAM entry in NPC */
3380         if (!mce_list->count) {
3381                 npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3382                 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, mcam_index, false);
3383                 goto end;
3384         }
3385
3386         /* Dump the updated list to HW */
3387         idx = mce_idx;
3388         last_idx = idx + mce_list->count - 1;
3389         hlist_for_each_entry(mce, &mce_list->head, node) {
3390                 if (idx > last_idx)
3391                         break;
3392
3393                 next_idx = idx + 1;
3394                 /* EOL should be set in last MCE */
3395                 err = nix_blk_setup_mce(rvu, nix_hw, idx, NIX_AQ_INSTOP_WRITE,
3396                                         mce->pcifunc, next_idx,
3397                                         0, 1,
3398                                         (next_idx > last_idx) ? true : false);
3399                 if (err)
3400                         goto end;
3401                 idx++;
3402         }
3403
3404 end:
3405         mutex_unlock(&mcast->mce_lock);
3406         return err;
3407 }
3408
3409 void nix_get_mce_list(struct rvu *rvu, u16 pcifunc, int type,
3410                       struct nix_mce_list **mce_list, int *mce_idx)
3411 {
3412         struct rvu_hwinfo *hw = rvu->hw;
3413         struct rvu_pfvf *pfvf;
3414
3415         if (!hw->cap.nix_rx_multicast ||
3416             !is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc & ~RVU_PFVF_FUNC_MASK))) {
3417                 *mce_list = NULL;
3418                 *mce_idx = 0;
3419                 return;
3420         }
3421
3422         /* Get this PF/VF func's MCE index */
3423         pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);
3424
3425         if (type == NIXLF_BCAST_ENTRY) {
3426                 *mce_list = &pfvf->bcast_mce_list;
3427                 *mce_idx = pfvf->bcast_mce_idx;
3428         } else if (type == NIXLF_ALLMULTI_ENTRY) {
3429                 *mce_list = &pfvf->mcast_mce_list;
3430                 *mce_idx = pfvf->mcast_mce_idx;
3431         } else if (type == NIXLF_PROMISC_ENTRY) {
3432                 *mce_list = &pfvf->promisc_mce_list;
3433                 *mce_idx = pfvf->promisc_mce_idx;
3434         }  else {
3435                 *mce_list = NULL;
3436                 *mce_idx = 0;
3437         }
3438 }
3439
3440 static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc,
3441                                int type, bool add)
3442 {
3443         int err = 0, nixlf, blkaddr, mcam_index, mce_idx;
3444         struct npc_mcam *mcam = &rvu->hw->mcam;
3445         struct rvu_hwinfo *hw = rvu->hw;
3446         struct nix_mce_list *mce_list;
3447         int pf;
3448
3449         /* skip multicast pkt replication for AF's VFs & SDP links */
3450         if (is_lbk_vf(rvu, pcifunc) || is_sdp_pfvf(pcifunc))
3451                 return 0;
3452
3453         if (!hw->cap.nix_rx_multicast)
3454                 return 0;
3455
3456         pf = rvu_get_pf(pcifunc);
3457         if (!is_pf_cgxmapped(rvu, pf))
3458                 return 0;
3459
3460         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
3461         if (blkaddr < 0)
3462                 return -EINVAL;
3463
3464         nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);
3465         if (nixlf < 0)
3466                 return -EINVAL;
3467
3468         nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx);
3469
3470         mcam_index = npc_get_nixlf_mcam_index(mcam,
3471                                               pcifunc & ~RVU_PFVF_FUNC_MASK,
3472                                               nixlf, type);
3473         err = nix_update_mce_list(rvu, pcifunc, mce_list,
3474                                   mce_idx, mcam_index, add);
3475         return err;
3476 }
3477
3478 static void nix_setup_mcast_grp(struct nix_hw *nix_hw)
3479 {
3480         struct nix_mcast_grp *mcast_grp = &nix_hw->mcast_grp;
3481
3482         INIT_LIST_HEAD(&mcast_grp->mcast_grp_head);
3483         mutex_init(&mcast_grp->mcast_grp_lock);
3484         mcast_grp->next_grp_index = 1;
3485         mcast_grp->count = 0;
3486 }
3487
3488 static int nix_setup_mce_tables(struct rvu *rvu, struct nix_hw *nix_hw)
3489 {
3490         struct nix_mcast *mcast = &nix_hw->mcast;
3491         int err, pf, numvfs, idx;
3492         struct rvu_pfvf *pfvf;
3493         u16 pcifunc;
3494         u64 cfg;
3495
3496         /* Skip PF0 (i.e AF) */
3497         for (pf = 1; pf < (rvu->cgx_mapped_pfs + 1); pf++) {
3498                 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
3499                 /* If PF is not enabled, nothing to do */
3500                 if (!((cfg >> 20) & 0x01))
3501                         continue;
3502                 /* Get numVFs attached to this PF */
3503                 numvfs = (cfg >> 12) & 0xFF;
3504
3505                 pfvf = &rvu->pf[pf];
3506
3507                 /* This NIX0/1 block mapped to PF ? */
3508                 if (pfvf->nix_blkaddr != nix_hw->blkaddr)
3509                         continue;
3510
3511                 /* save start idx of broadcast mce list */
3512                 pfvf->bcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1, NIX_MCAST_INGRESS);
3513                 nix_mce_list_init(&pfvf->bcast_mce_list, numvfs + 1);
3514
3515                 /* save start idx of multicast mce list */
3516                 pfvf->mcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1, NIX_MCAST_INGRESS);
3517                 nix_mce_list_init(&pfvf->mcast_mce_list, numvfs + 1);
3518
3519                 /* save the start idx of promisc mce list */
3520                 pfvf->promisc_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1, NIX_MCAST_INGRESS);
3521                 nix_mce_list_init(&pfvf->promisc_mce_list, numvfs + 1);
3522
3523                 for (idx = 0; idx < (numvfs + 1); idx++) {
3524                         /* idx-0 is for PF, followed by VFs */
3525                         pcifunc = (pf << RVU_PFVF_PF_SHIFT);
3526                         pcifunc |= idx;
3527                         /* Add dummy entries now, so that we don't have to check
3528                          * for whether AQ_OP should be INIT/WRITE later on.
3529                          * Will be updated when a NIXLF is attached/detached to
3530                          * these PF/VFs.
3531                          */
3532                         err = nix_blk_setup_mce(rvu, nix_hw,
3533                                                 pfvf->bcast_mce_idx + idx,
3534                                                 NIX_AQ_INSTOP_INIT,
3535                                                 pcifunc, 0, 0, 1, true);
3536                         if (err)
3537                                 return err;
3538
3539                         /* add dummy entries to multicast mce list */
3540                         err = nix_blk_setup_mce(rvu, nix_hw,
3541                                                 pfvf->mcast_mce_idx + idx,
3542                                                 NIX_AQ_INSTOP_INIT,
3543                                                 pcifunc, 0, 0, 1, true);
3544                         if (err)
3545                                 return err;
3546
3547                         /* add dummy entries to promisc mce list */
3548                         err = nix_blk_setup_mce(rvu, nix_hw,
3549                                                 pfvf->promisc_mce_idx + idx,
3550                                                 NIX_AQ_INSTOP_INIT,
3551                                                 pcifunc, 0, 0, 1, true);
3552                         if (err)
3553                                 return err;
3554                 }
3555         }
3556         return 0;
3557 }
3558
3559 static int nix_setup_mcast(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
3560 {
3561         struct nix_mcast *mcast = &nix_hw->mcast;
3562         struct rvu_hwinfo *hw = rvu->hw;
3563         int err, size;
3564
3565         size = (rvu_read64(rvu, blkaddr, NIX_AF_CONST3) >> 16) & 0x0F;
3566         size = BIT_ULL(size);
3567
3568         /* Allocate bitmap for rx mce entries */
3569         mcast->mce_counter[NIX_MCAST_INGRESS].max = 256UL << MC_TBL_SIZE;
3570         err = rvu_alloc_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]);
3571         if (err)
3572                 return -ENOMEM;
3573
3574         /* Allocate bitmap for tx mce entries */
3575         mcast->mce_counter[NIX_MCAST_EGRESS].max = MC_TX_MAX;
3576         err = rvu_alloc_bitmap(&mcast->mce_counter[NIX_MCAST_EGRESS]);
3577         if (err) {
3578                 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]);
3579                 return -ENOMEM;
3580         }
3581
3582         /* Alloc memory for multicast/mirror replication entries */
3583         err = qmem_alloc(rvu->dev, &mcast->mce_ctx,
3584                          mcast->mce_counter[NIX_MCAST_INGRESS].max, size);
3585         if (err) {
3586                 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]);
3587                 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_EGRESS]);
3588                 return -ENOMEM;
3589         }
3590
3591         rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BASE,
3592                     (u64)mcast->mce_ctx->iova);
3593
3594         /* Set max list length equal to max no of VFs per PF  + PF itself */
3595         rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG,
3596                     BIT_ULL(36) | (hw->max_vfs_per_pf << 4) | MC_TBL_SIZE);
3597
3598         /* Alloc memory for multicast replication buffers */
3599         size = rvu_read64(rvu, blkaddr, NIX_AF_MC_MIRROR_CONST) & 0xFFFF;
3600         err = qmem_alloc(rvu->dev, &mcast->mcast_buf,
3601                          (8UL << MC_BUF_CNT), size);
3602         if (err) {
3603                 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_INGRESS]);
3604                 rvu_free_bitmap(&mcast->mce_counter[NIX_MCAST_EGRESS]);
3605                 return -ENOMEM;
3606         }
3607
3608         rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_BASE,
3609                     (u64)mcast->mcast_buf->iova);
3610
3611         /* Alloc pkind for NIX internal RX multicast/mirror replay */
3612         mcast->replay_pkind = rvu_alloc_rsrc(&hw->pkind.rsrc);
3613
3614         rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_CFG,
3615                     BIT_ULL(63) | (mcast->replay_pkind << 24) |
3616                     BIT_ULL(20) | MC_BUF_CNT);
3617
3618         mutex_init(&mcast->mce_lock);
3619
3620         nix_setup_mcast_grp(nix_hw);
3621
3622         return nix_setup_mce_tables(rvu, nix_hw);
3623 }
3624
3625 static int nix_setup_txvlan(struct rvu *rvu, struct nix_hw *nix_hw)
3626 {
3627         struct nix_txvlan *vlan = &nix_hw->txvlan;
3628         int err;
3629
3630         /* Allocate resource bimap for tx vtag def registers*/
3631         vlan->rsrc.max = NIX_TX_VTAG_DEF_MAX;
3632         err = rvu_alloc_bitmap(&vlan->rsrc);
3633         if (err)
3634                 return -ENOMEM;
3635
3636         /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */
3637         vlan->entry2pfvf_map = devm_kcalloc(rvu->dev, vlan->rsrc.max,
3638                                             sizeof(u16), GFP_KERNEL);
3639         if (!vlan->entry2pfvf_map)
3640                 goto free_mem;
3641
3642         mutex_init(&vlan->rsrc_lock);
3643         return 0;
3644
3645 free_mem:
3646         kfree(vlan->rsrc.bmap);
3647         return -ENOMEM;
3648 }
3649
3650 static int nix_setup_txschq(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr)
3651 {
3652         struct nix_txsch *txsch;
3653         int err, lvl, schq;
3654         u64 cfg, reg;
3655
3656         /* Get scheduler queue count of each type and alloc
3657          * bitmap for each for alloc/free/attach operations.
3658          */
3659         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
3660                 txsch = &nix_hw->txsch[lvl];
3661                 txsch->lvl = lvl;
3662                 switch (lvl) {
3663                 case NIX_TXSCH_LVL_SMQ:
3664                         reg = NIX_AF_MDQ_CONST;
3665                         break;
3666                 case NIX_TXSCH_LVL_TL4:
3667                         reg = NIX_AF_TL4_CONST;
3668                         break;
3669                 case NIX_TXSCH_LVL_TL3:
3670                         reg = NIX_AF_TL3_CONST;
3671                         break;
3672                 case NIX_TXSCH_LVL_TL2:
3673                         reg = NIX_AF_TL2_CONST;
3674                         break;
3675                 case NIX_TXSCH_LVL_TL1:
3676                         reg = NIX_AF_TL1_CONST;
3677                         break;
3678                 }
3679                 cfg = rvu_read64(rvu, blkaddr, reg);
3680                 txsch->schq.max = cfg & 0xFFFF;
3681                 err = rvu_alloc_bitmap(&txsch->schq);
3682                 if (err)
3683                         return err;
3684
3685                 /* Allocate memory for scheduler queues to
3686                  * PF/VF pcifunc mapping info.
3687                  */
3688                 txsch->pfvf_map = devm_kcalloc(rvu->dev, txsch->schq.max,
3689                                                sizeof(u32), GFP_KERNEL);
3690                 if (!txsch->pfvf_map)
3691                         return -ENOMEM;
3692                 for (schq = 0; schq < txsch->schq.max; schq++)
3693                         txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE);
3694         }
3695
3696         /* Setup a default value of 8192 as DWRR MTU */
3697         if (rvu->hw->cap.nix_common_dwrr_mtu ||
3698             rvu->hw->cap.nix_multiple_dwrr_mtu) {
3699                 rvu_write64(rvu, blkaddr,
3700                             nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_RPM),
3701                             convert_bytes_to_dwrr_mtu(8192));
3702                 rvu_write64(rvu, blkaddr,
3703                             nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_LBK),
3704                             convert_bytes_to_dwrr_mtu(8192));
3705                 rvu_write64(rvu, blkaddr,
3706                             nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_SDP),
3707                             convert_bytes_to_dwrr_mtu(8192));
3708         }
3709
3710         return 0;
3711 }
3712
3713 int rvu_nix_reserve_mark_format(struct rvu *rvu, struct nix_hw *nix_hw,
3714                                 int blkaddr, u32 cfg)
3715 {
3716         int fmt_idx;
3717
3718         for (fmt_idx = 0; fmt_idx < nix_hw->mark_format.in_use; fmt_idx++) {
3719                 if (nix_hw->mark_format.cfg[fmt_idx] == cfg)
3720                         return fmt_idx;
3721         }
3722         if (fmt_idx >= nix_hw->mark_format.total)
3723                 return -ERANGE;
3724
3725         rvu_write64(rvu, blkaddr, NIX_AF_MARK_FORMATX_CTL(fmt_idx), cfg);
3726         nix_hw->mark_format.cfg[fmt_idx] = cfg;
3727         nix_hw->mark_format.in_use++;
3728         return fmt_idx;
3729 }
3730
3731 static int nix_af_mark_format_setup(struct rvu *rvu, struct nix_hw *nix_hw,
3732                                     int blkaddr)
3733 {
3734         u64 cfgs[] = {
3735                 [NIX_MARK_CFG_IP_DSCP_RED]         = 0x10003,
3736                 [NIX_MARK_CFG_IP_DSCP_YELLOW]      = 0x11200,
3737                 [NIX_MARK_CFG_IP_DSCP_YELLOW_RED]  = 0x11203,
3738                 [NIX_MARK_CFG_IP_ECN_RED]          = 0x6000c,
3739                 [NIX_MARK_CFG_IP_ECN_YELLOW]       = 0x60c00,
3740                 [NIX_MARK_CFG_IP_ECN_YELLOW_RED]   = 0x60c0c,
3741                 [NIX_MARK_CFG_VLAN_DEI_RED]        = 0x30008,
3742                 [NIX_MARK_CFG_VLAN_DEI_YELLOW]     = 0x30800,
3743                 [NIX_MARK_CFG_VLAN_DEI_YELLOW_RED] = 0x30808,
3744         };
3745         int i, rc;
3746         u64 total;
3747
3748         total = (rvu_read64(rvu, blkaddr, NIX_AF_PSE_CONST) & 0xFF00) >> 8;
3749         nix_hw->mark_format.total = (u8)total;
3750         nix_hw->mark_format.cfg = devm_kcalloc(rvu->dev, total, sizeof(u32),
3751                                                GFP_KERNEL);
3752         if (!nix_hw->mark_format.cfg)
3753                 return -ENOMEM;
3754         for (i = 0; i < NIX_MARK_CFG_MAX; i++) {
3755                 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfgs[i]);
3756                 if (rc < 0)
3757                         dev_err(rvu->dev, "Err %d in setup mark format %d\n",
3758                                 i, rc);
3759         }
3760
3761         return 0;
3762 }
3763
3764 static void rvu_get_lbk_link_max_frs(struct rvu *rvu,  u16 *max_mtu)
3765 {
3766         /* CN10K supports LBK FIFO size 72 KB */
3767         if (rvu->hw->lbk_bufsize == 0x12000)
3768                 *max_mtu = CN10K_LBK_LINK_MAX_FRS;
3769         else
3770                 *max_mtu = NIC_HW_MAX_FRS;
3771 }
3772
3773 static void rvu_get_lmac_link_max_frs(struct rvu *rvu, u16 *max_mtu)
3774 {
3775         int fifo_size = rvu_cgx_get_fifolen(rvu);
3776
3777         /* RPM supports FIFO len 128 KB and RPM2 supports double the
3778          * FIFO len to accommodate 8 LMACS
3779          */
3780         if (fifo_size == 0x20000 || fifo_size == 0x40000)
3781                 *max_mtu = CN10K_LMAC_LINK_MAX_FRS;
3782         else
3783                 *max_mtu = NIC_HW_MAX_FRS;
3784 }
3785
3786 int rvu_mbox_handler_nix_get_hw_info(struct rvu *rvu, struct msg_req *req,
3787                                      struct nix_hw_info *rsp)
3788 {
3789         u16 pcifunc = req->hdr.pcifunc;
3790         u64 dwrr_mtu;
3791         int blkaddr;
3792
3793         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
3794         if (blkaddr < 0)
3795                 return NIX_AF_ERR_AF_LF_INVALID;
3796
3797         if (is_lbk_vf(rvu, pcifunc))
3798                 rvu_get_lbk_link_max_frs(rvu, &rsp->max_mtu);
3799         else
3800                 rvu_get_lmac_link_max_frs(rvu, &rsp->max_mtu);
3801
3802         rsp->min_mtu = NIC_HW_MIN_FRS;
3803
3804         if (!rvu->hw->cap.nix_common_dwrr_mtu &&
3805             !rvu->hw->cap.nix_multiple_dwrr_mtu) {
3806                 /* Return '1' on OTx2 */
3807                 rsp->rpm_dwrr_mtu = 1;
3808                 rsp->sdp_dwrr_mtu = 1;
3809                 rsp->lbk_dwrr_mtu = 1;
3810                 return 0;
3811         }
3812
3813         /* Return DWRR_MTU for TLx_SCHEDULE[RR_WEIGHT] config */
3814         dwrr_mtu = rvu_read64(rvu, blkaddr,
3815                               nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_RPM));
3816         rsp->rpm_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu);
3817
3818         dwrr_mtu = rvu_read64(rvu, blkaddr,
3819                               nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_SDP));
3820         rsp->sdp_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu);
3821
3822         dwrr_mtu = rvu_read64(rvu, blkaddr,
3823                               nix_get_dwrr_mtu_reg(rvu->hw, SMQ_LINK_TYPE_LBK));
3824         rsp->lbk_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu);
3825
3826         return 0;
3827 }
3828
3829 int rvu_mbox_handler_nix_stats_rst(struct rvu *rvu, struct msg_req *req,
3830                                    struct msg_rsp *rsp)
3831 {
3832         u16 pcifunc = req->hdr.pcifunc;
3833         int i, nixlf, blkaddr, err;
3834         u64 stats;
3835
3836         err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
3837         if (err)
3838                 return err;
3839
3840         /* Get stats count supported by HW */
3841         stats = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
3842
3843         /* Reset tx stats */
3844         for (i = 0; i < ((stats >> 24) & 0xFF); i++)
3845                 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_STATX(nixlf, i), 0);
3846
3847         /* Reset rx stats */
3848         for (i = 0; i < ((stats >> 32) & 0xFF); i++)
3849                 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_STATX(nixlf, i), 0);
3850
3851         return 0;
3852 }
3853
3854 /* Returns the ALG index to be set into NPC_RX_ACTION */
3855 static int get_flowkey_alg_idx(struct nix_hw *nix_hw, u32 flow_cfg)
3856 {
3857         int i;
3858
3859         /* Scan over exiting algo entries to find a match */
3860         for (i = 0; i < nix_hw->flowkey.in_use; i++)
3861                 if (nix_hw->flowkey.flowkey[i] == flow_cfg)
3862                         return i;
3863
3864         return -ERANGE;
3865 }
3866
3867 static int set_flowkey_fields(struct nix_rx_flowkey_alg *alg, u32 flow_cfg)
3868 {
3869         int idx, nr_field, key_off, field_marker, keyoff_marker;
3870         int max_key_off, max_bit_pos, group_member;
3871         struct nix_rx_flowkey_alg *field;
3872         struct nix_rx_flowkey_alg tmp;
3873         u32 key_type, valid_key;
3874         u32 l3_l4_src_dst;
3875         int l4_key_offset = 0;
3876
3877         if (!alg)
3878                 return -EINVAL;
3879
3880 #define FIELDS_PER_ALG  5
3881 #define MAX_KEY_OFF     40
3882         /* Clear all fields */
3883         memset(alg, 0, sizeof(uint64_t) * FIELDS_PER_ALG);
3884
3885         /* Each of the 32 possible flow key algorithm definitions should
3886          * fall into above incremental config (except ALG0). Otherwise a
3887          * single NPC MCAM entry is not sufficient for supporting RSS.
3888          *
3889          * If a different definition or combination needed then NPC MCAM
3890          * has to be programmed to filter such pkts and it's action should
3891          * point to this definition to calculate flowtag or hash.
3892          *
3893          * The `for loop` goes over _all_ protocol field and the following
3894          * variables depicts the state machine forward progress logic.
3895          *
3896          * keyoff_marker - Enabled when hash byte length needs to be accounted
3897          * in field->key_offset update.
3898          * field_marker - Enabled when a new field needs to be selected.
3899          * group_member - Enabled when protocol is part of a group.
3900          */
3901
3902         /* Last 4 bits (31:28) are reserved to specify SRC, DST
3903          * selection for L3, L4 i.e IPV[4,6]_SRC, IPV[4,6]_DST,
3904          * [TCP,UDP,SCTP]_SRC, [TCP,UDP,SCTP]_DST
3905          * 31 => L3_SRC, 30 => L3_DST, 29 => L4_SRC, 28 => L4_DST
3906          */
3907         l3_l4_src_dst = flow_cfg;
3908         /* Reset these 4 bits, so that these won't be part of key */
3909         flow_cfg &= NIX_FLOW_KEY_TYPE_L3_L4_MASK;
3910
3911         keyoff_marker = 0; max_key_off = 0; group_member = 0;
3912         nr_field = 0; key_off = 0; field_marker = 1;
3913         field = &tmp; max_bit_pos = fls(flow_cfg);
3914         for (idx = 0;
3915              idx < max_bit_pos && nr_field < FIELDS_PER_ALG &&
3916              key_off < MAX_KEY_OFF; idx++) {
3917                 key_type = BIT(idx);
3918                 valid_key = flow_cfg & key_type;
3919                 /* Found a field marker, reset the field values */
3920                 if (field_marker)
3921                         memset(&tmp, 0, sizeof(tmp));
3922
3923                 field_marker = true;
3924                 keyoff_marker = true;
3925                 switch (key_type) {
3926                 case NIX_FLOW_KEY_TYPE_PORT:
3927                         field->sel_chan = true;
3928                         /* This should be set to 1, when SEL_CHAN is set */
3929                         field->bytesm1 = 1;
3930                         break;
3931                 case NIX_FLOW_KEY_TYPE_IPV4_PROTO:
3932                         field->lid = NPC_LID_LC;
3933                         field->hdr_offset = 9; /* offset */
3934                         field->bytesm1 = 0; /* 1 byte */
3935                         field->ltype_match = NPC_LT_LC_IP;
3936                         field->ltype_mask = 0xF;
3937                         break;
3938                 case NIX_FLOW_KEY_TYPE_IPV4:
3939                 case NIX_FLOW_KEY_TYPE_INNR_IPV4:
3940                         field->lid = NPC_LID_LC;
3941                         field->ltype_match = NPC_LT_LC_IP;
3942                         if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV4) {
3943                                 field->lid = NPC_LID_LG;
3944                                 field->ltype_match = NPC_LT_LG_TU_IP;
3945                         }
3946                         field->hdr_offset = 12; /* SIP offset */
3947                         field->bytesm1 = 7; /* SIP + DIP, 8 bytes */
3948
3949                         /* Only SIP */
3950                         if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_SRC_ONLY)
3951                                 field->bytesm1 = 3; /* SIP, 4 bytes */
3952
3953                         if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_DST_ONLY) {
3954                                 /* Both SIP + DIP */
3955                                 if (field->bytesm1 == 3) {
3956                                         field->bytesm1 = 7; /* SIP + DIP, 8B */
3957                                 } else {
3958                                         /* Only DIP */
3959                                         field->hdr_offset = 16; /* DIP off */
3960                                         field->bytesm1 = 3; /* DIP, 4 bytes */
3961                                 }
3962                         }
3963
3964                         field->ltype_mask = 0xF; /* Match only IPv4 */
3965                         keyoff_marker = false;
3966                         break;
3967                 case NIX_FLOW_KEY_TYPE_IPV6:
3968                 case NIX_FLOW_KEY_TYPE_INNR_IPV6:
3969                         field->lid = NPC_LID_LC;
3970                         field->ltype_match = NPC_LT_LC_IP6;
3971                         if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV6) {
3972                                 field->lid = NPC_LID_LG;
3973                                 field->ltype_match = NPC_LT_LG_TU_IP6;
3974                         }
3975                         field->hdr_offset = 8; /* SIP offset */
3976                         field->bytesm1 = 31; /* SIP + DIP, 32 bytes */
3977
3978                         /* Only SIP */
3979                         if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_SRC_ONLY)
3980                                 field->bytesm1 = 15; /* SIP, 16 bytes */
3981
3982                         if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L3_DST_ONLY) {
3983                                 /* Both SIP + DIP */
3984                                 if (field->bytesm1 == 15) {
3985                                         /* SIP + DIP, 32 bytes */
3986                                         field->bytesm1 = 31;
3987                                 } else {
3988                                         /* Only DIP */
3989                                         field->hdr_offset = 24; /* DIP off */
3990                                         field->bytesm1 = 15; /* DIP,16 bytes */
3991                                 }
3992                         }
3993                         field->ltype_mask = 0xF; /* Match only IPv6 */
3994                         break;
3995                 case NIX_FLOW_KEY_TYPE_TCP:
3996                 case NIX_FLOW_KEY_TYPE_UDP:
3997                 case NIX_FLOW_KEY_TYPE_SCTP:
3998                 case NIX_FLOW_KEY_TYPE_INNR_TCP:
3999                 case NIX_FLOW_KEY_TYPE_INNR_UDP:
4000                 case NIX_FLOW_KEY_TYPE_INNR_SCTP:
4001                         field->lid = NPC_LID_LD;
4002                         if (key_type == NIX_FLOW_KEY_TYPE_INNR_TCP ||
4003                             key_type == NIX_FLOW_KEY_TYPE_INNR_UDP ||
4004                             key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP)
4005                                 field->lid = NPC_LID_LH;
4006                         field->bytesm1 = 3; /* Sport + Dport, 4 bytes */
4007
4008                         if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L4_SRC_ONLY)
4009                                 field->bytesm1 = 1; /* SRC, 2 bytes */
4010
4011                         if (l3_l4_src_dst & NIX_FLOW_KEY_TYPE_L4_DST_ONLY) {
4012                                 /* Both SRC + DST */
4013                                 if (field->bytesm1 == 1) {
4014                                         /* SRC + DST, 4 bytes */
4015                                         field->bytesm1 = 3;
4016                                 } else {
4017                                         /* Only DIP */
4018                                         field->hdr_offset = 2; /* DST off */
4019                                         field->bytesm1 = 1; /* DST, 2 bytes */
4020                                 }
4021                         }
4022
4023                         /* Enum values for NPC_LID_LD and NPC_LID_LG are same,
4024                          * so no need to change the ltype_match, just change
4025                          * the lid for inner protocols
4026                          */
4027                         BUILD_BUG_ON((int)NPC_LT_LD_TCP !=
4028                                      (int)NPC_LT_LH_TU_TCP);
4029                         BUILD_BUG_ON((int)NPC_LT_LD_UDP !=
4030                                      (int)NPC_LT_LH_TU_UDP);
4031                         BUILD_BUG_ON((int)NPC_LT_LD_SCTP !=
4032                                      (int)NPC_LT_LH_TU_SCTP);
4033
4034                         if ((key_type == NIX_FLOW_KEY_TYPE_TCP ||
4035                              key_type == NIX_FLOW_KEY_TYPE_INNR_TCP) &&
4036                             valid_key) {
4037                                 field->ltype_match |= NPC_LT_LD_TCP;
4038                                 group_member = true;
4039                         } else if ((key_type == NIX_FLOW_KEY_TYPE_UDP ||
4040                                     key_type == NIX_FLOW_KEY_TYPE_INNR_UDP) &&
4041                                    valid_key) {
4042                                 field->ltype_match |= NPC_LT_LD_UDP;
4043                                 group_member = true;
4044                         } else if ((key_type == NIX_FLOW_KEY_TYPE_SCTP ||
4045                                     key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) &&
4046                                    valid_key) {
4047                                 field->ltype_match |= NPC_LT_LD_SCTP;
4048                                 group_member = true;
4049                         }
4050                         field->ltype_mask = ~field->ltype_match;
4051                         if (key_type == NIX_FLOW_KEY_TYPE_SCTP ||
4052                             key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) {
4053                                 /* Handle the case where any of the group item
4054                                  * is enabled in the group but not the final one
4055                                  */
4056                                 if (group_member) {
4057                                         valid_key = true;
4058                                         group_member = false;
4059                                 }
4060                         } else {
4061                                 field_marker = false;
4062                                 keyoff_marker = false;
4063                         }
4064
4065                         /* TCP/UDP/SCTP and ESP/AH falls at same offset so
4066                          * remember the TCP key offset of 40 byte hash key.
4067                          */
4068                         if (key_type == NIX_FLOW_KEY_TYPE_TCP)
4069                                 l4_key_offset = key_off;
4070                         break;
4071                 case NIX_FLOW_KEY_TYPE_NVGRE:
4072                         field->lid = NPC_LID_LD;
4073                         field->hdr_offset = 4; /* VSID offset */
4074                         field->bytesm1 = 2;
4075                         field->ltype_match = NPC_LT_LD_NVGRE;
4076                         field->ltype_mask = 0xF;
4077                         break;
4078                 case NIX_FLOW_KEY_TYPE_VXLAN:
4079                 case NIX_FLOW_KEY_TYPE_GENEVE:
4080                         field->lid = NPC_LID_LE;
4081                         field->bytesm1 = 2;
4082                         field->hdr_offset = 4;
4083                         field->ltype_mask = 0xF;
4084                         field_marker = false;
4085                         keyoff_marker = false;
4086
4087                         if (key_type == NIX_FLOW_KEY_TYPE_VXLAN && valid_key) {
4088                                 field->ltype_match |= NPC_LT_LE_VXLAN;
4089                                 group_member = true;
4090                         }
4091
4092                         if (key_type == NIX_FLOW_KEY_TYPE_GENEVE && valid_key) {
4093                                 field->ltype_match |= NPC_LT_LE_GENEVE;
4094                                 group_member = true;
4095                         }
4096
4097                         if (key_type == NIX_FLOW_KEY_TYPE_GENEVE) {
4098                                 if (group_member) {
4099                                         field->ltype_mask = ~field->ltype_match;
4100                                         field_marker = true;
4101                                         keyoff_marker = true;
4102                                         valid_key = true;
4103                                         group_member = false;
4104                                 }
4105                         }
4106                         break;
4107                 case NIX_FLOW_KEY_TYPE_ETH_DMAC:
4108                 case NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC:
4109                         field->lid = NPC_LID_LA;
4110                         field->ltype_match = NPC_LT_LA_ETHER;
4111                         if (key_type == NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC) {
4112                                 field->lid = NPC_LID_LF;
4113                                 field->ltype_match = NPC_LT_LF_TU_ETHER;
4114                         }
4115                         field->hdr_offset = 0;
4116                         field->bytesm1 = 5; /* DMAC 6 Byte */
4117                         field->ltype_mask = 0xF;
4118                         break;
4119                 case NIX_FLOW_KEY_TYPE_IPV6_EXT:
4120                         field->lid = NPC_LID_LC;
4121                         field->hdr_offset = 40; /* IPV6 hdr */
4122                         field->bytesm1 = 0; /* 1 Byte ext hdr*/
4123                         field->ltype_match = NPC_LT_LC_IP6_EXT;
4124                         field->ltype_mask = 0xF;
4125                         break;
4126                 case NIX_FLOW_KEY_TYPE_GTPU:
4127                         field->lid = NPC_LID_LE;
4128                         field->hdr_offset = 4;
4129                         field->bytesm1 = 3; /* 4 bytes TID*/
4130                         field->ltype_match = NPC_LT_LE_GTPU;
4131                         field->ltype_mask = 0xF;
4132                         break;
4133                 case NIX_FLOW_KEY_TYPE_CUSTOM0:
4134                         field->lid = NPC_LID_LC;
4135                         field->hdr_offset = 6;
4136                         field->bytesm1 = 1; /* 2 Bytes*/
4137                         field->ltype_match = NPC_LT_LC_CUSTOM0;
4138                         field->ltype_mask = 0xF;
4139                         break;
4140                 case NIX_FLOW_KEY_TYPE_VLAN:
4141                         field->lid = NPC_LID_LB;
4142                         field->hdr_offset = 2; /* Skip TPID (2-bytes) */
4143                         field->bytesm1 = 1; /* 2 Bytes (Actually 12 bits) */
4144                         field->ltype_match = NPC_LT_LB_CTAG;
4145                         field->ltype_mask = 0xF;
4146                         field->fn_mask = 1; /* Mask out the first nibble */
4147                         break;
4148                 case NIX_FLOW_KEY_TYPE_AH:
4149                 case NIX_FLOW_KEY_TYPE_ESP:
4150                         field->hdr_offset = 0;
4151                         field->bytesm1 = 7; /* SPI + sequence number */
4152                         field->ltype_mask = 0xF;
4153                         field->lid = NPC_LID_LE;
4154                         field->ltype_match = NPC_LT_LE_ESP;
4155                         if (key_type == NIX_FLOW_KEY_TYPE_AH) {
4156                                 field->lid = NPC_LID_LD;
4157                                 field->ltype_match = NPC_LT_LD_AH;
4158                                 field->hdr_offset = 4;
4159                                 keyoff_marker = false;
4160                         }
4161                         break;
4162                 }
4163                 field->ena = 1;
4164
4165                 /* Found a valid flow key type */
4166                 if (valid_key) {
4167                         /* Use the key offset of TCP/UDP/SCTP fields
4168                          * for ESP/AH fields.
4169                          */
4170                         if (key_type == NIX_FLOW_KEY_TYPE_ESP ||
4171                             key_type == NIX_FLOW_KEY_TYPE_AH)
4172                                 key_off = l4_key_offset;
4173                         field->key_offset = key_off;
4174                         memcpy(&alg[nr_field], field, sizeof(*field));
4175                         max_key_off = max(max_key_off, field->bytesm1 + 1);
4176
4177                         /* Found a field marker, get the next field */
4178                         if (field_marker)
4179                                 nr_field++;
4180                 }
4181
4182                 /* Found a keyoff marker, update the new key_off */
4183                 if (keyoff_marker) {
4184                         key_off += max_key_off;
4185                         max_key_off = 0;
4186                 }
4187         }
4188         /* Processed all the flow key types */
4189         if (idx == max_bit_pos && key_off <= MAX_KEY_OFF)
4190                 return 0;
4191         else
4192                 return NIX_AF_ERR_RSS_NOSPC_FIELD;
4193 }
4194
4195 static int reserve_flowkey_alg_idx(struct rvu *rvu, int blkaddr, u32 flow_cfg)
4196 {
4197         u64 field[FIELDS_PER_ALG];
4198         struct nix_hw *hw;
4199         int fid, rc;
4200
4201         hw = get_nix_hw(rvu->hw, blkaddr);
4202         if (!hw)
4203                 return NIX_AF_ERR_INVALID_NIXBLK;
4204
4205         /* No room to add new flow hash algoritham */
4206         if (hw->flowkey.in_use >= NIX_FLOW_KEY_ALG_MAX)
4207                 return NIX_AF_ERR_RSS_NOSPC_ALGO;
4208
4209         /* Generate algo fields for the given flow_cfg */
4210         rc = set_flowkey_fields((struct nix_rx_flowkey_alg *)field, flow_cfg);
4211         if (rc)
4212                 return rc;
4213
4214         /* Update ALGX_FIELDX register with generated fields */
4215         for (fid = 0; fid < FIELDS_PER_ALG; fid++)
4216                 rvu_write64(rvu, blkaddr,
4217                             NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(hw->flowkey.in_use,
4218                                                            fid), field[fid]);
4219
4220         /* Store the flow_cfg for futher lookup */
4221         rc = hw->flowkey.in_use;
4222         hw->flowkey.flowkey[rc] = flow_cfg;
4223         hw->flowkey.in_use++;
4224
4225         return rc;
4226 }
4227
4228 int rvu_mbox_handler_nix_rss_flowkey_cfg(struct rvu *rvu,
4229                                          struct nix_rss_flowkey_cfg *req,
4230                                          struct nix_rss_flowkey_cfg_rsp *rsp)
4231 {
4232         u16 pcifunc = req->hdr.pcifunc;
4233         int alg_idx, nixlf, blkaddr;
4234         struct nix_hw *nix_hw;
4235         int err;
4236
4237         err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
4238         if (err)
4239                 return err;
4240
4241         nix_hw = get_nix_hw(rvu->hw, blkaddr);
4242         if (!nix_hw)
4243                 return NIX_AF_ERR_INVALID_NIXBLK;
4244
4245         alg_idx = get_flowkey_alg_idx(nix_hw, req->flowkey_cfg);
4246         /* Failed to get algo index from the exiting list, reserve new  */
4247         if (alg_idx < 0) {
4248                 alg_idx = reserve_flowkey_alg_idx(rvu, blkaddr,
4249                                                   req->flowkey_cfg);
4250                 if (alg_idx < 0)
4251                         return alg_idx;
4252         }
4253         rsp->alg_idx = alg_idx;
4254         rvu_npc_update_flowkey_alg_idx(rvu, pcifunc, nixlf, req->group,
4255                                        alg_idx, req->mcam_index);
4256         return 0;
4257 }
4258
4259 static int nix_rx_flowkey_alg_cfg(struct rvu *rvu, int blkaddr)
4260 {
4261         u32 flowkey_cfg, minkey_cfg;
4262         int alg, fid, rc;
4263
4264         /* Disable all flow key algx fieldx */
4265         for (alg = 0; alg < NIX_FLOW_KEY_ALG_MAX; alg++) {
4266                 for (fid = 0; fid < FIELDS_PER_ALG; fid++)
4267                         rvu_write64(rvu, blkaddr,
4268                                     NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(alg, fid),
4269                                     0);
4270         }
4271
4272         /* IPv4/IPv6 SIP/DIPs */
4273         flowkey_cfg = NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6;
4274         rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
4275         if (rc < 0)
4276                 return rc;
4277
4278         /* TCPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
4279         minkey_cfg = flowkey_cfg;
4280         flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP;
4281         rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
4282         if (rc < 0)
4283                 return rc;
4284
4285         /* UDPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
4286         flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP;
4287         rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
4288         if (rc < 0)
4289                 return rc;
4290
4291         /* SCTPv4/v6 4-tuple, SIP, DIP, Sport, Dport */
4292         flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_SCTP;
4293         rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
4294         if (rc < 0)
4295                 return rc;
4296
4297         /* TCP/UDP v4/v6 4-tuple, rest IP pkts 2-tuple */
4298         flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
4299                         NIX_FLOW_KEY_TYPE_UDP;
4300         rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
4301         if (rc < 0)
4302                 return rc;
4303
4304         /* TCP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
4305         flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
4306                         NIX_FLOW_KEY_TYPE_SCTP;
4307         rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
4308         if (rc < 0)
4309                 return rc;
4310
4311         /* UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
4312         flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP |
4313                         NIX_FLOW_KEY_TYPE_SCTP;
4314         rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
4315         if (rc < 0)
4316                 return rc;
4317
4318         /* TCP/UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */
4319         flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP |
4320                       NIX_FLOW_KEY_TYPE_UDP | NIX_FLOW_KEY_TYPE_SCTP;
4321         rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg);
4322         if (rc < 0)
4323                 return rc;
4324
4325         return 0;
4326 }
4327
4328 int rvu_mbox_handler_nix_set_mac_addr(struct rvu *rvu,
4329                                       struct nix_set_mac_addr *req,
4330                                       struct msg_rsp *rsp)
4331 {
4332         bool from_vf = req->hdr.pcifunc & RVU_PFVF_FUNC_MASK;
4333         u16 pcifunc = req->hdr.pcifunc;
4334         int blkaddr, nixlf, err;
4335         struct rvu_pfvf *pfvf;
4336
4337         err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
4338         if (err)
4339                 return err;
4340
4341         pfvf = rvu_get_pfvf(rvu, pcifunc);
4342
4343         /* untrusted VF can't overwrite admin(PF) changes */
4344         if (!test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) &&
4345             (from_vf && test_bit(PF_SET_VF_MAC, &pfvf->flags))) {
4346                 dev_warn(rvu->dev,
4347                          "MAC address set by admin(PF) cannot be overwritten by untrusted VF");
4348                 return -EPERM;
4349         }
4350
4351         ether_addr_copy(pfvf->mac_addr, req->mac_addr);
4352
4353         rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf,
4354                                     pfvf->rx_chan_base, req->mac_addr);
4355
4356         if (test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && from_vf)
4357                 ether_addr_copy(pfvf->default_mac, req->mac_addr);
4358
4359         rvu_switch_update_rules(rvu, pcifunc);
4360
4361         return 0;
4362 }
4363
4364 int rvu_mbox_handler_nix_get_mac_addr(struct rvu *rvu,
4365                                       struct msg_req *req,
4366                                       struct nix_get_mac_addr_rsp *rsp)
4367 {
4368         u16 pcifunc = req->hdr.pcifunc;
4369         struct rvu_pfvf *pfvf;
4370
4371         if (!is_nixlf_attached(rvu, pcifunc))
4372                 return NIX_AF_ERR_AF_LF_INVALID;
4373
4374         pfvf = rvu_get_pfvf(rvu, pcifunc);
4375
4376         ether_addr_copy(rsp->mac_addr, pfvf->mac_addr);
4377
4378         return 0;
4379 }
4380
4381 int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req,
4382                                      struct msg_rsp *rsp)
4383 {
4384         bool allmulti, promisc, nix_rx_multicast;
4385         u16 pcifunc = req->hdr.pcifunc;
4386         struct rvu_pfvf *pfvf;
4387         int nixlf, err;
4388
4389         pfvf = rvu_get_pfvf(rvu, pcifunc);
4390         promisc = req->mode & NIX_RX_MODE_PROMISC ? true : false;
4391         allmulti = req->mode & NIX_RX_MODE_ALLMULTI ? true : false;
4392         pfvf->use_mce_list = req->mode & NIX_RX_MODE_USE_MCE ? true : false;
4393
4394         nix_rx_multicast = rvu->hw->cap.nix_rx_multicast & pfvf->use_mce_list;
4395
4396         if (is_vf(pcifunc) && !nix_rx_multicast &&
4397             (promisc || allmulti)) {
4398                 dev_warn_ratelimited(rvu->dev,
4399                                      "VF promisc/multicast not supported\n");
4400                 return 0;
4401         }
4402
4403         /* untrusted VF can't configure promisc/allmulti */
4404         if (is_vf(pcifunc) && !test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) &&
4405             (promisc || allmulti))
4406                 return 0;
4407
4408         err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
4409         if (err)
4410                 return err;
4411
4412         if (nix_rx_multicast) {
4413                 /* add/del this PF_FUNC to/from mcast pkt replication list */
4414                 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_ALLMULTI_ENTRY,
4415                                           allmulti);
4416                 if (err) {
4417                         dev_err(rvu->dev,
4418                                 "Failed to update pcifunc 0x%x to multicast list\n",
4419                                 pcifunc);
4420                         return err;
4421                 }
4422
4423                 /* add/del this PF_FUNC to/from promisc pkt replication list */
4424                 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_PROMISC_ENTRY,
4425                                           promisc);
4426                 if (err) {
4427                         dev_err(rvu->dev,
4428                                 "Failed to update pcifunc 0x%x to promisc list\n",
4429                                 pcifunc);
4430                         return err;
4431                 }
4432         }
4433
4434         /* install/uninstall allmulti entry */
4435         if (allmulti) {
4436                 rvu_npc_install_allmulti_entry(rvu, pcifunc, nixlf,
4437                                                pfvf->rx_chan_base);
4438         } else {
4439                 if (!nix_rx_multicast)
4440                         rvu_npc_enable_allmulti_entry(rvu, pcifunc, nixlf, false);
4441         }
4442
4443         /* install/uninstall promisc entry */
4444         if (promisc)
4445                 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf,
4446                                               pfvf->rx_chan_base,
4447                                               pfvf->rx_chan_cnt);
4448         else
4449                 if (!nix_rx_multicast)
4450                         rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf, false);
4451
4452         return 0;
4453 }
4454
4455 static void nix_find_link_frs(struct rvu *rvu,
4456                               struct nix_frs_cfg *req, u16 pcifunc)
4457 {
4458         int pf = rvu_get_pf(pcifunc);
4459         struct rvu_pfvf *pfvf;
4460         int maxlen, minlen;
4461         int numvfs, hwvf;
4462         int vf;
4463
4464         /* Update with requester's min/max lengths */
4465         pfvf = rvu_get_pfvf(rvu, pcifunc);
4466         pfvf->maxlen = req->maxlen;
4467         if (req->update_minlen)
4468                 pfvf->minlen = req->minlen;
4469
4470         maxlen = req->maxlen;
4471         minlen = req->update_minlen ? req->minlen : 0;
4472
4473         /* Get this PF's numVFs and starting hwvf */
4474         rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);
4475
4476         /* For each VF, compare requested max/minlen */
4477         for (vf = 0; vf < numvfs; vf++) {
4478                 pfvf =  &rvu->hwvf[hwvf + vf];
4479                 if (pfvf->maxlen > maxlen)
4480                         maxlen = pfvf->maxlen;
4481                 if (req->update_minlen &&
4482                     pfvf->minlen && pfvf->minlen < minlen)
4483                         minlen = pfvf->minlen;
4484         }
4485
4486         /* Compare requested max/minlen with PF's max/minlen */
4487         pfvf = &rvu->pf[pf];
4488         if (pfvf->maxlen > maxlen)
4489                 maxlen = pfvf->maxlen;
4490         if (req->update_minlen &&
4491             pfvf->minlen && pfvf->minlen < minlen)
4492                 minlen = pfvf->minlen;
4493
4494         /* Update the request with max/min PF's and it's VF's max/min */
4495         req->maxlen = maxlen;
4496         if (req->update_minlen)
4497                 req->minlen = minlen;
4498 }
4499
4500 int rvu_mbox_handler_nix_set_hw_frs(struct rvu *rvu, struct nix_frs_cfg *req,
4501                                     struct msg_rsp *rsp)
4502 {
4503         struct rvu_hwinfo *hw = rvu->hw;
4504         u16 pcifunc = req->hdr.pcifunc;
4505         int pf = rvu_get_pf(pcifunc);
4506         int blkaddr, link = -1;
4507         struct nix_hw *nix_hw;
4508         struct rvu_pfvf *pfvf;
4509         u8 cgx = 0, lmac = 0;
4510         u16 max_mtu;
4511         u64 cfg;
4512
4513         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
4514         if (blkaddr < 0)
4515                 return NIX_AF_ERR_AF_LF_INVALID;
4516
4517         nix_hw = get_nix_hw(rvu->hw, blkaddr);
4518         if (!nix_hw)
4519                 return NIX_AF_ERR_INVALID_NIXBLK;
4520
4521         if (is_lbk_vf(rvu, pcifunc))
4522                 rvu_get_lbk_link_max_frs(rvu, &max_mtu);
4523         else
4524                 rvu_get_lmac_link_max_frs(rvu, &max_mtu);
4525
4526         if (!req->sdp_link && req->maxlen > max_mtu)
4527                 return NIX_AF_ERR_FRS_INVALID;
4528
4529         if (req->update_minlen && req->minlen < NIC_HW_MIN_FRS)
4530                 return NIX_AF_ERR_FRS_INVALID;
4531
4532         /* Check if config is for SDP link */
4533         if (req->sdp_link) {
4534                 if (!hw->sdp_links)
4535                         return NIX_AF_ERR_RX_LINK_INVALID;
4536                 link = hw->cgx_links + hw->lbk_links;
4537                 goto linkcfg;
4538         }
4539
4540         /* Check if the request is from CGX mapped RVU PF */
4541         if (is_pf_cgxmapped(rvu, pf)) {
4542                 /* Get CGX and LMAC to which this PF is mapped and find link */
4543                 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx, &lmac);
4544                 link = (cgx * hw->lmac_per_cgx) + lmac;
4545         } else if (pf == 0) {
4546                 /* For VFs of PF0 ingress is LBK port, so config LBK link */
4547                 pfvf = rvu_get_pfvf(rvu, pcifunc);
4548                 link = hw->cgx_links + pfvf->lbkid;
4549         }
4550
4551         if (link < 0)
4552                 return NIX_AF_ERR_RX_LINK_INVALID;
4553
4554 linkcfg:
4555         nix_find_link_frs(rvu, req, pcifunc);
4556
4557         cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link));
4558         cfg = (cfg & ~(0xFFFFULL << 16)) | ((u64)req->maxlen << 16);
4559         if (req->update_minlen)
4560                 cfg = (cfg & ~0xFFFFULL) | req->minlen;
4561         rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), cfg);
4562
4563         return 0;
4564 }
4565
4566 int rvu_mbox_handler_nix_set_rx_cfg(struct rvu *rvu, struct nix_rx_cfg *req,
4567                                     struct msg_rsp *rsp)
4568 {
4569         int nixlf, blkaddr, err;
4570         u64 cfg;
4571
4572         err = nix_get_nixlf(rvu, req->hdr.pcifunc, &nixlf, &blkaddr);
4573         if (err)
4574                 return err;
4575
4576         cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf));
4577         /* Set the interface configuration */
4578         if (req->len_verify & BIT(0))
4579                 cfg |= BIT_ULL(41);
4580         else
4581                 cfg &= ~BIT_ULL(41);
4582
4583         if (req->len_verify & BIT(1))
4584                 cfg |= BIT_ULL(40);
4585         else
4586                 cfg &= ~BIT_ULL(40);
4587
4588         if (req->len_verify & NIX_RX_DROP_RE)
4589                 cfg |= BIT_ULL(32);
4590         else
4591                 cfg &= ~BIT_ULL(32);
4592
4593         if (req->csum_verify & BIT(0))
4594                 cfg |= BIT_ULL(37);
4595         else
4596                 cfg &= ~BIT_ULL(37);
4597
4598         rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), cfg);
4599
4600         return 0;
4601 }
4602
4603 static u64 rvu_get_lbk_link_credits(struct rvu *rvu, u16 lbk_max_frs)
4604 {
4605         return 1600; /* 16 * max LBK datarate = 16 * 100Gbps */
4606 }
4607
4608 static void nix_link_config(struct rvu *rvu, int blkaddr,
4609                             struct nix_hw *nix_hw)
4610 {
4611         struct rvu_hwinfo *hw = rvu->hw;
4612         int cgx, lmac_cnt, slink, link;
4613         u16 lbk_max_frs, lmac_max_frs;
4614         unsigned long lmac_bmap;
4615         u64 tx_credits, cfg;
4616         u64 lmac_fifo_len;
4617         int iter;
4618
4619         rvu_get_lbk_link_max_frs(rvu, &lbk_max_frs);
4620         rvu_get_lmac_link_max_frs(rvu, &lmac_max_frs);
4621
4622         /* Set default min/max packet lengths allowed on NIX Rx links.
4623          *
4624          * With HW reset minlen value of 60byte, HW will treat ARP pkts
4625          * as undersize and report them to SW as error pkts, hence
4626          * setting it to 40 bytes.
4627          */
4628         for (link = 0; link < hw->cgx_links; link++) {
4629                 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link),
4630                                 ((u64)lmac_max_frs << 16) | NIC_HW_MIN_FRS);
4631         }
4632
4633         for (link = hw->cgx_links; link < hw->lbk_links; link++) {
4634                 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link),
4635                             ((u64)lbk_max_frs << 16) | NIC_HW_MIN_FRS);
4636         }
4637         if (hw->sdp_links) {
4638                 link = hw->cgx_links + hw->lbk_links;
4639                 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link),
4640                             SDP_HW_MAX_FRS << 16 | NIC_HW_MIN_FRS);
4641         }
4642
4643         /* Get MCS external bypass status for CN10K-B */
4644         if (mcs_get_blkcnt() == 1) {
4645                 /* Adjust for 2 credits when external bypass is disabled */
4646                 nix_hw->cc_mcs_cnt = is_mcs_bypass(0) ? 0 : 2;
4647         }
4648
4649         /* Set credits for Tx links assuming max packet length allowed.
4650          * This will be reconfigured based on MTU set for PF/VF.
4651          */
4652         for (cgx = 0; cgx < hw->cgx; cgx++) {
4653                 lmac_cnt = cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu));
4654                 /* Skip when cgx is not available or lmac cnt is zero */
4655                 if (lmac_cnt <= 0)
4656                         continue;
4657                 slink = cgx * hw->lmac_per_cgx;
4658
4659                 /* Get LMAC id's from bitmap */
4660                 lmac_bmap = cgx_get_lmac_bmap(rvu_cgx_pdata(cgx, rvu));
4661                 for_each_set_bit(iter, &lmac_bmap, rvu->hw->lmac_per_cgx) {
4662                         lmac_fifo_len = rvu_cgx_get_lmac_fifolen(rvu, cgx, iter);
4663                         if (!lmac_fifo_len) {
4664                                 dev_err(rvu->dev,
4665                                         "%s: Failed to get CGX/RPM%d:LMAC%d FIFO size\n",
4666                                         __func__, cgx, iter);
4667                                 continue;
4668                         }
4669                         tx_credits = (lmac_fifo_len - lmac_max_frs) / 16;
4670                         /* Enable credits and set credit pkt count to max allowed */
4671                         cfg =  (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1);
4672                         cfg |= FIELD_PREP(NIX_AF_LINKX_MCS_CNT_MASK, nix_hw->cc_mcs_cnt);
4673
4674                         link = iter + slink;
4675                         nix_hw->tx_credits[link] = tx_credits;
4676                         rvu_write64(rvu, blkaddr,
4677                                     NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg);
4678                 }
4679         }
4680
4681         /* Set Tx credits for LBK link */
4682         slink = hw->cgx_links;
4683         for (link = slink; link < (slink + hw->lbk_links); link++) {
4684                 tx_credits = rvu_get_lbk_link_credits(rvu, lbk_max_frs);
4685                 nix_hw->tx_credits[link] = tx_credits;
4686                 /* Enable credits and set credit pkt count to max allowed */
4687                 tx_credits =  (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1);
4688                 rvu_write64(rvu, blkaddr,
4689                             NIX_AF_TX_LINKX_NORM_CREDIT(link), tx_credits);
4690         }
4691 }
4692
4693 static int nix_calibrate_x2p(struct rvu *rvu, int blkaddr)
4694 {
4695         int idx, err;
4696         u64 status;
4697
4698         /* Start X2P bus calibration */
4699         rvu_write64(rvu, blkaddr, NIX_AF_CFG,
4700                     rvu_read64(rvu, blkaddr, NIX_AF_CFG) | BIT_ULL(9));
4701         /* Wait for calibration to complete */
4702         err = rvu_poll_reg(rvu, blkaddr,
4703                            NIX_AF_STATUS, BIT_ULL(10), false);
4704         if (err) {
4705                 dev_err(rvu->dev, "NIX X2P bus calibration failed\n");
4706                 return err;
4707         }
4708
4709         status = rvu_read64(rvu, blkaddr, NIX_AF_STATUS);
4710         /* Check if CGX devices are ready */
4711         for (idx = 0; idx < rvu->cgx_cnt_max; idx++) {
4712                 /* Skip when cgx port is not available */
4713                 if (!rvu_cgx_pdata(idx, rvu) ||
4714                     (status & (BIT_ULL(16 + idx))))
4715                         continue;
4716                 dev_err(rvu->dev,
4717                         "CGX%d didn't respond to NIX X2P calibration\n", idx);
4718                 err = -EBUSY;
4719         }
4720
4721         /* Check if LBK is ready */
4722         if (!(status & BIT_ULL(19))) {
4723                 dev_err(rvu->dev,
4724                         "LBK didn't respond to NIX X2P calibration\n");
4725                 err = -EBUSY;
4726         }
4727
4728         /* Clear 'calibrate_x2p' bit */
4729         rvu_write64(rvu, blkaddr, NIX_AF_CFG,
4730                     rvu_read64(rvu, blkaddr, NIX_AF_CFG) & ~BIT_ULL(9));
4731         if (err || (status & 0x3FFULL))
4732                 dev_err(rvu->dev,
4733                         "NIX X2P calibration failed, status 0x%llx\n", status);
4734         if (err)
4735                 return err;
4736         return 0;
4737 }
4738
4739 static int nix_aq_init(struct rvu *rvu, struct rvu_block *block)
4740 {
4741         u64 cfg;
4742         int err;
4743
4744         /* Set admin queue endianness */
4745         cfg = rvu_read64(rvu, block->addr, NIX_AF_CFG);
4746 #ifdef __BIG_ENDIAN
4747         cfg |= BIT_ULL(8);
4748         rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg);
4749 #else
4750         cfg &= ~BIT_ULL(8);
4751         rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg);
4752 #endif
4753
4754         /* Do not bypass NDC cache */
4755         cfg = rvu_read64(rvu, block->addr, NIX_AF_NDC_CFG);
4756         cfg &= ~0x3FFEULL;
4757 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
4758         /* Disable caching of SQB aka SQEs */
4759         cfg |= 0x04ULL;
4760 #endif
4761         rvu_write64(rvu, block->addr, NIX_AF_NDC_CFG, cfg);
4762
4763         /* Result structure can be followed by RQ/SQ/CQ context at
4764          * RES + 128bytes and a write mask at RES + 256 bytes, depending on
4765          * operation type. Alloc sufficient result memory for all operations.
4766          */
4767         err = rvu_aq_alloc(rvu, &block->aq,
4768                            Q_COUNT(AQ_SIZE), sizeof(struct nix_aq_inst_s),
4769                            ALIGN(sizeof(struct nix_aq_res_s), 128) + 256);
4770         if (err)
4771                 return err;
4772
4773         rvu_write64(rvu, block->addr, NIX_AF_AQ_CFG, AQ_SIZE);
4774         rvu_write64(rvu, block->addr,
4775                     NIX_AF_AQ_BASE, (u64)block->aq->inst->iova);
4776         return 0;
4777 }
4778
4779 static void rvu_nix_setup_capabilities(struct rvu *rvu, int blkaddr)
4780 {
4781         struct rvu_hwinfo *hw = rvu->hw;
4782         u64 hw_const;
4783
4784         hw_const = rvu_read64(rvu, blkaddr, NIX_AF_CONST1);
4785
4786         /* On OcteonTx2 DWRR quantum is directly configured into each of
4787          * the transmit scheduler queues. And PF/VF drivers were free to
4788          * config any value upto 2^24.
4789          * On CN10K, HW is modified, the quantum configuration at scheduler
4790          * queues is in terms of weight. And SW needs to setup a base DWRR MTU
4791          * at NIX_AF_DWRR_RPM_MTU / NIX_AF_DWRR_SDP_MTU. HW will do
4792          * 'DWRR MTU * weight' to get the quantum.
4793          *
4794          * Check if HW uses a common MTU for all DWRR quantum configs.
4795          * On OcteonTx2 this register field is '0'.
4796          */
4797         if ((((hw_const >> 56) & 0x10) == 0x10) && !(hw_const & BIT_ULL(61)))
4798                 hw->cap.nix_common_dwrr_mtu = true;
4799
4800         if (hw_const & BIT_ULL(61))
4801                 hw->cap.nix_multiple_dwrr_mtu = true;
4802 }
4803
4804 static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw)
4805 {
4806         const struct npc_lt_def_cfg *ltdefs;
4807         struct rvu_hwinfo *hw = rvu->hw;
4808         int blkaddr = nix_hw->blkaddr;
4809         struct rvu_block *block;
4810         int err;
4811         u64 cfg;
4812
4813         block = &hw->block[blkaddr];
4814
4815         if (is_rvu_96xx_B0(rvu)) {
4816                 /* As per a HW errata in 96xx A0/B0 silicon, NIX may corrupt
4817                  * internal state when conditional clocks are turned off.
4818                  * Hence enable them.
4819                  */
4820                 rvu_write64(rvu, blkaddr, NIX_AF_CFG,
4821                             rvu_read64(rvu, blkaddr, NIX_AF_CFG) | 0x40ULL);
4822
4823                 /* Set chan/link to backpressure TL3 instead of TL2 */
4824                 rvu_write64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL, 0x01);
4825
4826                 /* Disable SQ manager's sticky mode operation (set TM6 = 0)
4827                  * This sticky mode is known to cause SQ stalls when multiple
4828                  * SQs are mapped to same SMQ and transmitting pkts at a time.
4829                  */
4830                 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS);
4831                 cfg &= ~BIT_ULL(15);
4832                 rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg);
4833         }
4834
4835         ltdefs = rvu->kpu.lt_def;
4836         /* Calibrate X2P bus to check if CGX/LBK links are fine */
4837         err = nix_calibrate_x2p(rvu, blkaddr);
4838         if (err)
4839                 return err;
4840
4841         /* Setup capabilities of the NIX block */
4842         rvu_nix_setup_capabilities(rvu, blkaddr);
4843
4844         /* Initialize admin queue */
4845         err = nix_aq_init(rvu, block);
4846         if (err)
4847                 return err;
4848
4849         /* Restore CINT timer delay to HW reset values */
4850         rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL);
4851
4852         cfg = rvu_read64(rvu, blkaddr, NIX_AF_SEB_CFG);
4853
4854         /* For better performance use NDC TX instead of NDC RX for SQ's SQEs" */
4855         cfg |= 1ULL;
4856         if (!is_rvu_otx2(rvu))
4857                 cfg |= NIX_PTP_1STEP_EN;
4858
4859         rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, cfg);
4860
4861         if (!is_rvu_otx2(rvu))
4862                 rvu_nix_block_cn10k_init(rvu, nix_hw);
4863
4864         if (is_block_implemented(hw, blkaddr)) {
4865                 err = nix_setup_txschq(rvu, nix_hw, blkaddr);
4866                 if (err)
4867                         return err;
4868
4869                 err = nix_setup_ipolicers(rvu, nix_hw, blkaddr);
4870                 if (err)
4871                         return err;
4872
4873                 err = nix_af_mark_format_setup(rvu, nix_hw, blkaddr);
4874                 if (err)
4875                         return err;
4876
4877                 err = nix_setup_mcast(rvu, nix_hw, blkaddr);
4878                 if (err)
4879                         return err;
4880
4881                 err = nix_setup_txvlan(rvu, nix_hw);
4882                 if (err)
4883                         return err;
4884
4885                 err = nix_setup_bpids(rvu, nix_hw, blkaddr);
4886                 if (err)
4887                         return err;
4888
4889                 /* Configure segmentation offload formats */
4890                 nix_setup_lso(rvu, nix_hw, blkaddr);
4891
4892                 /* Config Outer/Inner L2, IP, TCP, UDP and SCTP NPC layer info.
4893                  * This helps HW protocol checker to identify headers
4894                  * and validate length and checksums.
4895                  */
4896                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OL2,
4897                             (ltdefs->rx_ol2.lid << 8) | (ltdefs->rx_ol2.ltype_match << 4) |
4898                             ltdefs->rx_ol2.ltype_mask);
4899                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4,
4900                             (ltdefs->rx_oip4.lid << 8) | (ltdefs->rx_oip4.ltype_match << 4) |
4901                             ltdefs->rx_oip4.ltype_mask);
4902                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4,
4903                             (ltdefs->rx_iip4.lid << 8) | (ltdefs->rx_iip4.ltype_match << 4) |
4904                             ltdefs->rx_iip4.ltype_mask);
4905                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6,
4906                             (ltdefs->rx_oip6.lid << 8) | (ltdefs->rx_oip6.ltype_match << 4) |
4907                             ltdefs->rx_oip6.ltype_mask);
4908                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6,
4909                             (ltdefs->rx_iip6.lid << 8) | (ltdefs->rx_iip6.ltype_match << 4) |
4910                             ltdefs->rx_iip6.ltype_mask);
4911                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OTCP,
4912                             (ltdefs->rx_otcp.lid << 8) | (ltdefs->rx_otcp.ltype_match << 4) |
4913                             ltdefs->rx_otcp.ltype_mask);
4914                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ITCP,
4915                             (ltdefs->rx_itcp.lid << 8) | (ltdefs->rx_itcp.ltype_match << 4) |
4916                             ltdefs->rx_itcp.ltype_mask);
4917                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OUDP,
4918                             (ltdefs->rx_oudp.lid << 8) | (ltdefs->rx_oudp.ltype_match << 4) |
4919                             ltdefs->rx_oudp.ltype_mask);
4920                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IUDP,
4921                             (ltdefs->rx_iudp.lid << 8) | (ltdefs->rx_iudp.ltype_match << 4) |
4922                             ltdefs->rx_iudp.ltype_mask);
4923                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OSCTP,
4924                             (ltdefs->rx_osctp.lid << 8) | (ltdefs->rx_osctp.ltype_match << 4) |
4925                             ltdefs->rx_osctp.ltype_mask);
4926                 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ISCTP,
4927                             (ltdefs->rx_isctp.lid << 8) | (ltdefs->rx_isctp.ltype_match << 4) |
4928                             ltdefs->rx_isctp.ltype_mask);
4929
4930                 if (!is_rvu_otx2(rvu)) {
4931                         /* Enable APAD calculation for other protocols
4932                          * matching APAD0 and APAD1 lt def registers.
4933                          */
4934                         rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD0,
4935                                     (ltdefs->rx_apad0.valid << 11) |
4936                                     (ltdefs->rx_apad0.lid << 8) |
4937                                     (ltdefs->rx_apad0.ltype_match << 4) |
4938                                     ltdefs->rx_apad0.ltype_mask);
4939                         rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD1,
4940                                     (ltdefs->rx_apad1.valid << 11) |
4941                                     (ltdefs->rx_apad1.lid << 8) |
4942                                     (ltdefs->rx_apad1.ltype_match << 4) |
4943                                     ltdefs->rx_apad1.ltype_mask);
4944
4945                         /* Receive ethertype defination register defines layer
4946                          * information in NPC_RESULT_S to identify the Ethertype
4947                          * location in L2 header. Used for Ethertype overwriting
4948                          * in inline IPsec flow.
4949                          */
4950                         rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(0),
4951                                     (ltdefs->rx_et[0].offset << 12) |
4952                                     (ltdefs->rx_et[0].valid << 11) |
4953                                     (ltdefs->rx_et[0].lid << 8) |
4954                                     (ltdefs->rx_et[0].ltype_match << 4) |
4955                                     ltdefs->rx_et[0].ltype_mask);
4956                         rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(1),
4957                                     (ltdefs->rx_et[1].offset << 12) |
4958                                     (ltdefs->rx_et[1].valid << 11) |
4959                                     (ltdefs->rx_et[1].lid << 8) |
4960                                     (ltdefs->rx_et[1].ltype_match << 4) |
4961                                     ltdefs->rx_et[1].ltype_mask);
4962                 }
4963
4964                 err = nix_rx_flowkey_alg_cfg(rvu, blkaddr);
4965                 if (err)
4966                         return err;
4967
4968                 nix_hw->tx_credits = kcalloc(hw->cgx_links + hw->lbk_links,
4969                                              sizeof(u64), GFP_KERNEL);
4970                 if (!nix_hw->tx_credits)
4971                         return -ENOMEM;
4972
4973                 /* Initialize CGX/LBK/SDP link credits, min/max pkt lengths */
4974                 nix_link_config(rvu, blkaddr, nix_hw);
4975
4976                 /* Enable Channel backpressure */
4977                 rvu_write64(rvu, blkaddr, NIX_AF_RX_CFG, BIT_ULL(0));
4978         }
4979         return 0;
4980 }
4981
4982 int rvu_nix_init(struct rvu *rvu)
4983 {
4984         struct rvu_hwinfo *hw = rvu->hw;
4985         struct nix_hw *nix_hw;
4986         int blkaddr = 0, err;
4987         int i = 0;
4988
4989         hw->nix = devm_kcalloc(rvu->dev, MAX_NIX_BLKS, sizeof(struct nix_hw),
4990                                GFP_KERNEL);
4991         if (!hw->nix)
4992                 return -ENOMEM;
4993
4994         blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
4995         while (blkaddr) {
4996                 nix_hw = &hw->nix[i];
4997                 nix_hw->rvu = rvu;
4998                 nix_hw->blkaddr = blkaddr;
4999                 err = rvu_nix_block_init(rvu, nix_hw);
5000                 if (err)
5001                         return err;
5002                 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
5003                 i++;
5004         }
5005
5006         return 0;
5007 }
5008
5009 static void rvu_nix_block_freemem(struct rvu *rvu, int blkaddr,
5010                                   struct rvu_block *block)
5011 {
5012         struct nix_txsch *txsch;
5013         struct nix_mcast *mcast;
5014         struct nix_txvlan *vlan;
5015         struct nix_hw *nix_hw;
5016         int lvl;
5017
5018         rvu_aq_free(rvu, block->aq);
5019
5020         if (is_block_implemented(rvu->hw, blkaddr)) {
5021                 nix_hw = get_nix_hw(rvu->hw, blkaddr);
5022                 if (!nix_hw)
5023                         return;
5024
5025                 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
5026                         txsch = &nix_hw->txsch[lvl];
5027                         kfree(txsch->schq.bmap);
5028                 }
5029
5030                 kfree(nix_hw->tx_credits);
5031
5032                 nix_ipolicer_freemem(rvu, nix_hw);
5033
5034                 vlan = &nix_hw->txvlan;
5035                 kfree(vlan->rsrc.bmap);
5036                 mutex_destroy(&vlan->rsrc_lock);
5037
5038                 mcast = &nix_hw->mcast;
5039                 qmem_free(rvu->dev, mcast->mce_ctx);
5040                 qmem_free(rvu->dev, mcast->mcast_buf);
5041                 mutex_destroy(&mcast->mce_lock);
5042         }
5043 }
5044
5045 void rvu_nix_freemem(struct rvu *rvu)
5046 {
5047         struct rvu_hwinfo *hw = rvu->hw;
5048         struct rvu_block *block;
5049         int blkaddr = 0;
5050
5051         blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
5052         while (blkaddr) {
5053                 block = &hw->block[blkaddr];
5054                 rvu_nix_block_freemem(rvu, blkaddr, block);
5055                 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
5056         }
5057 }
5058
5059 static void nix_mcast_update_action(struct rvu *rvu,
5060                                     struct nix_mcast_grp_elem *elem)
5061 {
5062         struct npc_mcam *mcam = &rvu->hw->mcam;
5063         struct nix_rx_action rx_action = { 0 };
5064         struct nix_tx_action tx_action = { 0 };
5065         int npc_blkaddr;
5066
5067         npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
5068         if (elem->dir == NIX_MCAST_INGRESS) {
5069                 *(u64 *)&rx_action = npc_get_mcam_action(rvu, mcam,
5070                                                          npc_blkaddr,
5071                                                          elem->mcam_index);
5072                 rx_action.index = elem->mce_start_index;
5073                 npc_set_mcam_action(rvu, mcam, npc_blkaddr, elem->mcam_index,
5074                                     *(u64 *)&rx_action);
5075         } else {
5076                 *(u64 *)&tx_action = npc_get_mcam_action(rvu, mcam,
5077                                                          npc_blkaddr,
5078                                                          elem->mcam_index);
5079                 tx_action.index = elem->mce_start_index;
5080                 npc_set_mcam_action(rvu, mcam, npc_blkaddr, elem->mcam_index,
5081                                     *(u64 *)&tx_action);
5082         }
5083 }
5084
5085 static void nix_mcast_update_mce_entry(struct rvu *rvu, u16 pcifunc, u8 is_active)
5086 {
5087         struct nix_mcast_grp_elem *elem;
5088         struct nix_mcast_grp *mcast_grp;
5089         struct nix_hw *nix_hw;
5090         int blkaddr;
5091
5092         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
5093         nix_hw = get_nix_hw(rvu->hw, blkaddr);
5094         if (!nix_hw)
5095                 return;
5096
5097         mcast_grp = &nix_hw->mcast_grp;
5098
5099         mutex_lock(&mcast_grp->mcast_grp_lock);
5100         list_for_each_entry(elem, &mcast_grp->mcast_grp_head, list) {
5101                 struct nix_mce_list *mce_list;
5102                 struct mce *mce;
5103
5104                 /* Iterate the group elements and disable the element which
5105                  * received the disable request.
5106                  */
5107                 mce_list = &elem->mcast_mce_list;
5108                 hlist_for_each_entry(mce, &mce_list->head, node) {
5109                         if (mce->pcifunc == pcifunc) {
5110                                 mce->is_active = is_active;
5111                                 break;
5112                         }
5113                 }
5114
5115                 /* Dump the updated list to HW */
5116                 if (elem->dir == NIX_MCAST_INGRESS)
5117                         nix_update_ingress_mce_list_hw(rvu, nix_hw, elem);
5118                 else
5119                         nix_update_egress_mce_list_hw(rvu, nix_hw, elem);
5120
5121                 /* Update the multicast index in NPC rule */
5122                 nix_mcast_update_action(rvu, elem);
5123         }
5124         mutex_unlock(&mcast_grp->mcast_grp_lock);
5125 }
5126
5127 int rvu_mbox_handler_nix_lf_start_rx(struct rvu *rvu, struct msg_req *req,
5128                                      struct msg_rsp *rsp)
5129 {
5130         u16 pcifunc = req->hdr.pcifunc;
5131         struct rvu_pfvf *pfvf;
5132         int nixlf, err;
5133
5134         err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
5135         if (err)
5136                 return err;
5137
5138         /* Enable the interface if it is in any multicast list */
5139         nix_mcast_update_mce_entry(rvu, pcifunc, 1);
5140
5141         rvu_npc_enable_default_entries(rvu, pcifunc, nixlf);
5142
5143         npc_mcam_enable_flows(rvu, pcifunc);
5144
5145         pfvf = rvu_get_pfvf(rvu, pcifunc);
5146         set_bit(NIXLF_INITIALIZED, &pfvf->flags);
5147
5148         rvu_switch_update_rules(rvu, pcifunc);
5149
5150         return rvu_cgx_start_stop_io(rvu, pcifunc, true);
5151 }
5152
5153 int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req,
5154                                     struct msg_rsp *rsp)
5155 {
5156         u16 pcifunc = req->hdr.pcifunc;
5157         struct rvu_pfvf *pfvf;
5158         int nixlf, err;
5159
5160         err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
5161         if (err)
5162                 return err;
5163
5164         rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
5165         /* Disable the interface if it is in any multicast list */
5166         nix_mcast_update_mce_entry(rvu, pcifunc, 0);
5167
5168
5169         pfvf = rvu_get_pfvf(rvu, pcifunc);
5170         clear_bit(NIXLF_INITIALIZED, &pfvf->flags);
5171
5172         err = rvu_cgx_start_stop_io(rvu, pcifunc, false);
5173         if (err)
5174                 return err;
5175
5176         rvu_cgx_tx_enable(rvu, pcifunc, true);
5177
5178         return 0;
5179 }
5180
5181 #define RX_SA_BASE  GENMASK_ULL(52, 7)
5182
5183 void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf)
5184 {
5185         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
5186         struct hwctx_disable_req ctx_req;
5187         int pf = rvu_get_pf(pcifunc);
5188         struct mac_ops *mac_ops;
5189         u8 cgx_id, lmac_id;
5190         u64 sa_base;
5191         void *cgxd;
5192         int err;
5193
5194         ctx_req.hdr.pcifunc = pcifunc;
5195
5196         /* Cleanup NPC MCAM entries, free Tx scheduler queues being used */
5197         rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf);
5198         rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf);
5199         nix_interface_deinit(rvu, pcifunc, nixlf);
5200         nix_rx_sync(rvu, blkaddr);
5201         nix_txschq_free(rvu, pcifunc);
5202
5203         clear_bit(NIXLF_INITIALIZED, &pfvf->flags);
5204
5205         rvu_cgx_start_stop_io(rvu, pcifunc, false);
5206
5207         if (pfvf->sq_ctx) {
5208                 ctx_req.ctype = NIX_AQ_CTYPE_SQ;
5209                 err = nix_lf_hwctx_disable(rvu, &ctx_req);
5210                 if (err)
5211                         dev_err(rvu->dev, "SQ ctx disable failed\n");
5212         }
5213
5214         if (pfvf->rq_ctx) {
5215                 ctx_req.ctype = NIX_AQ_CTYPE_RQ;
5216                 err = nix_lf_hwctx_disable(rvu, &ctx_req);
5217                 if (err)
5218                         dev_err(rvu->dev, "RQ ctx disable failed\n");
5219         }
5220
5221         if (pfvf->cq_ctx) {
5222                 ctx_req.ctype = NIX_AQ_CTYPE_CQ;
5223                 err = nix_lf_hwctx_disable(rvu, &ctx_req);
5224                 if (err)
5225                         dev_err(rvu->dev, "CQ ctx disable failed\n");
5226         }
5227
5228         /* reset HW config done for Switch headers */
5229         rvu_npc_set_parse_mode(rvu, pcifunc, OTX2_PRIV_FLAGS_DEFAULT,
5230                                (PKIND_TX | PKIND_RX), 0, 0, 0, 0);
5231
5232         /* Disabling CGX and NPC config done for PTP */
5233         if (pfvf->hw_rx_tstamp_en) {
5234                 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
5235                 cgxd = rvu_cgx_pdata(cgx_id, rvu);
5236                 mac_ops = get_mac_ops(cgxd);
5237                 mac_ops->mac_enadis_ptp_config(cgxd, lmac_id, false);
5238                 /* Undo NPC config done for PTP */
5239                 if (npc_config_ts_kpuaction(rvu, pf, pcifunc, false))
5240                         dev_err(rvu->dev, "NPC config for PTP failed\n");
5241                 pfvf->hw_rx_tstamp_en = false;
5242         }
5243
5244         /* reset priority flow control config */
5245         rvu_cgx_prio_flow_ctrl_cfg(rvu, pcifunc, 0, 0, 0);
5246
5247         /* reset 802.3x flow control config */
5248         rvu_cgx_cfg_pause_frm(rvu, pcifunc, 0, 0);
5249
5250         nix_ctx_free(rvu, pfvf);
5251
5252         nix_free_all_bandprof(rvu, pcifunc);
5253
5254         sa_base = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(nixlf));
5255         if (FIELD_GET(RX_SA_BASE, sa_base)) {
5256                 err = rvu_cpt_ctx_flush(rvu, pcifunc);
5257                 if (err)
5258                         dev_err(rvu->dev,
5259                                 "CPT ctx flush failed with error: %d\n", err);
5260         }
5261 }
5262
5263 #define NIX_AF_LFX_TX_CFG_PTP_EN        BIT_ULL(32)
5264
5265 static int rvu_nix_lf_ptp_tx_cfg(struct rvu *rvu, u16 pcifunc, bool enable)
5266 {
5267         struct rvu_hwinfo *hw = rvu->hw;
5268         struct rvu_block *block;
5269         int blkaddr, pf;
5270         int nixlf;
5271         u64 cfg;
5272
5273         pf = rvu_get_pf(pcifunc);
5274         if (!is_mac_feature_supported(rvu, pf, RVU_LMAC_FEAT_PTP))
5275                 return 0;
5276
5277         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
5278         if (blkaddr < 0)
5279                 return NIX_AF_ERR_AF_LF_INVALID;
5280
5281         block = &hw->block[blkaddr];
5282         nixlf = rvu_get_lf(rvu, block, pcifunc, 0);
5283         if (nixlf < 0)
5284                 return NIX_AF_ERR_AF_LF_INVALID;
5285
5286         cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf));
5287
5288         if (enable)
5289                 cfg |= NIX_AF_LFX_TX_CFG_PTP_EN;
5290         else
5291                 cfg &= ~NIX_AF_LFX_TX_CFG_PTP_EN;
5292
5293         rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg);
5294
5295         return 0;
5296 }
5297
5298 int rvu_mbox_handler_nix_lf_ptp_tx_enable(struct rvu *rvu, struct msg_req *req,
5299                                           struct msg_rsp *rsp)
5300 {
5301         return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, true);
5302 }
5303
5304 int rvu_mbox_handler_nix_lf_ptp_tx_disable(struct rvu *rvu, struct msg_req *req,
5305                                            struct msg_rsp *rsp)
5306 {
5307         return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, false);
5308 }
5309
5310 int rvu_mbox_handler_nix_lso_format_cfg(struct rvu *rvu,
5311                                         struct nix_lso_format_cfg *req,
5312                                         struct nix_lso_format_cfg_rsp *rsp)
5313 {
5314         u16 pcifunc = req->hdr.pcifunc;
5315         struct nix_hw *nix_hw;
5316         struct rvu_pfvf *pfvf;
5317         int blkaddr, idx, f;
5318         u64 reg;
5319
5320         pfvf = rvu_get_pfvf(rvu, pcifunc);
5321         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
5322         if (!pfvf->nixlf || blkaddr < 0)
5323                 return NIX_AF_ERR_AF_LF_INVALID;
5324
5325         nix_hw = get_nix_hw(rvu->hw, blkaddr);
5326         if (!nix_hw)
5327                 return NIX_AF_ERR_INVALID_NIXBLK;
5328
5329         /* Find existing matching LSO format, if any */
5330         for (idx = 0; idx < nix_hw->lso.in_use; idx++) {
5331                 for (f = 0; f < NIX_LSO_FIELD_MAX; f++) {
5332                         reg = rvu_read64(rvu, blkaddr,
5333                                          NIX_AF_LSO_FORMATX_FIELDX(idx, f));
5334                         if (req->fields[f] != (reg & req->field_mask))
5335                                 break;
5336                 }
5337
5338                 if (f == NIX_LSO_FIELD_MAX)
5339                         break;
5340         }
5341
5342         if (idx < nix_hw->lso.in_use) {
5343                 /* Match found */
5344                 rsp->lso_format_idx = idx;
5345                 return 0;
5346         }
5347
5348         if (nix_hw->lso.in_use == nix_hw->lso.total)
5349                 return NIX_AF_ERR_LSO_CFG_FAIL;
5350
5351         rsp->lso_format_idx = nix_hw->lso.in_use++;
5352
5353         for (f = 0; f < NIX_LSO_FIELD_MAX; f++)
5354                 rvu_write64(rvu, blkaddr,
5355                             NIX_AF_LSO_FORMATX_FIELDX(rsp->lso_format_idx, f),
5356                             req->fields[f]);
5357
5358         return 0;
5359 }
5360
5361 #define IPSEC_GEN_CFG_EGRP    GENMASK_ULL(50, 48)
5362 #define IPSEC_GEN_CFG_OPCODE  GENMASK_ULL(47, 32)
5363 #define IPSEC_GEN_CFG_PARAM1  GENMASK_ULL(31, 16)
5364 #define IPSEC_GEN_CFG_PARAM2  GENMASK_ULL(15, 0)
5365
5366 #define CPT_INST_QSEL_BLOCK   GENMASK_ULL(28, 24)
5367 #define CPT_INST_QSEL_PF_FUNC GENMASK_ULL(23, 8)
5368 #define CPT_INST_QSEL_SLOT    GENMASK_ULL(7, 0)
5369
5370 #define CPT_INST_CREDIT_TH    GENMASK_ULL(53, 32)
5371 #define CPT_INST_CREDIT_BPID  GENMASK_ULL(30, 22)
5372 #define CPT_INST_CREDIT_CNT   GENMASK_ULL(21, 0)
5373
5374 static void nix_inline_ipsec_cfg(struct rvu *rvu, struct nix_inline_ipsec_cfg *req,
5375                                  int blkaddr)
5376 {
5377         u8 cpt_idx, cpt_blkaddr;
5378         u64 val;
5379
5380         cpt_idx = (blkaddr == BLKADDR_NIX0) ? 0 : 1;
5381         if (req->enable) {
5382                 val = 0;
5383                 /* Enable context prefetching */
5384                 if (!is_rvu_otx2(rvu))
5385                         val |= BIT_ULL(51);
5386
5387                 /* Set OPCODE and EGRP */
5388                 val |= FIELD_PREP(IPSEC_GEN_CFG_EGRP, req->gen_cfg.egrp);
5389                 val |= FIELD_PREP(IPSEC_GEN_CFG_OPCODE, req->gen_cfg.opcode);
5390                 val |= FIELD_PREP(IPSEC_GEN_CFG_PARAM1, req->gen_cfg.param1);
5391                 val |= FIELD_PREP(IPSEC_GEN_CFG_PARAM2, req->gen_cfg.param2);
5392
5393                 rvu_write64(rvu, blkaddr, NIX_AF_RX_IPSEC_GEN_CFG, val);
5394
5395                 /* Set CPT queue for inline IPSec */
5396                 val = FIELD_PREP(CPT_INST_QSEL_SLOT, req->inst_qsel.cpt_slot);
5397                 val |= FIELD_PREP(CPT_INST_QSEL_PF_FUNC,
5398                                   req->inst_qsel.cpt_pf_func);
5399
5400                 if (!is_rvu_otx2(rvu)) {
5401                         cpt_blkaddr = (cpt_idx == 0) ? BLKADDR_CPT0 :
5402                                                        BLKADDR_CPT1;
5403                         val |= FIELD_PREP(CPT_INST_QSEL_BLOCK, cpt_blkaddr);
5404                 }
5405
5406                 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_INST_QSEL(cpt_idx),
5407                             val);
5408
5409                 /* Set CPT credit */
5410                 val = rvu_read64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx));
5411                 if ((val & 0x3FFFFF) != 0x3FFFFF)
5412                         rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx),
5413                                     0x3FFFFF - val);
5414
5415                 val = FIELD_PREP(CPT_INST_CREDIT_CNT, req->cpt_credit);
5416                 val |= FIELD_PREP(CPT_INST_CREDIT_BPID, req->bpid);
5417                 val |= FIELD_PREP(CPT_INST_CREDIT_TH, req->credit_th);
5418                 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), val);
5419         } else {
5420                 rvu_write64(rvu, blkaddr, NIX_AF_RX_IPSEC_GEN_CFG, 0x0);
5421                 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_INST_QSEL(cpt_idx),
5422                             0x0);
5423                 val = rvu_read64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx));
5424                 if ((val & 0x3FFFFF) != 0x3FFFFF)
5425                         rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx),
5426                                     0x3FFFFF - val);
5427         }
5428 }
5429
5430 int rvu_mbox_handler_nix_inline_ipsec_cfg(struct rvu *rvu,
5431                                           struct nix_inline_ipsec_cfg *req,
5432                                           struct msg_rsp *rsp)
5433 {
5434         if (!is_block_implemented(rvu->hw, BLKADDR_CPT0))
5435                 return 0;
5436
5437         nix_inline_ipsec_cfg(rvu, req, BLKADDR_NIX0);
5438         if (is_block_implemented(rvu->hw, BLKADDR_CPT1))
5439                 nix_inline_ipsec_cfg(rvu, req, BLKADDR_NIX1);
5440
5441         return 0;
5442 }
5443
5444 int rvu_mbox_handler_nix_read_inline_ipsec_cfg(struct rvu *rvu,
5445                                                struct msg_req *req,
5446                                                struct nix_inline_ipsec_cfg *rsp)
5447
5448 {
5449         u64 val;
5450
5451         if (!is_block_implemented(rvu->hw, BLKADDR_CPT0))
5452                 return 0;
5453
5454         val = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_RX_IPSEC_GEN_CFG);
5455         rsp->gen_cfg.egrp = FIELD_GET(IPSEC_GEN_CFG_EGRP, val);
5456         rsp->gen_cfg.opcode = FIELD_GET(IPSEC_GEN_CFG_OPCODE, val);
5457         rsp->gen_cfg.param1 = FIELD_GET(IPSEC_GEN_CFG_PARAM1, val);
5458         rsp->gen_cfg.param2 = FIELD_GET(IPSEC_GEN_CFG_PARAM2, val);
5459
5460         val = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_RX_CPTX_CREDIT(0));
5461         rsp->cpt_credit = FIELD_GET(CPT_INST_CREDIT_CNT, val);
5462         rsp->credit_th = FIELD_GET(CPT_INST_CREDIT_TH, val);
5463         rsp->bpid = FIELD_GET(CPT_INST_CREDIT_BPID, val);
5464
5465         return 0;
5466 }
5467
5468 int rvu_mbox_handler_nix_inline_ipsec_lf_cfg(struct rvu *rvu,
5469                                              struct nix_inline_ipsec_lf_cfg *req,
5470                                              struct msg_rsp *rsp)
5471 {
5472         int lf, blkaddr, err;
5473         u64 val;
5474
5475         if (!is_block_implemented(rvu->hw, BLKADDR_CPT0))
5476                 return 0;
5477
5478         err = nix_get_nixlf(rvu, req->hdr.pcifunc, &lf, &blkaddr);
5479         if (err)
5480                 return err;
5481
5482         if (req->enable) {
5483                 /* Set TT, TAG_CONST, SA_POW2_SIZE and LENM1_MAX */
5484                 val = (u64)req->ipsec_cfg0.tt << 44 |
5485                       (u64)req->ipsec_cfg0.tag_const << 20 |
5486                       (u64)req->ipsec_cfg0.sa_pow2_size << 16 |
5487                       req->ipsec_cfg0.lenm1_max;
5488
5489                 if (blkaddr == BLKADDR_NIX1)
5490                         val |= BIT_ULL(46);
5491
5492                 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG0(lf), val);
5493
5494                 /* Set SA_IDX_W and SA_IDX_MAX */
5495                 val = (u64)req->ipsec_cfg1.sa_idx_w << 32 |
5496                       req->ipsec_cfg1.sa_idx_max;
5497                 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG1(lf), val);
5498
5499                 /* Set SA base address */
5500                 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(lf),
5501                             req->sa_base_addr);
5502         } else {
5503                 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG0(lf), 0x0);
5504                 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG1(lf), 0x0);
5505                 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(lf),
5506                             0x0);
5507         }
5508
5509         return 0;
5510 }
5511
5512 void rvu_nix_reset_mac(struct rvu_pfvf *pfvf, int pcifunc)
5513 {
5514         bool from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK);
5515
5516         /* overwrite vf mac address with default_mac */
5517         if (from_vf)
5518                 ether_addr_copy(pfvf->mac_addr, pfvf->default_mac);
5519 }
5520
5521 /* NIX ingress policers or bandwidth profiles APIs */
5522 static void nix_config_rx_pkt_policer_precolor(struct rvu *rvu, int blkaddr)
5523 {
5524         struct npc_lt_def_cfg defs, *ltdefs;
5525
5526         ltdefs = &defs;
5527         memcpy(ltdefs, rvu->kpu.lt_def, sizeof(struct npc_lt_def_cfg));
5528
5529         /* Extract PCP and DEI fields from outer VLAN from byte offset
5530          * 2 from the start of LB_PTR (ie TAG).
5531          * VLAN0 is Outer VLAN and VLAN1 is Inner VLAN. Inner VLAN
5532          * fields are considered when 'Tunnel enable' is set in profile.
5533          */
5534         rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN0_PCP_DEI,
5535                     (2UL << 12) | (ltdefs->ovlan.lid << 8) |
5536                     (ltdefs->ovlan.ltype_match << 4) |
5537                     ltdefs->ovlan.ltype_mask);
5538         rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN1_PCP_DEI,
5539                     (2UL << 12) | (ltdefs->ivlan.lid << 8) |
5540                     (ltdefs->ivlan.ltype_match << 4) |
5541                     ltdefs->ivlan.ltype_mask);
5542
5543         /* DSCP field in outer and tunneled IPv4 packets */
5544         rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4_DSCP,
5545                     (1UL << 12) | (ltdefs->rx_oip4.lid << 8) |
5546                     (ltdefs->rx_oip4.ltype_match << 4) |
5547                     ltdefs->rx_oip4.ltype_mask);
5548         rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4_DSCP,
5549                     (1UL << 12) | (ltdefs->rx_iip4.lid << 8) |
5550                     (ltdefs->rx_iip4.ltype_match << 4) |
5551                     ltdefs->rx_iip4.ltype_mask);
5552
5553         /* DSCP field (traffic class) in outer and tunneled IPv6 packets */
5554         rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6_DSCP,
5555                     (1UL << 11) | (ltdefs->rx_oip6.lid << 8) |
5556                     (ltdefs->rx_oip6.ltype_match << 4) |
5557                     ltdefs->rx_oip6.ltype_mask);
5558         rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6_DSCP,
5559                     (1UL << 11) | (ltdefs->rx_iip6.lid << 8) |
5560                     (ltdefs->rx_iip6.ltype_match << 4) |
5561                     ltdefs->rx_iip6.ltype_mask);
5562 }
5563
5564 static int nix_init_policer_context(struct rvu *rvu, struct nix_hw *nix_hw,
5565                                     int layer, int prof_idx)
5566 {
5567         struct nix_cn10k_aq_enq_req aq_req;
5568         int rc;
5569
5570         memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req));
5571
5572         aq_req.qidx = (prof_idx & 0x3FFF) | (layer << 14);
5573         aq_req.ctype = NIX_AQ_CTYPE_BANDPROF;
5574         aq_req.op = NIX_AQ_INSTOP_INIT;
5575
5576         /* Context is all zeros, submit to AQ */
5577         rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw,
5578                                      (struct nix_aq_enq_req *)&aq_req, NULL);
5579         if (rc)
5580                 dev_err(rvu->dev, "Failed to INIT bandwidth profile layer %d profile %d\n",
5581                         layer, prof_idx);
5582         return rc;
5583 }
5584
5585 static int nix_setup_ipolicers(struct rvu *rvu,
5586                                struct nix_hw *nix_hw, int blkaddr)
5587 {
5588         struct rvu_hwinfo *hw = rvu->hw;
5589         struct nix_ipolicer *ipolicer;
5590         int err, layer, prof_idx;
5591         u64 cfg;
5592
5593         cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST);
5594         if (!(cfg & BIT_ULL(61))) {
5595                 hw->cap.ipolicer = false;
5596                 return 0;
5597         }
5598
5599         hw->cap.ipolicer = true;
5600         nix_hw->ipolicer = devm_kcalloc(rvu->dev, BAND_PROF_NUM_LAYERS,
5601                                         sizeof(*ipolicer), GFP_KERNEL);
5602         if (!nix_hw->ipolicer)
5603                 return -ENOMEM;
5604
5605         cfg = rvu_read64(rvu, blkaddr, NIX_AF_PL_CONST);
5606
5607         for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
5608                 ipolicer = &nix_hw->ipolicer[layer];
5609                 switch (layer) {
5610                 case BAND_PROF_LEAF_LAYER:
5611                         ipolicer->band_prof.max = cfg & 0XFFFF;
5612                         break;
5613                 case BAND_PROF_MID_LAYER:
5614                         ipolicer->band_prof.max = (cfg >> 16) & 0XFFFF;
5615                         break;
5616                 case BAND_PROF_TOP_LAYER:
5617                         ipolicer->band_prof.max = (cfg >> 32) & 0XFFFF;
5618                         break;
5619                 }
5620
5621                 if (!ipolicer->band_prof.max)
5622                         continue;
5623
5624                 err = rvu_alloc_bitmap(&ipolicer->band_prof);
5625                 if (err)
5626                         return err;
5627
5628                 ipolicer->pfvf_map = devm_kcalloc(rvu->dev,
5629                                                   ipolicer->band_prof.max,
5630                                                   sizeof(u16), GFP_KERNEL);
5631                 if (!ipolicer->pfvf_map)
5632                         return -ENOMEM;
5633
5634                 ipolicer->match_id = devm_kcalloc(rvu->dev,
5635                                                   ipolicer->band_prof.max,
5636                                                   sizeof(u16), GFP_KERNEL);
5637                 if (!ipolicer->match_id)
5638                         return -ENOMEM;
5639
5640                 for (prof_idx = 0;
5641                      prof_idx < ipolicer->band_prof.max; prof_idx++) {
5642                         /* Set AF as current owner for INIT ops to succeed */
5643                         ipolicer->pfvf_map[prof_idx] = 0x00;
5644
5645                         /* There is no enable bit in the profile context,
5646                          * so no context disable. So let's INIT them here
5647                          * so that PF/VF later on have to just do WRITE to
5648                          * setup policer rates and config.
5649                          */
5650                         err = nix_init_policer_context(rvu, nix_hw,
5651                                                        layer, prof_idx);
5652                         if (err)
5653                                 return err;
5654                 }
5655
5656                 /* Allocate memory for maintaining ref_counts for MID level
5657                  * profiles, this will be needed for leaf layer profiles'
5658                  * aggregation.
5659                  */
5660                 if (layer != BAND_PROF_MID_LAYER)
5661                         continue;
5662
5663                 ipolicer->ref_count = devm_kcalloc(rvu->dev,
5664                                                    ipolicer->band_prof.max,
5665                                                    sizeof(u16), GFP_KERNEL);
5666                 if (!ipolicer->ref_count)
5667                         return -ENOMEM;
5668         }
5669
5670         /* Set policer timeunit to 2us ie  (19 + 1) * 100 nsec = 2us */
5671         rvu_write64(rvu, blkaddr, NIX_AF_PL_TS, 19);
5672
5673         nix_config_rx_pkt_policer_precolor(rvu, blkaddr);
5674
5675         return 0;
5676 }
5677
5678 static void nix_ipolicer_freemem(struct rvu *rvu, struct nix_hw *nix_hw)
5679 {
5680         struct nix_ipolicer *ipolicer;
5681         int layer;
5682
5683         if (!rvu->hw->cap.ipolicer)
5684                 return;
5685
5686         for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
5687                 ipolicer = &nix_hw->ipolicer[layer];
5688
5689                 if (!ipolicer->band_prof.max)
5690                         continue;
5691
5692                 kfree(ipolicer->band_prof.bmap);
5693         }
5694 }
5695
5696 static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req,
5697                                struct nix_hw *nix_hw, u16 pcifunc)
5698 {
5699         struct nix_ipolicer *ipolicer;
5700         int layer, hi_layer, prof_idx;
5701
5702         /* Bits [15:14] in profile index represent layer */
5703         layer = (req->qidx >> 14) & 0x03;
5704         prof_idx = req->qidx & 0x3FFF;
5705
5706         ipolicer = &nix_hw->ipolicer[layer];
5707         if (prof_idx >= ipolicer->band_prof.max)
5708                 return -EINVAL;
5709
5710         /* Check if the profile is allocated to the requesting PCIFUNC or not
5711          * with the exception of AF. AF is allowed to read and update contexts.
5712          */
5713         if (pcifunc && ipolicer->pfvf_map[prof_idx] != pcifunc)
5714                 return -EINVAL;
5715
5716         /* If this profile is linked to higher layer profile then check
5717          * if that profile is also allocated to the requesting PCIFUNC
5718          * or not.
5719          */
5720         if (!req->prof.hl_en)
5721                 return 0;
5722
5723         /* Leaf layer profile can link only to mid layer and
5724          * mid layer to top layer.
5725          */
5726         if (layer == BAND_PROF_LEAF_LAYER)
5727                 hi_layer = BAND_PROF_MID_LAYER;
5728         else if (layer == BAND_PROF_MID_LAYER)
5729                 hi_layer = BAND_PROF_TOP_LAYER;
5730         else
5731                 return -EINVAL;
5732
5733         ipolicer = &nix_hw->ipolicer[hi_layer];
5734         prof_idx = req->prof.band_prof_id;
5735         if (prof_idx >= ipolicer->band_prof.max ||
5736             ipolicer->pfvf_map[prof_idx] != pcifunc)
5737                 return -EINVAL;
5738
5739         return 0;
5740 }
5741
5742 int rvu_mbox_handler_nix_bandprof_alloc(struct rvu *rvu,
5743                                         struct nix_bandprof_alloc_req *req,
5744                                         struct nix_bandprof_alloc_rsp *rsp)
5745 {
5746         int blkaddr, layer, prof, idx, err;
5747         u16 pcifunc = req->hdr.pcifunc;
5748         struct nix_ipolicer *ipolicer;
5749         struct nix_hw *nix_hw;
5750
5751         if (!rvu->hw->cap.ipolicer)
5752                 return NIX_AF_ERR_IPOLICER_NOTSUPP;
5753
5754         err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
5755         if (err)
5756                 return err;
5757
5758         mutex_lock(&rvu->rsrc_lock);
5759         for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
5760                 if (layer == BAND_PROF_INVAL_LAYER)
5761                         continue;
5762                 if (!req->prof_count[layer])
5763                         continue;
5764
5765                 ipolicer = &nix_hw->ipolicer[layer];
5766                 for (idx = 0; idx < req->prof_count[layer]; idx++) {
5767                         /* Allocate a max of 'MAX_BANDPROF_PER_PFFUNC' profiles */
5768                         if (idx == MAX_BANDPROF_PER_PFFUNC)
5769                                 break;
5770
5771                         prof = rvu_alloc_rsrc(&ipolicer->band_prof);
5772                         if (prof < 0)
5773                                 break;
5774                         rsp->prof_count[layer]++;
5775                         rsp->prof_idx[layer][idx] = prof;
5776                         ipolicer->pfvf_map[prof] = pcifunc;
5777                 }
5778         }
5779         mutex_unlock(&rvu->rsrc_lock);
5780         return 0;
5781 }
5782
5783 static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc)
5784 {
5785         int blkaddr, layer, prof_idx, err;
5786         struct nix_ipolicer *ipolicer;
5787         struct nix_hw *nix_hw;
5788
5789         if (!rvu->hw->cap.ipolicer)
5790                 return NIX_AF_ERR_IPOLICER_NOTSUPP;
5791
5792         err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
5793         if (err)
5794                 return err;
5795
5796         mutex_lock(&rvu->rsrc_lock);
5797         /* Free all the profiles allocated to the PCIFUNC */
5798         for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
5799                 if (layer == BAND_PROF_INVAL_LAYER)
5800                         continue;
5801                 ipolicer = &nix_hw->ipolicer[layer];
5802
5803                 for (prof_idx = 0; prof_idx < ipolicer->band_prof.max; prof_idx++) {
5804                         if (ipolicer->pfvf_map[prof_idx] != pcifunc)
5805                                 continue;
5806
5807                         /* Clear ratelimit aggregation, if any */
5808                         if (layer == BAND_PROF_LEAF_LAYER &&
5809                             ipolicer->match_id[prof_idx])
5810                                 nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx);
5811
5812                         ipolicer->pfvf_map[prof_idx] = 0x00;
5813                         ipolicer->match_id[prof_idx] = 0;
5814                         rvu_free_rsrc(&ipolicer->band_prof, prof_idx);
5815                 }
5816         }
5817         mutex_unlock(&rvu->rsrc_lock);
5818         return 0;
5819 }
5820
5821 int rvu_mbox_handler_nix_bandprof_free(struct rvu *rvu,
5822                                        struct nix_bandprof_free_req *req,
5823                                        struct msg_rsp *rsp)
5824 {
5825         int blkaddr, layer, prof_idx, idx, err;
5826         u16 pcifunc = req->hdr.pcifunc;
5827         struct nix_ipolicer *ipolicer;
5828         struct nix_hw *nix_hw;
5829
5830         if (req->free_all)
5831                 return nix_free_all_bandprof(rvu, pcifunc);
5832
5833         if (!rvu->hw->cap.ipolicer)
5834                 return NIX_AF_ERR_IPOLICER_NOTSUPP;
5835
5836         err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
5837         if (err)
5838                 return err;
5839
5840         mutex_lock(&rvu->rsrc_lock);
5841         /* Free the requested profile indices */
5842         for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
5843                 if (layer == BAND_PROF_INVAL_LAYER)
5844                         continue;
5845                 if (!req->prof_count[layer])
5846                         continue;
5847
5848                 ipolicer = &nix_hw->ipolicer[layer];
5849                 for (idx = 0; idx < req->prof_count[layer]; idx++) {
5850                         if (idx == MAX_BANDPROF_PER_PFFUNC)
5851                                 break;
5852                         prof_idx = req->prof_idx[layer][idx];
5853                         if (prof_idx >= ipolicer->band_prof.max ||
5854                             ipolicer->pfvf_map[prof_idx] != pcifunc)
5855                                 continue;
5856
5857                         /* Clear ratelimit aggregation, if any */
5858                         if (layer == BAND_PROF_LEAF_LAYER &&
5859                             ipolicer->match_id[prof_idx])
5860                                 nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx);
5861
5862                         ipolicer->pfvf_map[prof_idx] = 0x00;
5863                         ipolicer->match_id[prof_idx] = 0;
5864                         rvu_free_rsrc(&ipolicer->band_prof, prof_idx);
5865                 }
5866         }
5867         mutex_unlock(&rvu->rsrc_lock);
5868         return 0;
5869 }
5870
5871 int nix_aq_context_read(struct rvu *rvu, struct nix_hw *nix_hw,
5872                         struct nix_cn10k_aq_enq_req *aq_req,
5873                         struct nix_cn10k_aq_enq_rsp *aq_rsp,
5874                         u16 pcifunc, u8 ctype, u32 qidx)
5875 {
5876         memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req));
5877         aq_req->hdr.pcifunc = pcifunc;
5878         aq_req->ctype = ctype;
5879         aq_req->op = NIX_AQ_INSTOP_READ;
5880         aq_req->qidx = qidx;
5881
5882         return rvu_nix_blk_aq_enq_inst(rvu, nix_hw,
5883                                        (struct nix_aq_enq_req *)aq_req,
5884                                        (struct nix_aq_enq_rsp *)aq_rsp);
5885 }
5886
5887 static int nix_ipolicer_map_leaf_midprofs(struct rvu *rvu,
5888                                           struct nix_hw *nix_hw,
5889                                           struct nix_cn10k_aq_enq_req *aq_req,
5890                                           struct nix_cn10k_aq_enq_rsp *aq_rsp,
5891                                           u32 leaf_prof, u16 mid_prof)
5892 {
5893         memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req));
5894         aq_req->hdr.pcifunc = 0x00;
5895         aq_req->ctype = NIX_AQ_CTYPE_BANDPROF;
5896         aq_req->op = NIX_AQ_INSTOP_WRITE;
5897         aq_req->qidx = leaf_prof;
5898
5899         aq_req->prof.band_prof_id = mid_prof;
5900         aq_req->prof_mask.band_prof_id = GENMASK(6, 0);
5901         aq_req->prof.hl_en = 1;
5902         aq_req->prof_mask.hl_en = 1;
5903
5904         return rvu_nix_blk_aq_enq_inst(rvu, nix_hw,
5905                                        (struct nix_aq_enq_req *)aq_req,
5906                                        (struct nix_aq_enq_rsp *)aq_rsp);
5907 }
5908
5909 int rvu_nix_setup_ratelimit_aggr(struct rvu *rvu, u16 pcifunc,
5910                                  u16 rq_idx, u16 match_id)
5911 {
5912         int leaf_prof, mid_prof, leaf_match;
5913         struct nix_cn10k_aq_enq_req aq_req;
5914         struct nix_cn10k_aq_enq_rsp aq_rsp;
5915         struct nix_ipolicer *ipolicer;
5916         struct nix_hw *nix_hw;
5917         int blkaddr, idx, rc;
5918
5919         if (!rvu->hw->cap.ipolicer)
5920                 return 0;
5921
5922         rc = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
5923         if (rc)
5924                 return rc;
5925
5926         /* Fetch the RQ's context to see if policing is enabled */
5927         rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, pcifunc,
5928                                  NIX_AQ_CTYPE_RQ, rq_idx);
5929         if (rc) {
5930                 dev_err(rvu->dev,
5931                         "%s: Failed to fetch RQ%d context of PFFUNC 0x%x\n",
5932                         __func__, rq_idx, pcifunc);
5933                 return rc;
5934         }
5935
5936         if (!aq_rsp.rq.policer_ena)
5937                 return 0;
5938
5939         /* Get the bandwidth profile ID mapped to this RQ */
5940         leaf_prof = aq_rsp.rq.band_prof_id;
5941
5942         ipolicer = &nix_hw->ipolicer[BAND_PROF_LEAF_LAYER];
5943         ipolicer->match_id[leaf_prof] = match_id;
5944
5945         /* Check if any other leaf profile is marked with same match_id */
5946         for (idx = 0; idx < ipolicer->band_prof.max; idx++) {
5947                 if (idx == leaf_prof)
5948                         continue;
5949                 if (ipolicer->match_id[idx] != match_id)
5950                         continue;
5951
5952                 leaf_match = idx;
5953                 break;
5954         }
5955
5956         if (idx == ipolicer->band_prof.max)
5957                 return 0;
5958
5959         /* Fetch the matching profile's context to check if it's already
5960          * mapped to a mid level profile.
5961          */
5962         rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00,
5963                                  NIX_AQ_CTYPE_BANDPROF, leaf_match);
5964         if (rc) {
5965                 dev_err(rvu->dev,
5966                         "%s: Failed to fetch context of leaf profile %d\n",
5967                         __func__, leaf_match);
5968                 return rc;
5969         }
5970
5971         ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER];
5972         if (aq_rsp.prof.hl_en) {
5973                 /* Get Mid layer prof index and map leaf_prof index
5974                  * also such that flows that are being steered
5975                  * to different RQs and marked with same match_id
5976                  * are rate limited in a aggregate fashion
5977                  */
5978                 mid_prof = aq_rsp.prof.band_prof_id;
5979                 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw,
5980                                                     &aq_req, &aq_rsp,
5981                                                     leaf_prof, mid_prof);
5982                 if (rc) {
5983                         dev_err(rvu->dev,
5984                                 "%s: Failed to map leaf(%d) and mid(%d) profiles\n",
5985                                 __func__, leaf_prof, mid_prof);
5986                         goto exit;
5987                 }
5988
5989                 mutex_lock(&rvu->rsrc_lock);
5990                 ipolicer->ref_count[mid_prof]++;
5991                 mutex_unlock(&rvu->rsrc_lock);
5992                 goto exit;
5993         }
5994
5995         /* Allocate a mid layer profile and
5996          * map both 'leaf_prof' and 'leaf_match' profiles to it.
5997          */
5998         mutex_lock(&rvu->rsrc_lock);
5999         mid_prof = rvu_alloc_rsrc(&ipolicer->band_prof);
6000         if (mid_prof < 0) {
6001                 dev_err(rvu->dev,
6002                         "%s: Unable to allocate mid layer profile\n", __func__);
6003                 mutex_unlock(&rvu->rsrc_lock);
6004                 goto exit;
6005         }
6006         mutex_unlock(&rvu->rsrc_lock);
6007         ipolicer->pfvf_map[mid_prof] = 0x00;
6008         ipolicer->ref_count[mid_prof] = 0;
6009
6010         /* Initialize mid layer profile same as 'leaf_prof' */
6011         rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00,
6012                                  NIX_AQ_CTYPE_BANDPROF, leaf_prof);
6013         if (rc) {
6014                 dev_err(rvu->dev,
6015                         "%s: Failed to fetch context of leaf profile %d\n",
6016                         __func__, leaf_prof);
6017                 goto exit;
6018         }
6019
6020         memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req));
6021         aq_req.hdr.pcifunc = 0x00;
6022         aq_req.qidx = (mid_prof & 0x3FFF) | (BAND_PROF_MID_LAYER << 14);
6023         aq_req.ctype = NIX_AQ_CTYPE_BANDPROF;
6024         aq_req.op = NIX_AQ_INSTOP_WRITE;
6025         memcpy(&aq_req.prof, &aq_rsp.prof, sizeof(struct nix_bandprof_s));
6026         memset((char *)&aq_req.prof_mask, 0xff, sizeof(struct nix_bandprof_s));
6027         /* Clear higher layer enable bit in the mid profile, just in case */
6028         aq_req.prof.hl_en = 0;
6029         aq_req.prof_mask.hl_en = 1;
6030
6031         rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw,
6032                                      (struct nix_aq_enq_req *)&aq_req, NULL);
6033         if (rc) {
6034                 dev_err(rvu->dev,
6035                         "%s: Failed to INIT context of mid layer profile %d\n",
6036                         __func__, mid_prof);
6037                 goto exit;
6038         }
6039
6040         /* Map both leaf profiles to this mid layer profile */
6041         rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw,
6042                                             &aq_req, &aq_rsp,
6043                                             leaf_prof, mid_prof);
6044         if (rc) {
6045                 dev_err(rvu->dev,
6046                         "%s: Failed to map leaf(%d) and mid(%d) profiles\n",
6047                         __func__, leaf_prof, mid_prof);
6048                 goto exit;
6049         }
6050
6051         mutex_lock(&rvu->rsrc_lock);
6052         ipolicer->ref_count[mid_prof]++;
6053         mutex_unlock(&rvu->rsrc_lock);
6054
6055         rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw,
6056                                             &aq_req, &aq_rsp,
6057                                             leaf_match, mid_prof);
6058         if (rc) {
6059                 dev_err(rvu->dev,
6060                         "%s: Failed to map leaf(%d) and mid(%d) profiles\n",
6061                         __func__, leaf_match, mid_prof);
6062                 ipolicer->ref_count[mid_prof]--;
6063                 goto exit;
6064         }
6065
6066         mutex_lock(&rvu->rsrc_lock);
6067         ipolicer->ref_count[mid_prof]++;
6068         mutex_unlock(&rvu->rsrc_lock);
6069
6070 exit:
6071         return rc;
6072 }
6073
6074 /* Called with mutex rsrc_lock */
6075 static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw,
6076                                      u32 leaf_prof)
6077 {
6078         struct nix_cn10k_aq_enq_req aq_req;
6079         struct nix_cn10k_aq_enq_rsp aq_rsp;
6080         struct nix_ipolicer *ipolicer;
6081         u16 mid_prof;
6082         int rc;
6083
6084         mutex_unlock(&rvu->rsrc_lock);
6085
6086         rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00,
6087                                  NIX_AQ_CTYPE_BANDPROF, leaf_prof);
6088
6089         mutex_lock(&rvu->rsrc_lock);
6090         if (rc) {
6091                 dev_err(rvu->dev,
6092                         "%s: Failed to fetch context of leaf profile %d\n",
6093                         __func__, leaf_prof);
6094                 return;
6095         }
6096
6097         if (!aq_rsp.prof.hl_en)
6098                 return;
6099
6100         mid_prof = aq_rsp.prof.band_prof_id;
6101         ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER];
6102         ipolicer->ref_count[mid_prof]--;
6103         /* If ref_count is zero, free mid layer profile */
6104         if (!ipolicer->ref_count[mid_prof]) {
6105                 ipolicer->pfvf_map[mid_prof] = 0x00;
6106                 rvu_free_rsrc(&ipolicer->band_prof, mid_prof);
6107         }
6108 }
6109
6110 int rvu_mbox_handler_nix_bandprof_get_hwinfo(struct rvu *rvu, struct msg_req *req,
6111                                              struct nix_bandprof_get_hwinfo_rsp *rsp)
6112 {
6113         struct nix_ipolicer *ipolicer;
6114         int blkaddr, layer, err;
6115         struct nix_hw *nix_hw;
6116         u64 tu;
6117
6118         if (!rvu->hw->cap.ipolicer)
6119                 return NIX_AF_ERR_IPOLICER_NOTSUPP;
6120
6121         err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr);
6122         if (err)
6123                 return err;
6124
6125         /* Return number of bandwidth profiles free at each layer */
6126         mutex_lock(&rvu->rsrc_lock);
6127         for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) {
6128                 if (layer == BAND_PROF_INVAL_LAYER)
6129                         continue;
6130
6131                 ipolicer = &nix_hw->ipolicer[layer];
6132                 rsp->prof_count[layer] = rvu_rsrc_free_count(&ipolicer->band_prof);
6133         }
6134         mutex_unlock(&rvu->rsrc_lock);
6135
6136         /* Set the policer timeunit in nanosec */
6137         tu = rvu_read64(rvu, blkaddr, NIX_AF_PL_TS) & GENMASK_ULL(9, 0);
6138         rsp->policer_timeunit = (tu + 1) * 100;
6139
6140         return 0;
6141 }
6142
6143 static struct nix_mcast_grp_elem *rvu_nix_mcast_find_grp_elem(struct nix_mcast_grp *mcast_grp,
6144                                                               u32 mcast_grp_idx)
6145 {
6146         struct nix_mcast_grp_elem *iter;
6147         bool is_found = false;
6148
6149         list_for_each_entry(iter, &mcast_grp->mcast_grp_head, list) {
6150                 if (iter->mcast_grp_idx == mcast_grp_idx) {
6151                         is_found = true;
6152                         break;
6153                 }
6154         }
6155
6156         if (is_found)
6157                 return iter;
6158
6159         return NULL;
6160 }
6161
6162 int rvu_nix_mcast_get_mce_index(struct rvu *rvu, u16 pcifunc, u32 mcast_grp_idx)
6163 {
6164         struct nix_mcast_grp_elem *elem;
6165         struct nix_mcast_grp *mcast_grp;
6166         struct nix_hw *nix_hw;
6167         int blkaddr, ret;
6168
6169         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
6170         nix_hw = get_nix_hw(rvu->hw, blkaddr);
6171         if (!nix_hw)
6172                 return NIX_AF_ERR_INVALID_NIXBLK;
6173
6174         mcast_grp = &nix_hw->mcast_grp;
6175         mutex_lock(&mcast_grp->mcast_grp_lock);
6176         elem = rvu_nix_mcast_find_grp_elem(mcast_grp, mcast_grp_idx);
6177         if (!elem)
6178                 ret = NIX_AF_ERR_INVALID_MCAST_GRP;
6179         else
6180                 ret = elem->mce_start_index;
6181
6182         mutex_unlock(&mcast_grp->mcast_grp_lock);
6183         return ret;
6184 }
6185
6186 void rvu_nix_mcast_flr_free_entries(struct rvu *rvu, u16 pcifunc)
6187 {
6188         struct nix_mcast_grp_destroy_req dreq = { 0 };
6189         struct nix_mcast_grp_update_req ureq = { 0 };
6190         struct nix_mcast_grp_update_rsp ursp = { 0 };
6191         struct nix_mcast_grp_elem *elem, *tmp;
6192         struct nix_mcast_grp *mcast_grp;
6193         struct nix_hw *nix_hw;
6194         int blkaddr;
6195
6196         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
6197         nix_hw = get_nix_hw(rvu->hw, blkaddr);
6198         if (!nix_hw)
6199                 return;
6200
6201         mcast_grp = &nix_hw->mcast_grp;
6202
6203         mutex_lock(&mcast_grp->mcast_grp_lock);
6204         list_for_each_entry_safe(elem, tmp, &mcast_grp->mcast_grp_head, list) {
6205                 struct nix_mce_list *mce_list;
6206                 struct hlist_node *tmp;
6207                 struct mce *mce;
6208
6209                 /* If the pcifunc which created the multicast/mirror
6210                  * group received an FLR, then delete the entire group.
6211                  */
6212                 if (elem->pcifunc == pcifunc) {
6213                         /* Delete group */
6214                         dreq.hdr.pcifunc = elem->pcifunc;
6215                         dreq.mcast_grp_idx = elem->mcast_grp_idx;
6216                         dreq.is_af = 1;
6217                         rvu_mbox_handler_nix_mcast_grp_destroy(rvu, &dreq, NULL);
6218                         continue;
6219                 }
6220
6221                 /* Iterate the group elements and delete the element which
6222                  * received the FLR.
6223                  */
6224                 mce_list = &elem->mcast_mce_list;
6225                 hlist_for_each_entry_safe(mce, tmp, &mce_list->head, node) {
6226                         if (mce->pcifunc == pcifunc) {
6227                                 ureq.hdr.pcifunc = pcifunc;
6228                                 ureq.num_mce_entry = 1;
6229                                 ureq.mcast_grp_idx = elem->mcast_grp_idx;
6230                                 ureq.op = NIX_MCAST_OP_DEL_ENTRY;
6231                                 ureq.pcifunc[0] = pcifunc;
6232                                 ureq.is_af = 1;
6233                                 rvu_mbox_handler_nix_mcast_grp_update(rvu, &ureq, &ursp);
6234                                 break;
6235                         }
6236                 }
6237         }
6238         mutex_unlock(&mcast_grp->mcast_grp_lock);
6239 }
6240
6241 int rvu_nix_mcast_update_mcam_entry(struct rvu *rvu, u16 pcifunc,
6242                                     u32 mcast_grp_idx, u16 mcam_index)
6243 {
6244         struct nix_mcast_grp_elem *elem;
6245         struct nix_mcast_grp *mcast_grp;
6246         struct nix_hw *nix_hw;
6247         int blkaddr, ret = 0;
6248
6249         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
6250         nix_hw = get_nix_hw(rvu->hw, blkaddr);
6251         if (!nix_hw)
6252                 return NIX_AF_ERR_INVALID_NIXBLK;
6253
6254         mcast_grp = &nix_hw->mcast_grp;
6255         mutex_lock(&mcast_grp->mcast_grp_lock);
6256         elem = rvu_nix_mcast_find_grp_elem(mcast_grp, mcast_grp_idx);
6257         if (!elem)
6258                 ret = NIX_AF_ERR_INVALID_MCAST_GRP;
6259         else
6260                 elem->mcam_index = mcam_index;
6261
6262         mutex_unlock(&mcast_grp->mcast_grp_lock);
6263         return ret;
6264 }
6265
6266 int rvu_mbox_handler_nix_mcast_grp_create(struct rvu *rvu,
6267                                           struct nix_mcast_grp_create_req *req,
6268                                           struct nix_mcast_grp_create_rsp *rsp)
6269 {
6270         struct nix_mcast_grp_elem *elem;
6271         struct nix_mcast_grp *mcast_grp;
6272         struct nix_hw *nix_hw;
6273         int blkaddr, err;
6274
6275         err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr);
6276         if (err)
6277                 return err;
6278
6279         mcast_grp = &nix_hw->mcast_grp;
6280         elem = kzalloc(sizeof(*elem), GFP_KERNEL);
6281         if (!elem)
6282                 return -ENOMEM;
6283
6284         INIT_HLIST_HEAD(&elem->mcast_mce_list.head);
6285         elem->mcam_index = -1;
6286         elem->mce_start_index = -1;
6287         elem->pcifunc = req->hdr.pcifunc;
6288         elem->dir = req->dir;
6289         elem->mcast_grp_idx = mcast_grp->next_grp_index++;
6290
6291         mutex_lock(&mcast_grp->mcast_grp_lock);
6292         list_add_tail(&elem->list, &mcast_grp->mcast_grp_head);
6293         mcast_grp->count++;
6294         mutex_unlock(&mcast_grp->mcast_grp_lock);
6295
6296         rsp->mcast_grp_idx = elem->mcast_grp_idx;
6297         return 0;
6298 }
6299
6300 int rvu_mbox_handler_nix_mcast_grp_destroy(struct rvu *rvu,
6301                                            struct nix_mcast_grp_destroy_req *req,
6302                                            struct msg_rsp *rsp)
6303 {
6304         struct npc_delete_flow_req uninstall_req = { 0 };
6305         struct npc_delete_flow_rsp uninstall_rsp = { 0 };
6306         struct nix_mcast_grp_elem *elem;
6307         struct nix_mcast_grp *mcast_grp;
6308         int blkaddr, err, ret = 0;
6309         struct nix_mcast *mcast;
6310         struct nix_hw *nix_hw;
6311
6312         err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr);
6313         if (err)
6314                 return err;
6315
6316         mcast_grp = &nix_hw->mcast_grp;
6317
6318         /* If AF is requesting for the deletion,
6319          * then AF is already taking the lock
6320          */
6321         if (!req->is_af)
6322                 mutex_lock(&mcast_grp->mcast_grp_lock);
6323
6324         elem = rvu_nix_mcast_find_grp_elem(mcast_grp, req->mcast_grp_idx);
6325         if (!elem) {
6326                 ret = NIX_AF_ERR_INVALID_MCAST_GRP;
6327                 goto unlock_grp;
6328         }
6329
6330         /* If no mce entries are associated with the group
6331          * then just remove it from the global list.
6332          */
6333         if (!elem->mcast_mce_list.count)
6334                 goto delete_grp;
6335
6336         /* Delete the associated mcam entry and
6337          * remove all mce entries from the group
6338          */
6339         mcast = &nix_hw->mcast;
6340         mutex_lock(&mcast->mce_lock);
6341         if (elem->mcam_index != -1) {
6342                 uninstall_req.hdr.pcifunc = req->hdr.pcifunc;
6343                 uninstall_req.entry = elem->mcam_index;
6344                 rvu_mbox_handler_npc_delete_flow(rvu, &uninstall_req, &uninstall_rsp);
6345         }
6346
6347         nix_free_mce_list(mcast, elem->mcast_mce_list.count,
6348                           elem->mce_start_index, elem->dir);
6349         nix_delete_mcast_mce_list(&elem->mcast_mce_list);
6350         mutex_unlock(&mcast->mce_lock);
6351
6352 delete_grp:
6353         list_del(&elem->list);
6354         kfree(elem);
6355         mcast_grp->count--;
6356
6357 unlock_grp:
6358         if (!req->is_af)
6359                 mutex_unlock(&mcast_grp->mcast_grp_lock);
6360
6361         return ret;
6362 }
6363
6364 int rvu_mbox_handler_nix_mcast_grp_update(struct rvu *rvu,
6365                                           struct nix_mcast_grp_update_req *req,
6366                                           struct nix_mcast_grp_update_rsp *rsp)
6367 {
6368         struct nix_mcast_grp_destroy_req dreq = { 0 };
6369         struct npc_mcam *mcam = &rvu->hw->mcam;
6370         struct nix_mcast_grp_elem *elem;
6371         struct nix_mcast_grp *mcast_grp;
6372         int blkaddr, err, npc_blkaddr;
6373         u16 prev_count, new_count;
6374         struct nix_mcast *mcast;
6375         struct nix_hw *nix_hw;
6376         int i, ret;
6377
6378         if (!req->num_mce_entry)
6379                 return 0;
6380
6381         err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr);
6382         if (err)
6383                 return err;
6384
6385         mcast_grp = &nix_hw->mcast_grp;
6386
6387         /* If AF is requesting for the updation,
6388          * then AF is already taking the lock
6389          */
6390         if (!req->is_af)
6391                 mutex_lock(&mcast_grp->mcast_grp_lock);
6392
6393         elem = rvu_nix_mcast_find_grp_elem(mcast_grp, req->mcast_grp_idx);
6394         if (!elem) {
6395                 ret = NIX_AF_ERR_INVALID_MCAST_GRP;
6396                 goto unlock_grp;
6397         }
6398
6399         /* If any pcifunc matches the group's pcifunc, then we can
6400          * delete the entire group.
6401          */
6402         if (req->op == NIX_MCAST_OP_DEL_ENTRY) {
6403                 for (i = 0; i < req->num_mce_entry; i++) {
6404                         if (elem->pcifunc == req->pcifunc[i]) {
6405                                 /* Delete group */
6406                                 dreq.hdr.pcifunc = elem->pcifunc;
6407                                 dreq.mcast_grp_idx = elem->mcast_grp_idx;
6408                                 dreq.is_af = 1;
6409                                 rvu_mbox_handler_nix_mcast_grp_destroy(rvu, &dreq, NULL);
6410                                 ret = 0;
6411                                 goto unlock_grp;
6412                         }
6413                 }
6414         }
6415
6416         mcast = &nix_hw->mcast;
6417         mutex_lock(&mcast->mce_lock);
6418         npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
6419         if (elem->mcam_index != -1)
6420                 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, elem->mcam_index, false);
6421
6422         prev_count = elem->mcast_mce_list.count;
6423         if (req->op == NIX_MCAST_OP_ADD_ENTRY) {
6424                 new_count = prev_count + req->num_mce_entry;
6425                 if (prev_count)
6426                         nix_free_mce_list(mcast, prev_count, elem->mce_start_index, elem->dir);
6427
6428                 elem->mce_start_index = nix_alloc_mce_list(mcast, new_count, elem->dir);
6429
6430                 /* It is possible not to get contiguous memory */
6431                 if (elem->mce_start_index < 0) {
6432                         if (elem->mcam_index != -1) {
6433                                 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr,
6434                                                       elem->mcam_index, true);
6435                                 ret = NIX_AF_ERR_NON_CONTIG_MCE_LIST;
6436                                 goto unlock_mce;
6437                         }
6438                 }
6439
6440                 ret = nix_add_mce_list_entry(rvu, nix_hw, elem, req);
6441                 if (ret) {
6442                         nix_free_mce_list(mcast, new_count, elem->mce_start_index, elem->dir);
6443                         if (prev_count)
6444                                 elem->mce_start_index = nix_alloc_mce_list(mcast,
6445                                                                            prev_count,
6446                                                                            elem->dir);
6447
6448                         if (elem->mcam_index != -1)
6449                                 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr,
6450                                                       elem->mcam_index, true);
6451
6452                         goto unlock_mce;
6453                 }
6454         } else {
6455                 if (!prev_count || prev_count < req->num_mce_entry) {
6456                         if (elem->mcam_index != -1)
6457                                 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr,
6458                                                       elem->mcam_index, true);
6459                         ret = NIX_AF_ERR_INVALID_MCAST_DEL_REQ;
6460                         goto unlock_mce;
6461                 }
6462
6463                 nix_free_mce_list(mcast, prev_count, elem->mce_start_index, elem->dir);
6464                 new_count = prev_count - req->num_mce_entry;
6465                 elem->mce_start_index = nix_alloc_mce_list(mcast, new_count, elem->dir);
6466                 ret = nix_del_mce_list_entry(rvu, nix_hw, elem, req);
6467                 if (ret) {
6468                         nix_free_mce_list(mcast, new_count, elem->mce_start_index, elem->dir);
6469                         elem->mce_start_index = nix_alloc_mce_list(mcast, prev_count, elem->dir);
6470                         if (elem->mcam_index != -1)
6471                                 npc_enable_mcam_entry(rvu, mcam,
6472                                                       npc_blkaddr,
6473                                                       elem->mcam_index,
6474                                                       true);
6475
6476                         goto unlock_mce;
6477                 }
6478         }
6479
6480         if (elem->mcam_index == -1) {
6481                 rsp->mce_start_index = elem->mce_start_index;
6482                 ret = 0;
6483                 goto unlock_mce;
6484         }
6485
6486         nix_mcast_update_action(rvu, elem);
6487         npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, elem->mcam_index, true);
6488         rsp->mce_start_index = elem->mce_start_index;
6489         ret = 0;
6490
6491 unlock_mce:
6492         mutex_unlock(&mcast->mce_lock);
6493
6494 unlock_grp:
6495         if (!req->is_af)
6496                 mutex_unlock(&mcast_grp->mcast_grp_lock);
6497
6498         return ret;
6499 }