drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/ip.h>
  41 #include <net/tcp.h>
  42 #include <net/udp.h>
  43 #include <net/checksum.h>
  44 #include <net/ip6_checksum.h>
  45 #include <net/udp_tunnel.h>
  46 #include <linux/workqueue.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/cache.h>
  49 #include <linux/log2.h>
  50 #include <linux/aer.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53 #include <linux/cpumask.h>
  54 #include <net/pkt_cls.h>
  55 #include <linux/hwmon.h>
  56 #include <linux/hwmon-sysfs.h>
  57 #include <net/page_pool.h>
  58
  59 #include "bnxt_hsi.h"
  60 #include "bnxt.h"
  61 #include "bnxt_ulp.h"
  62 #include "bnxt_sriov.h"
  63 #include "bnxt_ethtool.h"
  64 #include "bnxt_dcb.h"
  65 #include "bnxt_xdp.h"
  66 #include "bnxt_vfr.h"
  67 #include "bnxt_tc.h"
  68 #include "bnxt_devlink.h"
  69 #include "bnxt_debugfs.h"
  70
  71 #define BNXT_TX_TIMEOUT         (5 * HZ)
  72 #define BNXT_DEF_MSG_ENABLE     (NETIF_MSG_DRV | NETIF_MSG_HW)
  73
  74 MODULE_LICENSE("GPL");
  75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  76
  77 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  78 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  79 #define BNXT_RX_COPY_THRESH 256
  80
  81 #define BNXT_TX_PUSH_THRESH 164
  82
  83 enum board_idx {
  84         BCM57301,
  85         BCM57302,
  86         BCM57304,
  87         BCM57417_NPAR,
  88         BCM58700,
  89         BCM57311,
  90         BCM57312,
  91         BCM57402,
  92         BCM57404,
  93         BCM57406,
  94         BCM57402_NPAR,
  95         BCM57407,
  96         BCM57412,
  97         BCM57414,
  98         BCM57416,
  99         BCM57417,
 100         BCM57412_NPAR,
 101         BCM57314,
 102         BCM57417_SFP,
 103         BCM57416_SFP,
 104         BCM57404_NPAR,
 105         BCM57406_NPAR,
 106         BCM57407_SFP,
 107         BCM57407_NPAR,
 108         BCM57414_NPAR,
 109         BCM57416_NPAR,
 110         BCM57452,
 111         BCM57454,
 112         BCM5745x_NPAR,
 113         BCM57508,
 114         BCM57504,
 115         BCM57502,
 116         BCM57508_NPAR,
 117         BCM57504_NPAR,
 118         BCM57502_NPAR,
 119         BCM58802,
 120         BCM58804,
 121         BCM58808,
 122         NETXTREME_E_VF,
 123         NETXTREME_C_VF,
 124         NETXTREME_S_VF,
 125         NETXTREME_E_P5_VF,
 126 };
 127
 128 /* indexed by enum above */
 129 static const struct {
 130         char *name;
 131 } board_info[] = {
 132         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 133         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 134         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 135         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 136         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 137         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 138         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 139         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 140         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 141         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 142         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 143         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 144         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 145         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 146         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 147         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 148         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 149         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 150         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 151         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 152         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 153         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 154         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 155         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 156         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 157         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 158         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 159         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 160         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 161         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 162         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 163         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 164         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 165         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 166         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 167         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 168         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 169         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 170         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 171         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 172         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 173         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 174 };
 175
 176 static const struct pci_device_id bnxt_pci_tbl[] = {
 177         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 178         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 179         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 180         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 182         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 183         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 184         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 185         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 186         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 187         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 188         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 189         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 190         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 191         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 192         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 193         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 194         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 195         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 196         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 197         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 198         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 199         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 200         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 201         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 202         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 203         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 204         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 205         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 206         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 207         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 208         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 209         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 210         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 211         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 212         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 213         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 214         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 215         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
 216         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 217         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
 218         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
 219         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 220         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
 221         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 222         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 223 #ifdef CONFIG_BNXT_SRIOV
 224         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 225         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 226         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 227         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 228         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 229         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 230         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 231         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 232         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 233         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 234         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 235 #endif
 236         { 0 }
 237 };
 238
 239 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 240
 241 static const u16 bnxt_vf_req_snif[] = {
 242         HWRM_FUNC_CFG,
 243         HWRM_FUNC_VF_CFG,
 244         HWRM_PORT_PHY_QCFG,
 245         HWRM_CFA_L2_FILTER_ALLOC,
 246 };
 247
 248 static const u16 bnxt_async_events_arr[] = {
 249         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 250         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
 251         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 252         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 253         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 254         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 255         ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
 256         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 257         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 258         ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
 259         ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
 260         ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
 261 };
 262
 263 static struct workqueue_struct *bnxt_pf_wq;
 264
 265 static bool bnxt_vf_pciid(enum board_idx idx)
 266 {
 267         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 268                 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
 269 }
 270
 271 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 272 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 273 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 274
 275 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 276                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 277
 278 #define BNXT_DB_CQ(db, idx)                                             \
 279         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 280
 281 #define BNXT_DB_NQ_P5(db, idx)                                          \
 282         writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
 283
 284 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 285         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 286
 287 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 288         writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
 289
 290 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 291 {
 292         if (bp->flags & BNXT_FLAG_CHIP_P5)
 293                 BNXT_DB_NQ_P5(db, idx);
 294         else
 295                 BNXT_DB_CQ(db, idx);
 296 }
 297
 298 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 299 {
 300         if (bp->flags & BNXT_FLAG_CHIP_P5)
 301                 BNXT_DB_NQ_ARM_P5(db, idx);
 302         else
 303                 BNXT_DB_CQ_ARM(db, idx);
 304 }
 305
 306 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 307 {
 308         if (bp->flags & BNXT_FLAG_CHIP_P5)
 309                 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
 310                        db->doorbell);
 311         else
 312                 BNXT_DB_CQ(db, idx);
 313 }
 314
 315 const u16 bnxt_lhint_arr[] = {
 316         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 317         TX_BD_FLAGS_LHINT_512_TO_1023,
 318         TX_BD_FLAGS_LHINT_1024_TO_2047,
 319         TX_BD_FLAGS_LHINT_1024_TO_2047,
 320         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 321         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 322         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 323         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 324         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 325         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 326         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 327         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 328         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 329         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 330         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 331         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 332         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 333         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 334         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 335 };
 336
 337 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 338 {
 339         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 340
 341         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 342                 return 0;
 343
 344         return md_dst->u.port_info.port_id;
 345 }
 346
 347 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 348 {
 349         struct bnxt *bp = netdev_priv(dev);
 350         struct tx_bd *txbd;
 351         struct tx_bd_ext *txbd1;
 352         struct netdev_queue *txq;
 353         int i;
 354         dma_addr_t mapping;
 355         unsigned int length, pad = 0;
 356         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 357         u16 prod, last_frag;
 358         struct pci_dev *pdev = bp->pdev;
 359         struct bnxt_tx_ring_info *txr;
 360         struct bnxt_sw_tx_bd *tx_buf;
 361
 362         i = skb_get_queue_mapping(skb);
 363         if (unlikely(i >= bp->tx_nr_rings)) {
 364                 dev_kfree_skb_any(skb);
 365                 return NETDEV_TX_OK;
 366         }
 367
 368         txq = netdev_get_tx_queue(dev, i);
 369         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 370         prod = txr->tx_prod;
 371
 372         free_size = bnxt_tx_avail(bp, txr);
 373         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 374                 netif_tx_stop_queue(txq);
 375                 return NETDEV_TX_BUSY;
 376         }
 377
 378         length = skb->len;
 379         len = skb_headlen(skb);
 380         last_frag = skb_shinfo(skb)->nr_frags;
 381
 382         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 383
 384         txbd->tx_bd_opaque = prod;
 385
 386         tx_buf = &txr->tx_buf_ring[prod];
 387         tx_buf->skb = skb;
 388         tx_buf->nr_frags = last_frag;
 389
 390         vlan_tag_flags = 0;
 391         cfa_action = bnxt_xmit_get_cfa_action(skb);
 392         if (skb_vlan_tag_present(skb)) {
 393                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 394                                  skb_vlan_tag_get(skb);
 395                 /* Currently supports 8021Q, 8021AD vlan offloads
 396                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 397                  */
 398                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 399                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 400         }
 401
 402         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 403                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 404                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 405                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 406                 void __iomem *db = txr->tx_db.doorbell;
 407                 void *pdata = tx_push_buf->data;
 408                 u64 *end;
 409                 int j, push_len;
 410
 411                 /* Set COAL_NOW to be ready quickly for the next push */
 412                 tx_push->tx_bd_len_flags_type =
 413                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 414                                         TX_BD_TYPE_LONG_TX_BD |
 415                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 416                                         TX_BD_FLAGS_COAL_NOW |
 417                                         TX_BD_FLAGS_PACKET_END |
 418                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 419
 420                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 421                         tx_push1->tx_bd_hsize_lflags =
 422                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 423                 else
 424                         tx_push1->tx_bd_hsize_lflags = 0;
 425
 426                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 427                 tx_push1->tx_bd_cfa_action =
 428                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 429
 430                 end = pdata + length;
 431                 end = PTR_ALIGN(end, 8) - 1;
 432                 *end = 0;
 433
 434                 skb_copy_from_linear_data(skb, pdata, len);
 435                 pdata += len;
 436                 for (j = 0; j < last_frag; j++) {
 437                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 438                         void *fptr;
 439
 440                         fptr = skb_frag_address_safe(frag);
 441                         if (!fptr)
 442                                 goto normal_tx;
 443
 444                         memcpy(pdata, fptr, skb_frag_size(frag));
 445                         pdata += skb_frag_size(frag);
 446                 }
 447
 448                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 449                 txbd->tx_bd_haddr = txr->data_mapping;
 450                 prod = NEXT_TX(prod);
 451                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 452                 memcpy(txbd, tx_push1, sizeof(*txbd));
 453                 prod = NEXT_TX(prod);
 454                 tx_push->doorbell =
 455                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 456                 txr->tx_prod = prod;
 457
 458                 tx_buf->is_push = 1;
 459                 netdev_tx_sent_queue(txq, skb->len);
 460                 wmb();  /* Sync is_push and byte queue before pushing data */
 461
 462                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 463                 if (push_len > 16) {
 464                         __iowrite64_copy(db, tx_push_buf, 16);
 465                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 466                                          (push_len - 16) << 1);
 467                 } else {
 468                         __iowrite64_copy(db, tx_push_buf, push_len);
 469                 }
 470
 471                 goto tx_done;
 472         }
 473
 474 normal_tx:
 475         if (length < BNXT_MIN_PKT_SIZE) {
 476                 pad = BNXT_MIN_PKT_SIZE - length;
 477                 if (skb_pad(skb, pad)) {
 478                         /* SKB already freed. */
 479                         tx_buf->skb = NULL;
 480                         return NETDEV_TX_OK;
 481                 }
 482                 length = BNXT_MIN_PKT_SIZE;
 483         }
 484
 485         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 486
 487         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 488                 dev_kfree_skb_any(skb);
 489                 tx_buf->skb = NULL;
 490                 return NETDEV_TX_OK;
 491         }
 492
 493         dma_unmap_addr_set(tx_buf, mapping, mapping);
 494         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 495                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 496
 497         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 498
 499         prod = NEXT_TX(prod);
 500         txbd1 = (struct tx_bd_ext *)
 501                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 502
 503         txbd1->tx_bd_hsize_lflags = 0;
 504         if (skb_is_gso(skb)) {
 505                 u32 hdr_len;
 506
 507                 if (skb->encapsulation)
 508                         hdr_len = skb_inner_network_offset(skb) +
 509                                 skb_inner_network_header_len(skb) +
 510                                 inner_tcp_hdrlen(skb);
 511                 else
 512                         hdr_len = skb_transport_offset(skb) +
 513                                 tcp_hdrlen(skb);
 514
 515                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 516                                         TX_BD_FLAGS_T_IPID |
 517                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 518                 length = skb_shinfo(skb)->gso_size;
 519                 txbd1->tx_bd_mss = cpu_to_le32(length);
 520                 length += hdr_len;
 521         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 522                 txbd1->tx_bd_hsize_lflags =
 523                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 524                 txbd1->tx_bd_mss = 0;
 525         }
 526
 527         length >>= 9;
 528         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 529                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 530                                      skb->len);
 531                 i = 0;
 532                 goto tx_dma_error;
 533         }
 534         flags |= bnxt_lhint_arr[length];
 535         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 536
 537         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 538         txbd1->tx_bd_cfa_action =
 539                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 540         for (i = 0; i < last_frag; i++) {
 541                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 542
 543                 prod = NEXT_TX(prod);
 544                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 545
 546                 len = skb_frag_size(frag);
 547                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 548                                            DMA_TO_DEVICE);
 549
 550                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 551                         goto tx_dma_error;
 552
 553                 tx_buf = &txr->tx_buf_ring[prod];
 554                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 555
 556                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 557
 558                 flags = len << TX_BD_LEN_SHIFT;
 559                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 560         }
 561
 562         flags &= ~TX_BD_LEN;
 563         txbd->tx_bd_len_flags_type =
 564                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 565                             TX_BD_FLAGS_PACKET_END);
 566
 567         netdev_tx_sent_queue(txq, skb->len);
 568
 569         /* Sync BD data before updating doorbell */
 570         wmb();
 571
 572         prod = NEXT_TX(prod);
 573         txr->tx_prod = prod;
 574
 575         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 576                 bnxt_db_write(bp, &txr->tx_db, prod);
 577
 578 tx_done:
 579
 580         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 581                 if (netdev_xmit_more() && !tx_buf->is_push)
 582                         bnxt_db_write(bp, &txr->tx_db, prod);
 583
 584                 netif_tx_stop_queue(txq);
 585
 586                 /* netif_tx_stop_queue() must be done before checking
 587                  * tx index in bnxt_tx_avail() below, because in
 588                  * bnxt_tx_int(), we update tx index before checking for
 589                  * netif_tx_queue_stopped().
 590                  */
 591                 smp_mb();
 592                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 593                         netif_tx_wake_queue(txq);
 594         }
 595         return NETDEV_TX_OK;
 596
 597 tx_dma_error:
 598         last_frag = i;
 599
 600         /* start back at beginning and unmap skb */
 601         prod = txr->tx_prod;
 602         tx_buf = &txr->tx_buf_ring[prod];
 603         tx_buf->skb = NULL;
 604         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 605                          skb_headlen(skb), PCI_DMA_TODEVICE);
 606         prod = NEXT_TX(prod);
 607
 608         /* unmap remaining mapped pages */
 609         for (i = 0; i < last_frag; i++) {
 610                 prod = NEXT_TX(prod);
 611                 tx_buf = &txr->tx_buf_ring[prod];
 612                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 613                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 614                                PCI_DMA_TODEVICE);
 615         }
 616
 617         dev_kfree_skb_any(skb);
 618         return NETDEV_TX_OK;
 619 }
 620
 621 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 622 {
 623         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 624         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 625         u16 cons = txr->tx_cons;
 626         struct pci_dev *pdev = bp->pdev;
 627         int i;
 628         unsigned int tx_bytes = 0;
 629
 630         for (i = 0; i < nr_pkts; i++) {
 631                 struct bnxt_sw_tx_bd *tx_buf;
 632                 struct sk_buff *skb;
 633                 int j, last;
 634
 635                 tx_buf = &txr->tx_buf_ring[cons];
 636                 cons = NEXT_TX(cons);
 637                 skb = tx_buf->skb;
 638                 tx_buf->skb = NULL;
 639
 640                 if (tx_buf->is_push) {
 641                         tx_buf->is_push = 0;
 642                         goto next_tx_int;
 643                 }
 644
 645                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 646                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 647                 last = tx_buf->nr_frags;
 648
 649                 for (j = 0; j < last; j++) {
 650                         cons = NEXT_TX(cons);
 651                         tx_buf = &txr->tx_buf_ring[cons];
 652                         dma_unmap_page(
 653                                 &pdev->dev,
 654                                 dma_unmap_addr(tx_buf, mapping),
 655                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 656                                 PCI_DMA_TODEVICE);
 657                 }
 658
 659 next_tx_int:
 660                 cons = NEXT_TX(cons);
 661
 662                 tx_bytes += skb->len;
 663                 dev_kfree_skb_any(skb);
 664         }
 665
 666         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 667         txr->tx_cons = cons;
 668
 669         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 670          * before checking for netif_tx_queue_stopped().  Without the
 671          * memory barrier, there is a small possibility that bnxt_start_xmit()
 672          * will miss it and cause the queue to be stopped forever.
 673          */
 674         smp_mb();
 675
 676         if (unlikely(netif_tx_queue_stopped(txq)) &&
 677             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 678                 __netif_tx_lock(txq, smp_processor_id());
 679                 if (netif_tx_queue_stopped(txq) &&
 680                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 681                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 682                         netif_tx_wake_queue(txq);
 683                 __netif_tx_unlock(txq);
 684         }
 685 }
 686
 687 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 688                                          struct bnxt_rx_ring_info *rxr,
 689                                          gfp_t gfp)
 690 {
 691         struct device *dev = &bp->pdev->dev;
 692         struct page *page;
 693
 694         page = page_pool_dev_alloc_pages(rxr->page_pool);
 695         if (!page)
 696                 return NULL;
 697
 698         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 699                                       DMA_ATTR_WEAK_ORDERING);
 700         if (dma_mapping_error(dev, *mapping)) {
 701                 page_pool_recycle_direct(rxr->page_pool, page);
 702                 return NULL;
 703         }
 704         *mapping += bp->rx_dma_offset;
 705         return page;
 706 }
 707
 708 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 709                                        gfp_t gfp)
 710 {
 711         u8 *data;
 712         struct pci_dev *pdev = bp->pdev;
 713
 714         data = kmalloc(bp->rx_buf_size, gfp);
 715         if (!data)
 716                 return NULL;
 717
 718         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 719                                         bp->rx_buf_use_size, bp->rx_dir,
 720                                         DMA_ATTR_WEAK_ORDERING);
 721
 722         if (dma_mapping_error(&pdev->dev, *mapping)) {
 723                 kfree(data);
 724                 data = NULL;
 725         }
 726         return data;
 727 }
 728
 729 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 730                        u16 prod, gfp_t gfp)
 731 {
 732         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 733         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 734         dma_addr_t mapping;
 735
 736         if (BNXT_RX_PAGE_MODE(bp)) {
 737                 struct page *page =
 738                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 739
 740                 if (!page)
 741                         return -ENOMEM;
 742
 743                 rx_buf->data = page;
 744                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 745         } else {
 746                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 747
 748                 if (!data)
 749                         return -ENOMEM;
 750
 751                 rx_buf->data = data;
 752                 rx_buf->data_ptr = data + bp->rx_offset;
 753         }
 754         rx_buf->mapping = mapping;
 755
 756         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 757         return 0;
 758 }
 759
 760 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 761 {
 762         u16 prod = rxr->rx_prod;
 763         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 764         struct rx_bd *cons_bd, *prod_bd;
 765
 766         prod_rx_buf = &rxr->rx_buf_ring[prod];
 767         cons_rx_buf = &rxr->rx_buf_ring[cons];
 768
 769         prod_rx_buf->data = data;
 770         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 771
 772         prod_rx_buf->mapping = cons_rx_buf->mapping;
 773
 774         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 775         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 776
 777         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 778 }
 779
 780 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 781 {
 782         u16 next, max = rxr->rx_agg_bmap_size;
 783
 784         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 785         if (next >= max)
 786                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 787         return next;
 788 }
 789
 790 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 791                                      struct bnxt_rx_ring_info *rxr,
 792                                      u16 prod, gfp_t gfp)
 793 {
 794         struct rx_bd *rxbd =
 795                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 796         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 797         struct pci_dev *pdev = bp->pdev;
 798         struct page *page;
 799         dma_addr_t mapping;
 800         u16 sw_prod = rxr->rx_sw_agg_prod;
 801         unsigned int offset = 0;
 802
 803         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 804                 page = rxr->rx_page;
 805                 if (!page) {
 806                         page = alloc_page(gfp);
 807                         if (!page)
 808                                 return -ENOMEM;
 809                         rxr->rx_page = page;
 810                         rxr->rx_page_offset = 0;
 811                 }
 812                 offset = rxr->rx_page_offset;
 813                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 814                 if (rxr->rx_page_offset == PAGE_SIZE)
 815                         rxr->rx_page = NULL;
 816                 else
 817                         get_page(page);
 818         } else {
 819                 page = alloc_page(gfp);
 820                 if (!page)
 821                         return -ENOMEM;
 822         }
 823
 824         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 825                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 826                                      DMA_ATTR_WEAK_ORDERING);
 827         if (dma_mapping_error(&pdev->dev, mapping)) {
 828                 __free_page(page);
 829                 return -EIO;
 830         }
 831
 832         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 833                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 834
 835         __set_bit(sw_prod, rxr->rx_agg_bmap);
 836         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 837         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 838
 839         rx_agg_buf->page = page;
 840         rx_agg_buf->offset = offset;
 841         rx_agg_buf->mapping = mapping;
 842         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 843         rxbd->rx_bd_opaque = sw_prod;
 844         return 0;
 845 }
 846
 847 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 848                                        struct bnxt_cp_ring_info *cpr,
 849                                        u16 cp_cons, u16 curr)
 850 {
 851         struct rx_agg_cmp *agg;
 852
 853         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 854         agg = (struct rx_agg_cmp *)
 855                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 856         return agg;
 857 }
 858
 859 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 860                                               struct bnxt_rx_ring_info *rxr,
 861                                               u16 agg_id, u16 curr)
 862 {
 863         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 864
 865         return &tpa_info->agg_arr[curr];
 866 }
 867
 868 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 869                                    u16 start, u32 agg_bufs, bool tpa)
 870 {
 871         struct bnxt_napi *bnapi = cpr->bnapi;
 872         struct bnxt *bp = bnapi->bp;
 873         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 874         u16 prod = rxr->rx_agg_prod;
 875         u16 sw_prod = rxr->rx_sw_agg_prod;
 876         bool p5_tpa = false;
 877         u32 i;
 878
 879         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 880                 p5_tpa = true;
 881
 882         for (i = 0; i < agg_bufs; i++) {
 883                 u16 cons;
 884                 struct rx_agg_cmp *agg;
 885                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 886                 struct rx_bd *prod_bd;
 887                 struct page *page;
 888
 889                 if (p5_tpa)
 890                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 891                 else
 892                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 893                 cons = agg->rx_agg_cmp_opaque;
 894                 __clear_bit(cons, rxr->rx_agg_bmap);
 895
 896                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 897                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 898
 899                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 900                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 901                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 902
 903                 /* It is possible for sw_prod to be equal to cons, so
 904                  * set cons_rx_buf->page to NULL first.
 905                  */
 906                 page = cons_rx_buf->page;
 907                 cons_rx_buf->page = NULL;
 908                 prod_rx_buf->page = page;
 909                 prod_rx_buf->offset = cons_rx_buf->offset;
 910
 911                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 912
 913                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 914
 915                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 916                 prod_bd->rx_bd_opaque = sw_prod;
 917
 918                 prod = NEXT_RX_AGG(prod);
 919                 sw_prod = NEXT_RX_AGG(sw_prod);
 920         }
 921         rxr->rx_agg_prod = prod;
 922         rxr->rx_sw_agg_prod = sw_prod;
 923 }
 924
 925 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 926                                         struct bnxt_rx_ring_info *rxr,
 927                                         u16 cons, void *data, u8 *data_ptr,
 928                                         dma_addr_t dma_addr,
 929                                         unsigned int offset_and_len)
 930 {
 931         unsigned int payload = offset_and_len >> 16;
 932         unsigned int len = offset_and_len & 0xffff;
 933         skb_frag_t *frag;
 934         struct page *page = data;
 935         u16 prod = rxr->rx_prod;
 936         struct sk_buff *skb;
 937         int off, err;
 938
 939         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 940         if (unlikely(err)) {
 941                 bnxt_reuse_rx_data(rxr, cons, data);
 942                 return NULL;
 943         }
 944         dma_addr -= bp->rx_dma_offset;
 945         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 946                              DMA_ATTR_WEAK_ORDERING);
 947         page_pool_release_page(rxr->page_pool, page);
 948
 949         if (unlikely(!payload))
 950                 payload = eth_get_headlen(bp->dev, data_ptr, len);
 951
 952         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 953         if (!skb) {
 954                 __free_page(page);
 955                 return NULL;
 956         }
 957
 958         off = (void *)data_ptr - page_address(page);
 959         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 960         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 961                payload + NET_IP_ALIGN);
 962
 963         frag = &skb_shinfo(skb)->frags[0];
 964         skb_frag_size_sub(frag, payload);
 965         skb_frag_off_add(frag, payload);
 966         skb->data_len -= payload;
 967         skb->tail += payload;
 968
 969         return skb;
 970 }
 971
 972 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 973                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 974                                    void *data, u8 *data_ptr,
 975                                    dma_addr_t dma_addr,
 976                                    unsigned int offset_and_len)
 977 {
 978         u16 prod = rxr->rx_prod;
 979         struct sk_buff *skb;
 980         int err;
 981
 982         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 983         if (unlikely(err)) {
 984                 bnxt_reuse_rx_data(rxr, cons, data);
 985                 return NULL;
 986         }
 987
 988         skb = build_skb(data, 0);
 989         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 990                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 991         if (!skb) {
 992                 kfree(data);
 993                 return NULL;
 994         }
 995
 996         skb_reserve(skb, bp->rx_offset);
 997         skb_put(skb, offset_and_len & 0xffff);
 998         return skb;
 999 }
1000
1001 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
1002                                      struct bnxt_cp_ring_info *cpr,
1003                                      struct sk_buff *skb, u16 idx,
1004                                      u32 agg_bufs, bool tpa)
1005 {
1006         struct bnxt_napi *bnapi = cpr->bnapi;
1007         struct pci_dev *pdev = bp->pdev;
1008         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1009         u16 prod = rxr->rx_agg_prod;
1010         bool p5_tpa = false;
1011         u32 i;
1012
1013         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1014                 p5_tpa = true;
1015
1016         for (i = 0; i < agg_bufs; i++) {
1017                 u16 cons, frag_len;
1018                 struct rx_agg_cmp *agg;
1019                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1020                 struct page *page;
1021                 dma_addr_t mapping;
1022
1023                 if (p5_tpa)
1024                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1025                 else
1026                         agg = bnxt_get_agg(bp, cpr, idx, i);
1027                 cons = agg->rx_agg_cmp_opaque;
1028                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1029                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1030
1031                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1032                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1033                                    cons_rx_buf->offset, frag_len);
1034                 __clear_bit(cons, rxr->rx_agg_bmap);
1035
1036                 /* It is possible for bnxt_alloc_rx_page() to allocate
1037                  * a sw_prod index that equals the cons index, so we
1038                  * need to clear the cons entry now.
1039                  */
1040                 mapping = cons_rx_buf->mapping;
1041                 page = cons_rx_buf->page;
1042                 cons_rx_buf->page = NULL;
1043
1044                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1045                         struct skb_shared_info *shinfo;
1046                         unsigned int nr_frags;
1047
1048                         shinfo = skb_shinfo(skb);
1049                         nr_frags = --shinfo->nr_frags;
1050                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1051
1052                         dev_kfree_skb(skb);
1053
1054                         cons_rx_buf->page = page;
1055
1056                         /* Update prod since possibly some pages have been
1057                          * allocated already.
1058                          */
1059                         rxr->rx_agg_prod = prod;
1060                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1061                         return NULL;
1062                 }
1063
1064                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1065                                      PCI_DMA_FROMDEVICE,
1066                                      DMA_ATTR_WEAK_ORDERING);
1067
1068                 skb->data_len += frag_len;
1069                 skb->len += frag_len;
1070                 skb->truesize += PAGE_SIZE;
1071
1072                 prod = NEXT_RX_AGG(prod);
1073         }
1074         rxr->rx_agg_prod = prod;
1075         return skb;
1076 }
1077
1078 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1079                                u8 agg_bufs, u32 *raw_cons)
1080 {
1081         u16 last;
1082         struct rx_agg_cmp *agg;
1083
1084         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1085         last = RING_CMP(*raw_cons);
1086         agg = (struct rx_agg_cmp *)
1087                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1088         return RX_AGG_CMP_VALID(agg, *raw_cons);
1089 }
1090
1091 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1092                                             unsigned int len,
1093                                             dma_addr_t mapping)
1094 {
1095         struct bnxt *bp = bnapi->bp;
1096         struct pci_dev *pdev = bp->pdev;
1097         struct sk_buff *skb;
1098
1099         skb = napi_alloc_skb(&bnapi->napi, len);
1100         if (!skb)
1101                 return NULL;
1102
1103         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1104                                 bp->rx_dir);
1105
1106         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1107                len + NET_IP_ALIGN);
1108
1109         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1110                                    bp->rx_dir);
1111
1112         skb_put(skb, len);
1113         return skb;
1114 }
1115
1116 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1117                            u32 *raw_cons, void *cmp)
1118 {
1119         struct rx_cmp *rxcmp = cmp;
1120         u32 tmp_raw_cons = *raw_cons;
1121         u8 cmp_type, agg_bufs = 0;
1122
1123         cmp_type = RX_CMP_TYPE(rxcmp);
1124
1125         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1126                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1127                             RX_CMP_AGG_BUFS) >>
1128                            RX_CMP_AGG_BUFS_SHIFT;
1129         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1130                 struct rx_tpa_end_cmp *tpa_end = cmp;
1131
1132                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1133                         return 0;
1134
1135                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1136         }
1137
1138         if (agg_bufs) {
1139                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1140                         return -EBUSY;
1141         }
1142         *raw_cons = tmp_raw_cons;
1143         return 0;
1144 }
1145
1146 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1147 {
1148         if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1149                 return;
1150
1151         if (BNXT_PF(bp))
1152                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1153         else
1154                 schedule_delayed_work(&bp->fw_reset_task, delay);
1155 }
1156
1157 static void bnxt_queue_sp_work(struct bnxt *bp)
1158 {
1159         if (BNXT_PF(bp))
1160                 queue_work(bnxt_pf_wq, &bp->sp_task);
1161         else
1162                 schedule_work(&bp->sp_task);
1163 }
1164
1165 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1166 {
1167         if (!rxr->bnapi->in_reset) {
1168                 rxr->bnapi->in_reset = true;
1169                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1170                         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1171                 else
1172                         set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
1173                 bnxt_queue_sp_work(bp);
1174         }
1175         rxr->rx_next_cons = 0xffff;
1176 }
1177
1178 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1179 {
1180         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1181         u16 idx = agg_id & MAX_TPA_P5_MASK;
1182
1183         if (test_bit(idx, map->agg_idx_bmap))
1184                 idx = find_first_zero_bit(map->agg_idx_bmap,
1185                                           BNXT_AGG_IDX_BMAP_SIZE);
1186         __set_bit(idx, map->agg_idx_bmap);
1187         map->agg_id_tbl[agg_id] = idx;
1188         return idx;
1189 }
1190
1191 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1192 {
1193         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1194
1195         __clear_bit(idx, map->agg_idx_bmap);
1196 }
1197
1198 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1199 {
1200         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1201
1202         return map->agg_id_tbl[agg_id];
1203 }
1204
1205 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1206                            struct rx_tpa_start_cmp *tpa_start,
1207                            struct rx_tpa_start_cmp_ext *tpa_start1)
1208 {
1209         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1210         struct bnxt_tpa_info *tpa_info;
1211         u16 cons, prod, agg_id;
1212         struct rx_bd *prod_bd;
1213         dma_addr_t mapping;
1214
1215         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1216                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1217                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1218         } else {
1219                 agg_id = TPA_START_AGG_ID(tpa_start);
1220         }
1221         cons = tpa_start->rx_tpa_start_cmp_opaque;
1222         prod = rxr->rx_prod;
1223         cons_rx_buf = &rxr->rx_buf_ring[cons];
1224         prod_rx_buf = &rxr->rx_buf_ring[prod];
1225         tpa_info = &rxr->rx_tpa[agg_id];
1226
1227         if (unlikely(cons != rxr->rx_next_cons ||
1228                      TPA_START_ERROR(tpa_start))) {
1229                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1230                             cons, rxr->rx_next_cons,
1231                             TPA_START_ERROR_CODE(tpa_start1));
1232                 bnxt_sched_reset(bp, rxr);
1233                 return;
1234         }
1235         /* Store cfa_code in tpa_info to use in tpa_end
1236          * completion processing.
1237          */
1238         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1239         prod_rx_buf->data = tpa_info->data;
1240         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1241
1242         mapping = tpa_info->mapping;
1243         prod_rx_buf->mapping = mapping;
1244
1245         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1246
1247         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1248
1249         tpa_info->data = cons_rx_buf->data;
1250         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1251         cons_rx_buf->data = NULL;
1252         tpa_info->mapping = cons_rx_buf->mapping;
1253
1254         tpa_info->len =
1255                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1256                                 RX_TPA_START_CMP_LEN_SHIFT;
1257         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1258                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1259
1260                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1261                 tpa_info->gso_type = SKB_GSO_TCPV4;
1262                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1263                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1264                         tpa_info->gso_type = SKB_GSO_TCPV6;
1265                 tpa_info->rss_hash =
1266                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1267         } else {
1268                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1269                 tpa_info->gso_type = 0;
1270                 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1271         }
1272         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1273         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1274         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1275         tpa_info->agg_count = 0;
1276
1277         rxr->rx_prod = NEXT_RX(prod);
1278         cons = NEXT_RX(cons);
1279         rxr->rx_next_cons = NEXT_RX(cons);
1280         cons_rx_buf = &rxr->rx_buf_ring[cons];
1281
1282         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1283         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1284         cons_rx_buf->data = NULL;
1285 }
1286
1287 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1288 {
1289         if (agg_bufs)
1290                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1291 }
1292
1293 #ifdef CONFIG_INET
1294 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1295 {
1296         struct udphdr *uh = NULL;
1297
1298         if (ip_proto == htons(ETH_P_IP)) {
1299                 struct iphdr *iph = (struct iphdr *)skb->data;
1300
1301                 if (iph->protocol == IPPROTO_UDP)
1302                         uh = (struct udphdr *)(iph + 1);
1303         } else {
1304                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1305
1306                 if (iph->nexthdr == IPPROTO_UDP)
1307                         uh = (struct udphdr *)(iph + 1);
1308         }
1309         if (uh) {
1310                 if (uh->check)
1311                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1312                 else
1313                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1314         }
1315 }
1316 #endif
1317
1318 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1319                                            int payload_off, int tcp_ts,
1320                                            struct sk_buff *skb)
1321 {
1322 #ifdef CONFIG_INET
1323         struct tcphdr *th;
1324         int len, nw_off;
1325         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1326         u32 hdr_info = tpa_info->hdr_info;
1327         bool loopback = false;
1328
1329         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1330         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1331         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1332
1333         /* If the packet is an internal loopback packet, the offsets will
1334          * have an extra 4 bytes.
1335          */
1336         if (inner_mac_off == 4) {
1337                 loopback = true;
1338         } else if (inner_mac_off > 4) {
1339                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1340                                             ETH_HLEN - 2));
1341
1342                 /* We only support inner iPv4/ipv6.  If we don't see the
1343                  * correct protocol ID, it must be a loopback packet where
1344                  * the offsets are off by 4.
1345                  */
1346                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1347                         loopback = true;
1348         }
1349         if (loopback) {
1350                 /* internal loopback packet, subtract all offsets by 4 */
1351                 inner_ip_off -= 4;
1352                 inner_mac_off -= 4;
1353                 outer_ip_off -= 4;
1354         }
1355
1356         nw_off = inner_ip_off - ETH_HLEN;
1357         skb_set_network_header(skb, nw_off);
1358         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1359                 struct ipv6hdr *iph = ipv6_hdr(skb);
1360
1361                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1362                 len = skb->len - skb_transport_offset(skb);
1363                 th = tcp_hdr(skb);
1364                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1365         } else {
1366                 struct iphdr *iph = ip_hdr(skb);
1367
1368                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1369                 len = skb->len - skb_transport_offset(skb);
1370                 th = tcp_hdr(skb);
1371                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1372         }
1373
1374         if (inner_mac_off) { /* tunnel */
1375                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1376                                             ETH_HLEN - 2));
1377
1378                 bnxt_gro_tunnel(skb, proto);
1379         }
1380 #endif
1381         return skb;
1382 }
1383
1384 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1385                                            int payload_off, int tcp_ts,
1386                                            struct sk_buff *skb)
1387 {
1388 #ifdef CONFIG_INET
1389         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1390         u32 hdr_info = tpa_info->hdr_info;
1391         int iphdr_len, nw_off;
1392
1393         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1394         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1395         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1396
1397         nw_off = inner_ip_off - ETH_HLEN;
1398         skb_set_network_header(skb, nw_off);
1399         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1400                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1401         skb_set_transport_header(skb, nw_off + iphdr_len);
1402
1403         if (inner_mac_off) { /* tunnel */
1404                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1405                                             ETH_HLEN - 2));
1406
1407                 bnxt_gro_tunnel(skb, proto);
1408         }
1409 #endif
1410         return skb;
1411 }
1412
1413 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1414 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1415
1416 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1417                                            int payload_off, int tcp_ts,
1418                                            struct sk_buff *skb)
1419 {
1420 #ifdef CONFIG_INET
1421         struct tcphdr *th;
1422         int len, nw_off, tcp_opt_len = 0;
1423
1424         if (tcp_ts)
1425                 tcp_opt_len = 12;
1426
1427         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1428                 struct iphdr *iph;
1429
1430                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1431                          ETH_HLEN;
1432                 skb_set_network_header(skb, nw_off);
1433                 iph = ip_hdr(skb);
1434                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1435                 len = skb->len - skb_transport_offset(skb);
1436                 th = tcp_hdr(skb);
1437                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1438         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1439                 struct ipv6hdr *iph;
1440
1441                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1442                          ETH_HLEN;
1443                 skb_set_network_header(skb, nw_off);
1444                 iph = ipv6_hdr(skb);
1445                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1446                 len = skb->len - skb_transport_offset(skb);
1447                 th = tcp_hdr(skb);
1448                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1449         } else {
1450                 dev_kfree_skb_any(skb);
1451                 return NULL;
1452         }
1453
1454         if (nw_off) /* tunnel */
1455                 bnxt_gro_tunnel(skb, skb->protocol);
1456 #endif
1457         return skb;
1458 }
1459
1460 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1461                                            struct bnxt_tpa_info *tpa_info,
1462                                            struct rx_tpa_end_cmp *tpa_end,
1463                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1464                                            struct sk_buff *skb)
1465 {
1466 #ifdef CONFIG_INET
1467         int payload_off;
1468         u16 segs;
1469
1470         segs = TPA_END_TPA_SEGS(tpa_end);
1471         if (segs == 1)
1472                 return skb;
1473
1474         NAPI_GRO_CB(skb)->count = segs;
1475         skb_shinfo(skb)->gso_size =
1476                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1477         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1478         if (bp->flags & BNXT_FLAG_CHIP_P5)
1479                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1480         else
1481                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1482         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1483         if (likely(skb))
1484                 tcp_gro_complete(skb);
1485 #endif
1486         return skb;
1487 }
1488
1489 /* Given the cfa_code of a received packet determine which
1490  * netdev (vf-rep or PF) the packet is destined to.
1491  */
1492 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1493 {
1494         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1495
1496         /* if vf-rep dev is NULL, the must belongs to the PF */
1497         return dev ? dev : bp->dev;
1498 }
1499
1500 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1501                                            struct bnxt_cp_ring_info *cpr,
1502                                            u32 *raw_cons,
1503                                            struct rx_tpa_end_cmp *tpa_end,
1504                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1505                                            u8 *event)
1506 {
1507         struct bnxt_napi *bnapi = cpr->bnapi;
1508         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1509         u8 *data_ptr, agg_bufs;
1510         unsigned int len;
1511         struct bnxt_tpa_info *tpa_info;
1512         dma_addr_t mapping;
1513         struct sk_buff *skb;
1514         u16 idx = 0, agg_id;
1515         void *data;
1516         bool gro;
1517
1518         if (unlikely(bnapi->in_reset)) {
1519                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1520
1521                 if (rc < 0)
1522                         return ERR_PTR(-EBUSY);
1523                 return NULL;
1524         }
1525
1526         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1527                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1528                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1529                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1530                 tpa_info = &rxr->rx_tpa[agg_id];
1531                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1532                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1533                                     agg_bufs, tpa_info->agg_count);
1534                         agg_bufs = tpa_info->agg_count;
1535                 }
1536                 tpa_info->agg_count = 0;
1537                 *event |= BNXT_AGG_EVENT;
1538                 bnxt_free_agg_idx(rxr, agg_id);
1539                 idx = agg_id;
1540                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1541         } else {
1542                 agg_id = TPA_END_AGG_ID(tpa_end);
1543                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1544                 tpa_info = &rxr->rx_tpa[agg_id];
1545                 idx = RING_CMP(*raw_cons);
1546                 if (agg_bufs) {
1547                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1548                                 return ERR_PTR(-EBUSY);
1549
1550                         *event |= BNXT_AGG_EVENT;
1551                         idx = NEXT_CMP(idx);
1552                 }
1553                 gro = !!TPA_END_GRO(tpa_end);
1554         }
1555         data = tpa_info->data;
1556         data_ptr = tpa_info->data_ptr;
1557         prefetch(data_ptr);
1558         len = tpa_info->len;
1559         mapping = tpa_info->mapping;
1560
1561         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1562                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1563                 if (agg_bufs > MAX_SKB_FRAGS)
1564                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1565                                     agg_bufs, (int)MAX_SKB_FRAGS);
1566                 return NULL;
1567         }
1568
1569         if (len <= bp->rx_copy_thresh) {
1570                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1571                 if (!skb) {
1572                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1573                         return NULL;
1574                 }
1575         } else {
1576                 u8 *new_data;
1577                 dma_addr_t new_mapping;
1578
1579                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1580                 if (!new_data) {
1581                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1582                         return NULL;
1583                 }
1584
1585                 tpa_info->data = new_data;
1586                 tpa_info->data_ptr = new_data + bp->rx_offset;
1587                 tpa_info->mapping = new_mapping;
1588
1589                 skb = build_skb(data, 0);
1590                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1591                                        bp->rx_buf_use_size, bp->rx_dir,
1592                                        DMA_ATTR_WEAK_ORDERING);
1593
1594                 if (!skb) {
1595                         kfree(data);
1596                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1597                         return NULL;
1598                 }
1599                 skb_reserve(skb, bp->rx_offset);
1600                 skb_put(skb, len);
1601         }
1602
1603         if (agg_bufs) {
1604                 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1605                 if (!skb) {
1606                         /* Page reuse already handled by bnxt_rx_pages(). */
1607                         return NULL;
1608                 }
1609         }
1610
1611         skb->protocol =
1612                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1613
1614         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1615                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1616
1617         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1618             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1619                 u16 vlan_proto = tpa_info->metadata >>
1620                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1621                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1622
1623                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1624         }
1625
1626         skb_checksum_none_assert(skb);
1627         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1628                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1629                 skb->csum_level =
1630                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1631         }
1632
1633         if (gro)
1634                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1635
1636         return skb;
1637 }
1638
1639 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1640                          struct rx_agg_cmp *rx_agg)
1641 {
1642         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1643         struct bnxt_tpa_info *tpa_info;
1644
1645         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1646         tpa_info = &rxr->rx_tpa[agg_id];
1647         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1648         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1649 }
1650
1651 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1652                              struct sk_buff *skb)
1653 {
1654         if (skb->dev != bp->dev) {
1655                 /* this packet belongs to a vf-rep */
1656                 bnxt_vf_rep_rx(bp, skb);
1657                 return;
1658         }
1659         skb_record_rx_queue(skb, bnapi->index);
1660         napi_gro_receive(&bnapi->napi, skb);
1661 }
1662
1663 /* returns the following:
1664  * 1       - 1 packet successfully received
1665  * 0       - successful TPA_START, packet not completed yet
1666  * -EBUSY  - completion ring does not have all the agg buffers yet
1667  * -ENOMEM - packet aborted due to out of memory
1668  * -EIO    - packet aborted due to hw error indicated in BD
1669  */
1670 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1671                        u32 *raw_cons, u8 *event)
1672 {
1673         struct bnxt_napi *bnapi = cpr->bnapi;
1674         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1675         struct net_device *dev = bp->dev;
1676         struct rx_cmp *rxcmp;
1677         struct rx_cmp_ext *rxcmp1;
1678         u32 tmp_raw_cons = *raw_cons;
1679         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1680         struct bnxt_sw_rx_bd *rx_buf;
1681         unsigned int len;
1682         u8 *data_ptr, agg_bufs, cmp_type;
1683         dma_addr_t dma_addr;
1684         struct sk_buff *skb;
1685         void *data;
1686         int rc = 0;
1687         u32 misc;
1688
1689         rxcmp = (struct rx_cmp *)
1690                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1691
1692         cmp_type = RX_CMP_TYPE(rxcmp);
1693
1694         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1695                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1696                 goto next_rx_no_prod_no_len;
1697         }
1698
1699         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1700         cp_cons = RING_CMP(tmp_raw_cons);
1701         rxcmp1 = (struct rx_cmp_ext *)
1702                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1703
1704         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1705                 return -EBUSY;
1706
1707         prod = rxr->rx_prod;
1708
1709         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1710                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1711                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1712
1713                 *event |= BNXT_RX_EVENT;
1714                 goto next_rx_no_prod_no_len;
1715
1716         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1717                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1718                                    (struct rx_tpa_end_cmp *)rxcmp,
1719                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1720
1721                 if (IS_ERR(skb))
1722                         return -EBUSY;
1723
1724                 rc = -ENOMEM;
1725                 if (likely(skb)) {
1726                         bnxt_deliver_skb(bp, bnapi, skb);
1727                         rc = 1;
1728                 }
1729                 *event |= BNXT_RX_EVENT;
1730                 goto next_rx_no_prod_no_len;
1731         }
1732
1733         cons = rxcmp->rx_cmp_opaque;
1734         if (unlikely(cons != rxr->rx_next_cons)) {
1735                 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1736
1737                 /* 0xffff is forced error, don't print it */
1738                 if (rxr->rx_next_cons != 0xffff)
1739                         netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1740                                     cons, rxr->rx_next_cons);
1741                 bnxt_sched_reset(bp, rxr);
1742                 return rc1;
1743         }
1744         rx_buf = &rxr->rx_buf_ring[cons];
1745         data = rx_buf->data;
1746         data_ptr = rx_buf->data_ptr;
1747         prefetch(data_ptr);
1748
1749         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1750         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1751
1752         if (agg_bufs) {
1753                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1754                         return -EBUSY;
1755
1756                 cp_cons = NEXT_CMP(cp_cons);
1757                 *event |= BNXT_AGG_EVENT;
1758         }
1759         *event |= BNXT_RX_EVENT;
1760
1761         rx_buf->data = NULL;
1762         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1763                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1764
1765                 bnxt_reuse_rx_data(rxr, cons, data);
1766                 if (agg_bufs)
1767                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1768                                                false);
1769
1770                 rc = -EIO;
1771                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1772                         bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1773                         if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1774                             !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1775                                 netdev_warn_once(bp->dev, "RX buffer error %x\n",
1776                                                  rx_err);
1777                                 bnxt_sched_reset(bp, rxr);
1778                         }
1779                 }
1780                 goto next_rx_no_len;
1781         }
1782
1783         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1784         dma_addr = rx_buf->mapping;
1785
1786         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1787                 rc = 1;
1788                 goto next_rx;
1789         }
1790
1791         if (len <= bp->rx_copy_thresh) {
1792                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1793                 bnxt_reuse_rx_data(rxr, cons, data);
1794                 if (!skb) {
1795                         if (agg_bufs)
1796                                 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1797                                                        agg_bufs, false);
1798                         rc = -ENOMEM;
1799                         goto next_rx;
1800                 }
1801         } else {
1802                 u32 payload;
1803
1804                 if (rx_buf->data_ptr == data_ptr)
1805                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1806                 else
1807                         payload = 0;
1808                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1809                                       payload | len);
1810                 if (!skb) {
1811                         rc = -ENOMEM;
1812                         goto next_rx;
1813                 }
1814         }
1815
1816         if (agg_bufs) {
1817                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1818                 if (!skb) {
1819                         rc = -ENOMEM;
1820                         goto next_rx;
1821                 }
1822         }
1823
1824         if (RX_CMP_HASH_VALID(rxcmp)) {
1825                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1826                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1827
1828                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1829                 if (hash_type != 1 && hash_type != 3)
1830                         type = PKT_HASH_TYPE_L3;
1831                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1832         }
1833
1834         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1835         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1836
1837         if ((rxcmp1->rx_cmp_flags2 &
1838              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1839             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1840                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1841                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1842                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1843
1844                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1845         }
1846
1847         skb_checksum_none_assert(skb);
1848         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1849                 if (dev->features & NETIF_F_RXCSUM) {
1850                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1851                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1852                 }
1853         } else {
1854                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1855                         if (dev->features & NETIF_F_RXCSUM)
1856                                 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
1857                 }
1858         }
1859
1860         bnxt_deliver_skb(bp, bnapi, skb);
1861         rc = 1;
1862
1863 next_rx:
1864         cpr->rx_packets += 1;
1865         cpr->rx_bytes += len;
1866
1867 next_rx_no_len:
1868         rxr->rx_prod = NEXT_RX(prod);
1869         rxr->rx_next_cons = NEXT_RX(cons);
1870
1871 next_rx_no_prod_no_len:
1872         *raw_cons = tmp_raw_cons;
1873
1874         return rc;
1875 }
1876
1877 /* In netpoll mode, if we are using a combined completion ring, we need to
1878  * discard the rx packets and recycle the buffers.
1879  */
1880 static int bnxt_force_rx_discard(struct bnxt *bp,
1881                                  struct bnxt_cp_ring_info *cpr,
1882                                  u32 *raw_cons, u8 *event)
1883 {
1884         u32 tmp_raw_cons = *raw_cons;
1885         struct rx_cmp_ext *rxcmp1;
1886         struct rx_cmp *rxcmp;
1887         u16 cp_cons;
1888         u8 cmp_type;
1889
1890         cp_cons = RING_CMP(tmp_raw_cons);
1891         rxcmp = (struct rx_cmp *)
1892                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1893
1894         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1895         cp_cons = RING_CMP(tmp_raw_cons);
1896         rxcmp1 = (struct rx_cmp_ext *)
1897                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1898
1899         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1900                 return -EBUSY;
1901
1902         cmp_type = RX_CMP_TYPE(rxcmp);
1903         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1904                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1905                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1906         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1907                 struct rx_tpa_end_cmp_ext *tpa_end1;
1908
1909                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1910                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1911                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1912         }
1913         return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1914 }
1915
1916 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1917 {
1918         struct bnxt_fw_health *fw_health = bp->fw_health;
1919         u32 reg = fw_health->regs[reg_idx];
1920         u32 reg_type, reg_off, val = 0;
1921
1922         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1923         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1924         switch (reg_type) {
1925         case BNXT_FW_HEALTH_REG_TYPE_CFG:
1926                 pci_read_config_dword(bp->pdev, reg_off, &val);
1927                 break;
1928         case BNXT_FW_HEALTH_REG_TYPE_GRC:
1929                 reg_off = fw_health->mapped_regs[reg_idx];
1930                 fallthrough;
1931         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1932                 val = readl(bp->bar0 + reg_off);
1933                 break;
1934         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1935                 val = readl(bp->bar1 + reg_off);
1936                 break;
1937         }
1938         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1939                 val &= fw_health->fw_reset_inprog_reg_mask;
1940         return val;
1941 }
1942
1943 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
1944 {
1945         int i;
1946
1947         for (i = 0; i < bp->rx_nr_rings; i++) {
1948                 u16 grp_idx = bp->rx_ring[i].bnapi->index;
1949                 struct bnxt_ring_grp_info *grp_info;
1950
1951                 grp_info = &bp->grp_info[grp_idx];
1952                 if (grp_info->agg_fw_ring_id == ring_id)
1953                         return grp_idx;
1954         }
1955         return INVALID_HW_RING_ID;
1956 }
1957
1958 #define BNXT_GET_EVENT_PORT(data)       \
1959         ((data) &                       \
1960          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1961
1962 #define BNXT_EVENT_RING_TYPE(data2)     \
1963         ((data2) &                      \
1964          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
1965
1966 #define BNXT_EVENT_RING_TYPE_RX(data2)  \
1967         (BNXT_EVENT_RING_TYPE(data2) == \
1968          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
1969
1970 static int bnxt_async_event_process(struct bnxt *bp,
1971                                     struct hwrm_async_event_cmpl *cmpl)
1972 {
1973         u16 event_id = le16_to_cpu(cmpl->event_id);
1974         u32 data1 = le32_to_cpu(cmpl->event_data1);
1975         u32 data2 = le32_to_cpu(cmpl->event_data2);
1976
1977         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1978         switch (event_id) {
1979         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1980                 struct bnxt_link_info *link_info = &bp->link_info;
1981
1982                 if (BNXT_VF(bp))
1983                         goto async_event_process_exit;
1984
1985                 /* print unsupported speed warning in forced speed mode only */
1986                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1987                     (data1 & 0x20000)) {
1988                         u16 fw_speed = link_info->force_link_speed;
1989                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1990
1991                         if (speed != SPEED_UNKNOWN)
1992                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1993                                             speed);
1994                 }
1995                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1996         }
1997                 fallthrough;
1998         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
1999         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2000                 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2001                 fallthrough;
2002         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2003                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2004                 break;
2005         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2006                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2007                 break;
2008         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2009                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2010
2011                 if (BNXT_VF(bp))
2012                         break;
2013
2014                 if (bp->pf.port_id != port_id)
2015                         break;
2016
2017                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2018                 break;
2019         }
2020         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2021                 if (BNXT_PF(bp))
2022                         goto async_event_process_exit;
2023                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2024                 break;
2025         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2026                 char *fatal_str = "non-fatal";
2027
2028                 if (!bp->fw_health)
2029                         goto async_event_process_exit;
2030
2031                 bp->fw_reset_timestamp = jiffies;
2032                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2033                 if (!bp->fw_reset_min_dsecs)
2034                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2035                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2036                 if (!bp->fw_reset_max_dsecs)
2037                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2038                 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2039                         fatal_str = "fatal";
2040                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2041                 }
2042                 netif_warn(bp, hw, bp->dev,
2043                            "Firmware %s reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2044                            fatal_str, data1, data2,
2045                            bp->fw_reset_min_dsecs * 100,
2046                            bp->fw_reset_max_dsecs * 100);
2047                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2048                 break;
2049         }
2050         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2051                 struct bnxt_fw_health *fw_health = bp->fw_health;
2052
2053                 if (!fw_health)
2054                         goto async_event_process_exit;
2055
2056                 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2057                 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2058                 if (!fw_health->enabled) {
2059                         netif_info(bp, drv, bp->dev,
2060                                    "Error recovery info: error recovery[0]\n");
2061                         break;
2062                 }
2063                 fw_health->tmr_multiplier =
2064                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2065                                      bp->current_interval * 10);
2066                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2067                 fw_health->last_fw_heartbeat =
2068                         bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2069                 fw_health->last_fw_reset_cnt =
2070                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2071                 netif_info(bp, drv, bp->dev,
2072                            "Error recovery info: error recovery[1], master[%d], reset count[%u], health status: 0x%x\n",
2073                            fw_health->master, fw_health->last_fw_reset_cnt,
2074                            bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG));
2075                 goto async_event_process_exit;
2076         }
2077         case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2078                 netif_notice(bp, hw, bp->dev,
2079                              "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2080                              data1, data2);
2081                 goto async_event_process_exit;
2082         case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2083                 struct bnxt_rx_ring_info *rxr;
2084                 u16 grp_idx;
2085
2086                 if (bp->flags & BNXT_FLAG_CHIP_P5)
2087                         goto async_event_process_exit;
2088
2089                 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2090                             BNXT_EVENT_RING_TYPE(data2), data1);
2091                 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2092                         goto async_event_process_exit;
2093
2094                 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2095                 if (grp_idx == INVALID_HW_RING_ID) {
2096                         netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2097                                     data1);
2098                         goto async_event_process_exit;
2099                 }
2100                 rxr = bp->bnapi[grp_idx]->rx_ring;
2101                 bnxt_sched_reset(bp, rxr);
2102                 goto async_event_process_exit;
2103         }
2104         case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2105                 struct bnxt_fw_health *fw_health = bp->fw_health;
2106
2107                 netif_notice(bp, hw, bp->dev,
2108                              "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2109                              data1, data2);
2110                 if (fw_health) {
2111                         fw_health->echo_req_data1 = data1;
2112                         fw_health->echo_req_data2 = data2;
2113                         set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2114                         break;
2115                 }
2116                 goto async_event_process_exit;
2117         }
2118         default:
2119                 goto async_event_process_exit;
2120         }
2121         bnxt_queue_sp_work(bp);
2122 async_event_process_exit:
2123         bnxt_ulp_async_events(bp, cmpl);
2124         return 0;
2125 }
2126
2127 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2128 {
2129         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2130         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2131         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2132                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2133
2134         switch (cmpl_type) {
2135         case CMPL_BASE_TYPE_HWRM_DONE:
2136                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2137                 if (seq_id == bp->hwrm_intr_seq_id)
2138                         bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2139                 else
2140                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2141                 break;
2142
2143         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2144                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2145
2146                 if ((vf_id < bp->pf.first_vf_id) ||
2147                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2148                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2149                                    vf_id);
2150                         return -EINVAL;
2151                 }
2152
2153                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2154                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2155                 bnxt_queue_sp_work(bp);
2156                 break;
2157
2158         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2159                 bnxt_async_event_process(bp,
2160                                          (struct hwrm_async_event_cmpl *)txcmp);
2161
2162         default:
2163                 break;
2164         }
2165
2166         return 0;
2167 }
2168
2169 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2170 {
2171         struct bnxt_napi *bnapi = dev_instance;
2172         struct bnxt *bp = bnapi->bp;
2173         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2174         u32 cons = RING_CMP(cpr->cp_raw_cons);
2175
2176         cpr->event_ctr++;
2177         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2178         napi_schedule(&bnapi->napi);
2179         return IRQ_HANDLED;
2180 }
2181
2182 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2183 {
2184         u32 raw_cons = cpr->cp_raw_cons;
2185         u16 cons = RING_CMP(raw_cons);
2186         struct tx_cmp *txcmp;
2187
2188         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2189
2190         return TX_CMP_VALID(txcmp, raw_cons);
2191 }
2192
2193 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2194 {
2195         struct bnxt_napi *bnapi = dev_instance;
2196         struct bnxt *bp = bnapi->bp;
2197         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2198         u32 cons = RING_CMP(cpr->cp_raw_cons);
2199         u32 int_status;
2200
2201         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2202
2203         if (!bnxt_has_work(bp, cpr)) {
2204                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2205                 /* return if erroneous interrupt */
2206                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2207                         return IRQ_NONE;
2208         }
2209
2210         /* disable ring IRQ */
2211         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2212
2213         /* Return here if interrupt is shared and is disabled. */
2214         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2215                 return IRQ_HANDLED;
2216
2217         napi_schedule(&bnapi->napi);
2218         return IRQ_HANDLED;
2219 }
2220
2221 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2222                             int budget)
2223 {
2224         struct bnxt_napi *bnapi = cpr->bnapi;
2225         u32 raw_cons = cpr->cp_raw_cons;
2226         u32 cons;
2227         int tx_pkts = 0;
2228         int rx_pkts = 0;
2229         u8 event = 0;
2230         struct tx_cmp *txcmp;
2231
2232         cpr->has_more_work = 0;
2233         cpr->had_work_done = 1;
2234         while (1) {
2235                 int rc;
2236
2237                 cons = RING_CMP(raw_cons);
2238                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2239
2240                 if (!TX_CMP_VALID(txcmp, raw_cons))
2241                         break;
2242
2243                 /* The valid test of the entry must be done first before
2244                  * reading any further.
2245                  */
2246                 dma_rmb();
2247                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2248                         tx_pkts++;
2249                         /* return full budget so NAPI will complete. */
2250                         if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2251                                 rx_pkts = budget;
2252                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2253                                 if (budget)
2254                                         cpr->has_more_work = 1;
2255                                 break;
2256                         }
2257                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2258                         if (likely(budget))
2259                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2260                         else
2261                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2262                                                            &event);
2263                         if (likely(rc >= 0))
2264                                 rx_pkts += rc;
2265                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2266                          * the NAPI budget.  Otherwise, we may potentially loop
2267                          * here forever if we consistently cannot allocate
2268                          * buffers.
2269                          */
2270                         else if (rc == -ENOMEM && budget)
2271                                 rx_pkts++;
2272                         else if (rc == -EBUSY)  /* partial completion */
2273                                 break;
2274                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2275                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2276                                     (TX_CMP_TYPE(txcmp) ==
2277                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2278                                     (TX_CMP_TYPE(txcmp) ==
2279                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2280                         bnxt_hwrm_handler(bp, txcmp);
2281                 }
2282                 raw_cons = NEXT_RAW_CMP(raw_cons);
2283
2284                 if (rx_pkts && rx_pkts == budget) {
2285                         cpr->has_more_work = 1;
2286                         break;
2287                 }
2288         }
2289
2290         if (event & BNXT_REDIRECT_EVENT)
2291                 xdp_do_flush_map();
2292
2293         if (event & BNXT_TX_EVENT) {
2294                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2295                 u16 prod = txr->tx_prod;
2296
2297                 /* Sync BD data before updating doorbell */
2298                 wmb();
2299
2300                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2301         }
2302
2303         cpr->cp_raw_cons = raw_cons;
2304         bnapi->tx_pkts += tx_pkts;
2305         bnapi->events |= event;
2306         return rx_pkts;
2307 }
2308
2309 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2310 {
2311         if (bnapi->tx_pkts) {
2312                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2313                 bnapi->tx_pkts = 0;
2314         }
2315
2316         if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2317                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2318
2319                 if (bnapi->events & BNXT_AGG_EVENT)
2320                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2321                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2322         }
2323         bnapi->events = 0;
2324 }
2325
2326 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2327                           int budget)
2328 {
2329         struct bnxt_napi *bnapi = cpr->bnapi;
2330         int rx_pkts;
2331
2332         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2333
2334         /* ACK completion ring before freeing tx ring and producing new
2335          * buffers in rx/agg rings to prevent overflowing the completion
2336          * ring.
2337          */
2338         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2339
2340         __bnxt_poll_work_done(bp, bnapi);
2341         return rx_pkts;
2342 }
2343
2344 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2345 {
2346         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2347         struct bnxt *bp = bnapi->bp;
2348         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2349         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2350         struct tx_cmp *txcmp;
2351         struct rx_cmp_ext *rxcmp1;
2352         u32 cp_cons, tmp_raw_cons;
2353         u32 raw_cons = cpr->cp_raw_cons;
2354         u32 rx_pkts = 0;
2355         u8 event = 0;
2356
2357         while (1) {
2358                 int rc;
2359
2360                 cp_cons = RING_CMP(raw_cons);
2361                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2362
2363                 if (!TX_CMP_VALID(txcmp, raw_cons))
2364                         break;
2365
2366                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2367                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2368                         cp_cons = RING_CMP(tmp_raw_cons);
2369                         rxcmp1 = (struct rx_cmp_ext *)
2370                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2371
2372                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2373                                 break;
2374
2375                         /* force an error to recycle the buffer */
2376                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2377                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2378
2379                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2380                         if (likely(rc == -EIO) && budget)
2381                                 rx_pkts++;
2382                         else if (rc == -EBUSY)  /* partial completion */
2383                                 break;
2384                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2385                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2386                         bnxt_hwrm_handler(bp, txcmp);
2387                 } else {
2388                         netdev_err(bp->dev,
2389                                    "Invalid completion received on special ring\n");
2390                 }
2391                 raw_cons = NEXT_RAW_CMP(raw_cons);
2392
2393                 if (rx_pkts == budget)
2394                         break;
2395         }
2396
2397         cpr->cp_raw_cons = raw_cons;
2398         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2399         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2400
2401         if (event & BNXT_AGG_EVENT)
2402                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2403
2404         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2405                 napi_complete_done(napi, rx_pkts);
2406                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2407         }
2408         return rx_pkts;
2409 }
2410
2411 static int bnxt_poll(struct napi_struct *napi, int budget)
2412 {
2413         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2414         struct bnxt *bp = bnapi->bp;
2415         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2416         int work_done = 0;
2417
2418         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2419                 napi_complete(napi);
2420                 return 0;
2421         }
2422         while (1) {
2423                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2424
2425                 if (work_done >= budget) {
2426                         if (!budget)
2427                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2428                         break;
2429                 }
2430
2431                 if (!bnxt_has_work(bp, cpr)) {
2432                         if (napi_complete_done(napi, work_done))
2433                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2434                         break;
2435                 }
2436         }
2437         if (bp->flags & BNXT_FLAG_DIM) {
2438                 struct dim_sample dim_sample = {};
2439
2440                 dim_update_sample(cpr->event_ctr,
2441                                   cpr->rx_packets,
2442                                   cpr->rx_bytes,
2443                                   &dim_sample);
2444                 net_dim(&cpr->dim, dim_sample);
2445         }
2446         return work_done;
2447 }
2448
2449 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2450 {
2451         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2452         int i, work_done = 0;
2453
2454         for (i = 0; i < 2; i++) {
2455                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2456
2457                 if (cpr2) {
2458                         work_done += __bnxt_poll_work(bp, cpr2,
2459                                                       budget - work_done);
2460                         cpr->has_more_work |= cpr2->has_more_work;
2461                 }
2462         }
2463         return work_done;
2464 }
2465
2466 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2467                                  u64 dbr_type)
2468 {
2469         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2470         int i;
2471
2472         for (i = 0; i < 2; i++) {
2473                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2474                 struct bnxt_db_info *db;
2475
2476                 if (cpr2 && cpr2->had_work_done) {
2477                         db = &cpr2->cp_db;
2478                         writeq(db->db_key64 | dbr_type |
2479                                RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2480                         cpr2->had_work_done = 0;
2481                 }
2482         }
2483         __bnxt_poll_work_done(bp, bnapi);
2484 }
2485
2486 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2487 {
2488         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2489         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2490         u32 raw_cons = cpr->cp_raw_cons;
2491         struct bnxt *bp = bnapi->bp;
2492         struct nqe_cn *nqcmp;
2493         int work_done = 0;
2494         u32 cons;
2495
2496         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2497                 napi_complete(napi);
2498                 return 0;
2499         }
2500         if (cpr->has_more_work) {
2501                 cpr->has_more_work = 0;
2502                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2503         }
2504         while (1) {
2505                 cons = RING_CMP(raw_cons);
2506                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2507
2508                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2509                         if (cpr->has_more_work)
2510                                 break;
2511
2512                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2513                         cpr->cp_raw_cons = raw_cons;
2514                         if (napi_complete_done(napi, work_done))
2515                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2516                                                   cpr->cp_raw_cons);
2517                         return work_done;
2518                 }
2519
2520                 /* The valid test of the entry must be done first before
2521                  * reading any further.
2522                  */
2523                 dma_rmb();
2524
2525                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2526                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2527                         struct bnxt_cp_ring_info *cpr2;
2528
2529                         cpr2 = cpr->cp_ring_arr[idx];
2530                         work_done += __bnxt_poll_work(bp, cpr2,
2531                                                       budget - work_done);
2532                         cpr->has_more_work |= cpr2->has_more_work;
2533                 } else {
2534                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2535                 }
2536                 raw_cons = NEXT_RAW_CMP(raw_cons);
2537         }
2538         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2539         if (raw_cons != cpr->cp_raw_cons) {
2540                 cpr->cp_raw_cons = raw_cons;
2541                 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2542         }
2543         return work_done;
2544 }
2545
2546 static void bnxt_free_tx_skbs(struct bnxt *bp)
2547 {
2548         int i, max_idx;
2549         struct pci_dev *pdev = bp->pdev;
2550
2551         if (!bp->tx_ring)
2552                 return;
2553
2554         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2555         for (i = 0; i < bp->tx_nr_rings; i++) {
2556                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2557                 int j;
2558
2559                 for (j = 0; j < max_idx;) {
2560                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2561                         struct sk_buff *skb;
2562                         int k, last;
2563
2564                         if (i < bp->tx_nr_rings_xdp &&
2565                             tx_buf->action == XDP_REDIRECT) {
2566                                 dma_unmap_single(&pdev->dev,
2567                                         dma_unmap_addr(tx_buf, mapping),
2568                                         dma_unmap_len(tx_buf, len),
2569                                         PCI_DMA_TODEVICE);
2570                                 xdp_return_frame(tx_buf->xdpf);
2571                                 tx_buf->action = 0;
2572                                 tx_buf->xdpf = NULL;
2573                                 j++;
2574                                 continue;
2575                         }
2576
2577                         skb = tx_buf->skb;
2578                         if (!skb) {
2579                                 j++;
2580                                 continue;
2581                         }
2582
2583                         tx_buf->skb = NULL;
2584
2585                         if (tx_buf->is_push) {
2586                                 dev_kfree_skb(skb);
2587                                 j += 2;
2588                                 continue;
2589                         }
2590
2591                         dma_unmap_single(&pdev->dev,
2592                                          dma_unmap_addr(tx_buf, mapping),
2593                                          skb_headlen(skb),
2594                                          PCI_DMA_TODEVICE);
2595
2596                         last = tx_buf->nr_frags;
2597                         j += 2;
2598                         for (k = 0; k < last; k++, j++) {
2599                                 int ring_idx = j & bp->tx_ring_mask;
2600                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2601
2602                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2603                                 dma_unmap_page(
2604                                         &pdev->dev,
2605                                         dma_unmap_addr(tx_buf, mapping),
2606                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2607                         }
2608                         dev_kfree_skb(skb);
2609                 }
2610                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2611         }
2612 }
2613
2614 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2615 {
2616         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2617         struct pci_dev *pdev = bp->pdev;
2618         struct bnxt_tpa_idx_map *map;
2619         int i, max_idx, max_agg_idx;
2620
2621         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2622         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2623         if (!rxr->rx_tpa)
2624                 goto skip_rx_tpa_free;
2625
2626         for (i = 0; i < bp->max_tpa; i++) {
2627                 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2628                 u8 *data = tpa_info->data;
2629
2630                 if (!data)
2631                         continue;
2632
2633                 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2634                                        bp->rx_buf_use_size, bp->rx_dir,
2635                                        DMA_ATTR_WEAK_ORDERING);
2636
2637                 tpa_info->data = NULL;
2638
2639                 kfree(data);
2640         }
2641
2642 skip_rx_tpa_free:
2643         for (i = 0; i < max_idx; i++) {
2644                 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2645                 dma_addr_t mapping = rx_buf->mapping;
2646                 void *data = rx_buf->data;
2647
2648                 if (!data)
2649                         continue;
2650
2651                 rx_buf->data = NULL;
2652                 if (BNXT_RX_PAGE_MODE(bp)) {
2653                         mapping -= bp->rx_dma_offset;
2654                         dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE,
2655                                              bp->rx_dir,
2656                                              DMA_ATTR_WEAK_ORDERING);
2657                         page_pool_recycle_direct(rxr->page_pool, data);
2658                 } else {
2659                         dma_unmap_single_attrs(&pdev->dev, mapping,
2660                                                bp->rx_buf_use_size, bp->rx_dir,
2661                                                DMA_ATTR_WEAK_ORDERING);
2662                         kfree(data);
2663                 }
2664         }
2665         for (i = 0; i < max_agg_idx; i++) {
2666                 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2667                 struct page *page = rx_agg_buf->page;
2668
2669                 if (!page)
2670                         continue;
2671
2672                 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2673                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
2674                                      DMA_ATTR_WEAK_ORDERING);
2675
2676                 rx_agg_buf->page = NULL;
2677                 __clear_bit(i, rxr->rx_agg_bmap);
2678
2679                 __free_page(page);
2680         }
2681         if (rxr->rx_page) {
2682                 __free_page(rxr->rx_page);
2683                 rxr->rx_page = NULL;
2684         }
2685         map = rxr->rx_tpa_idx_map;
2686         if (map)
2687                 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2688 }
2689
2690 static void bnxt_free_rx_skbs(struct bnxt *bp)
2691 {
2692         int i;
2693
2694         if (!bp->rx_ring)
2695                 return;
2696
2697         for (i = 0; i < bp->rx_nr_rings; i++)
2698                 bnxt_free_one_rx_ring_skbs(bp, i);
2699 }
2700
2701 static void bnxt_free_skbs(struct bnxt *bp)
2702 {
2703         bnxt_free_tx_skbs(bp);
2704         bnxt_free_rx_skbs(bp);
2705 }
2706
2707 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len)
2708 {
2709         u8 init_val = mem_init->init_val;
2710         u16 offset = mem_init->offset;
2711         u8 *p2 = p;
2712         int i;
2713
2714         if (!init_val)
2715                 return;
2716         if (offset == BNXT_MEM_INVALID_OFFSET) {
2717                 memset(p, init_val, len);
2718                 return;
2719         }
2720         for (i = 0; i < len; i += mem_init->size)
2721                 *(p2 + i + offset) = init_val;
2722 }
2723
2724 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2725 {
2726         struct pci_dev *pdev = bp->pdev;
2727         int i;
2728
2729         for (i = 0; i < rmem->nr_pages; i++) {
2730                 if (!rmem->pg_arr[i])
2731                         continue;
2732
2733                 dma_free_coherent(&pdev->dev, rmem->page_size,
2734                                   rmem->pg_arr[i], rmem->dma_arr[i]);
2735
2736                 rmem->pg_arr[i] = NULL;
2737         }
2738         if (rmem->pg_tbl) {
2739                 size_t pg_tbl_size = rmem->nr_pages * 8;
2740
2741                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2742                         pg_tbl_size = rmem->page_size;
2743                 dma_free_coherent(&pdev->dev, pg_tbl_size,
2744                                   rmem->pg_tbl, rmem->pg_tbl_map);
2745                 rmem->pg_tbl = NULL;
2746         }
2747         if (rmem->vmem_size && *rmem->vmem) {
2748                 vfree(*rmem->vmem);
2749                 *rmem->vmem = NULL;
2750         }
2751 }
2752
2753 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2754 {
2755         struct pci_dev *pdev = bp->pdev;
2756         u64 valid_bit = 0;
2757         int i;
2758
2759         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2760                 valid_bit = PTU_PTE_VALID;
2761         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2762                 size_t pg_tbl_size = rmem->nr_pages * 8;
2763
2764                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2765                         pg_tbl_size = rmem->page_size;
2766                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2767                                                   &rmem->pg_tbl_map,
2768                                                   GFP_KERNEL);
2769                 if (!rmem->pg_tbl)
2770                         return -ENOMEM;
2771         }
2772
2773         for (i = 0; i < rmem->nr_pages; i++) {
2774                 u64 extra_bits = valid_bit;
2775
2776                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2777                                                      rmem->page_size,
2778                                                      &rmem->dma_arr[i],
2779                                                      GFP_KERNEL);
2780                 if (!rmem->pg_arr[i])
2781                         return -ENOMEM;
2782
2783                 if (rmem->mem_init)
2784                         bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i],
2785                                           rmem->page_size);
2786                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2787                         if (i == rmem->nr_pages - 2 &&
2788                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2789                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2790                         else if (i == rmem->nr_pages - 1 &&
2791                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2792                                 extra_bits |= PTU_PTE_LAST;
2793                         rmem->pg_tbl[i] =
2794                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2795                 }
2796         }
2797
2798         if (rmem->vmem_size) {
2799                 *rmem->vmem = vzalloc(rmem->vmem_size);
2800                 if (!(*rmem->vmem))
2801                         return -ENOMEM;
2802         }
2803         return 0;
2804 }
2805
2806 static void bnxt_free_tpa_info(struct bnxt *bp)
2807 {
2808         int i;
2809
2810         for (i = 0; i < bp->rx_nr_rings; i++) {
2811                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2812
2813                 kfree(rxr->rx_tpa_idx_map);
2814                 rxr->rx_tpa_idx_map = NULL;
2815                 if (rxr->rx_tpa) {
2816                         kfree(rxr->rx_tpa[0].agg_arr);
2817                         rxr->rx_tpa[0].agg_arr = NULL;
2818                 }
2819                 kfree(rxr->rx_tpa);
2820                 rxr->rx_tpa = NULL;
2821         }
2822 }
2823
2824 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2825 {
2826         int i, j, total_aggs = 0;
2827
2828         bp->max_tpa = MAX_TPA;
2829         if (bp->flags & BNXT_FLAG_CHIP_P5) {
2830                 if (!bp->max_tpa_v2)
2831                         return 0;
2832                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2833                 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2834         }
2835
2836         for (i = 0; i < bp->rx_nr_rings; i++) {
2837                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2838                 struct rx_agg_cmp *agg;
2839
2840                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2841                                       GFP_KERNEL);
2842                 if (!rxr->rx_tpa)
2843                         return -ENOMEM;
2844
2845                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2846                         continue;
2847                 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2848                 rxr->rx_tpa[0].agg_arr = agg;
2849                 if (!agg)
2850                         return -ENOMEM;
2851                 for (j = 1; j < bp->max_tpa; j++)
2852                         rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2853                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2854                                               GFP_KERNEL);
2855                 if (!rxr->rx_tpa_idx_map)
2856                         return -ENOMEM;
2857         }
2858         return 0;
2859 }
2860
2861 static void bnxt_free_rx_rings(struct bnxt *bp)
2862 {
2863         int i;
2864
2865         if (!bp->rx_ring)
2866                 return;
2867
2868         bnxt_free_tpa_info(bp);
2869         for (i = 0; i < bp->rx_nr_rings; i++) {
2870                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2871                 struct bnxt_ring_struct *ring;
2872
2873                 if (rxr->xdp_prog)
2874                         bpf_prog_put(rxr->xdp_prog);
2875
2876                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2877                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2878
2879                 page_pool_destroy(rxr->page_pool);
2880                 rxr->page_pool = NULL;
2881
2882                 kfree(rxr->rx_agg_bmap);
2883                 rxr->rx_agg_bmap = NULL;
2884
2885                 ring = &rxr->rx_ring_struct;
2886                 bnxt_free_ring(bp, &ring->ring_mem);
2887
2888                 ring = &rxr->rx_agg_ring_struct;
2889                 bnxt_free_ring(bp, &ring->ring_mem);
2890         }
2891 }
2892
2893 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2894                                    struct bnxt_rx_ring_info *rxr)
2895 {
2896         struct page_pool_params pp = { 0 };
2897
2898         pp.pool_size = bp->rx_ring_size;
2899         pp.nid = dev_to_node(&bp->pdev->dev);
2900         pp.dev = &bp->pdev->dev;
2901         pp.dma_dir = DMA_BIDIRECTIONAL;
2902
2903         rxr->page_pool = page_pool_create(&pp);
2904         if (IS_ERR(rxr->page_pool)) {
2905                 int err = PTR_ERR(rxr->page_pool);
2906
2907                 rxr->page_pool = NULL;
2908                 return err;
2909         }
2910         return 0;
2911 }
2912
2913 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2914 {
2915         int i, rc = 0, agg_rings = 0;
2916
2917         if (!bp->rx_ring)
2918                 return -ENOMEM;
2919
2920         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2921                 agg_rings = 1;
2922
2923         for (i = 0; i < bp->rx_nr_rings; i++) {
2924                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2925                 struct bnxt_ring_struct *ring;
2926
2927                 ring = &rxr->rx_ring_struct;
2928
2929                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2930                 if (rc)
2931                         return rc;
2932
2933                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
2934                 if (rc < 0)
2935                         return rc;
2936
2937                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2938                                                 MEM_TYPE_PAGE_POOL,
2939                                                 rxr->page_pool);
2940                 if (rc) {
2941                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2942                         return rc;
2943                 }
2944
2945                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2946                 if (rc)
2947                         return rc;
2948
2949                 ring->grp_idx = i;
2950                 if (agg_rings) {
2951                         u16 mem_size;
2952
2953                         ring = &rxr->rx_agg_ring_struct;
2954                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2955                         if (rc)
2956                                 return rc;
2957
2958                         ring->grp_idx = i;
2959                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2960                         mem_size = rxr->rx_agg_bmap_size / 8;
2961                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2962                         if (!rxr->rx_agg_bmap)
2963                                 return -ENOMEM;
2964                 }
2965         }
2966         if (bp->flags & BNXT_FLAG_TPA)
2967                 rc = bnxt_alloc_tpa_info(bp);
2968         return rc;
2969 }
2970
2971 static void bnxt_free_tx_rings(struct bnxt *bp)
2972 {
2973         int i;
2974         struct pci_dev *pdev = bp->pdev;
2975
2976         if (!bp->tx_ring)
2977                 return;
2978
2979         for (i = 0; i < bp->tx_nr_rings; i++) {
2980                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2981                 struct bnxt_ring_struct *ring;
2982
2983                 if (txr->tx_push) {
2984                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2985                                           txr->tx_push, txr->tx_push_mapping);
2986                         txr->tx_push = NULL;
2987                 }
2988
2989                 ring = &txr->tx_ring_struct;
2990
2991                 bnxt_free_ring(bp, &ring->ring_mem);
2992         }
2993 }
2994
2995 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2996 {
2997         int i, j, rc;
2998         struct pci_dev *pdev = bp->pdev;
2999
3000         bp->tx_push_size = 0;
3001         if (bp->tx_push_thresh) {
3002                 int push_size;
3003
3004                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3005                                         bp->tx_push_thresh);
3006
3007                 if (push_size > 256) {
3008                         push_size = 0;
3009                         bp->tx_push_thresh = 0;
3010                 }
3011
3012                 bp->tx_push_size = push_size;
3013         }
3014
3015         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3016                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3017                 struct bnxt_ring_struct *ring;
3018                 u8 qidx;
3019
3020                 ring = &txr->tx_ring_struct;
3021
3022                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3023                 if (rc)
3024                         return rc;
3025
3026                 ring->grp_idx = txr->bnapi->index;
3027                 if (bp->tx_push_size) {
3028                         dma_addr_t mapping;
3029
3030                         /* One pre-allocated DMA buffer to backup
3031                          * TX push operation
3032                          */
3033                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
3034                                                 bp->tx_push_size,
3035                                                 &txr->tx_push_mapping,
3036                                                 GFP_KERNEL);
3037
3038                         if (!txr->tx_push)
3039                                 return -ENOMEM;
3040
3041                         mapping = txr->tx_push_mapping +
3042                                 sizeof(struct tx_push_bd);
3043                         txr->data_mapping = cpu_to_le64(mapping);
3044                 }
3045                 qidx = bp->tc_to_qidx[j];
3046                 ring->queue_id = bp->q_info[qidx].queue_id;
3047                 if (i < bp->tx_nr_rings_xdp)
3048                         continue;
3049                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3050                         j++;
3051         }
3052         return 0;
3053 }
3054
3055 static void bnxt_free_cp_rings(struct bnxt *bp)
3056 {
3057         int i;
3058
3059         if (!bp->bnapi)
3060                 return;
3061
3062         for (i = 0; i < bp->cp_nr_rings; i++) {
3063                 struct bnxt_napi *bnapi = bp->bnapi[i];
3064                 struct bnxt_cp_ring_info *cpr;
3065                 struct bnxt_ring_struct *ring;
3066                 int j;
3067
3068                 if (!bnapi)
3069                         continue;
3070
3071                 cpr = &bnapi->cp_ring;
3072                 ring = &cpr->cp_ring_struct;
3073
3074                 bnxt_free_ring(bp, &ring->ring_mem);
3075
3076                 for (j = 0; j < 2; j++) {
3077                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3078
3079                         if (cpr2) {
3080                                 ring = &cpr2->cp_ring_struct;
3081                                 bnxt_free_ring(bp, &ring->ring_mem);
3082                                 kfree(cpr2);
3083                                 cpr->cp_ring_arr[j] = NULL;
3084                         }
3085                 }
3086         }
3087 }
3088
3089 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3090 {
3091         struct bnxt_ring_mem_info *rmem;
3092         struct bnxt_ring_struct *ring;
3093         struct bnxt_cp_ring_info *cpr;
3094         int rc;
3095
3096         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3097         if (!cpr)
3098                 return NULL;
3099
3100         ring = &cpr->cp_ring_struct;
3101         rmem = &ring->ring_mem;
3102         rmem->nr_pages = bp->cp_nr_pages;
3103         rmem->page_size = HW_CMPD_RING_SIZE;
3104         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3105         rmem->dma_arr = cpr->cp_desc_mapping;
3106         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3107         rc = bnxt_alloc_ring(bp, rmem);
3108         if (rc) {
3109                 bnxt_free_ring(bp, rmem);
3110                 kfree(cpr);
3111                 cpr = NULL;
3112         }
3113         return cpr;
3114 }
3115
3116 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3117 {
3118         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3119         int i, rc, ulp_base_vec, ulp_msix;
3120
3121         ulp_msix = bnxt_get_ulp_msix_num(bp);
3122         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3123         for (i = 0; i < bp->cp_nr_rings; i++) {
3124                 struct bnxt_napi *bnapi = bp->bnapi[i];
3125                 struct bnxt_cp_ring_info *cpr;
3126                 struct bnxt_ring_struct *ring;
3127
3128                 if (!bnapi)
3129                         continue;
3130
3131                 cpr = &bnapi->cp_ring;
3132                 cpr->bnapi = bnapi;
3133                 ring = &cpr->cp_ring_struct;
3134
3135                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3136                 if (rc)
3137                         return rc;
3138
3139                 if (ulp_msix && i >= ulp_base_vec)
3140                         ring->map_idx = i + ulp_msix;
3141                 else
3142                         ring->map_idx = i;
3143
3144                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3145                         continue;
3146
3147                 if (i < bp->rx_nr_rings) {
3148                         struct bnxt_cp_ring_info *cpr2 =
3149                                 bnxt_alloc_cp_sub_ring(bp);
3150
3151                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3152                         if (!cpr2)
3153                                 return -ENOMEM;
3154                         cpr2->bnapi = bnapi;
3155                 }
3156                 if ((sh && i < bp->tx_nr_rings) ||
3157                     (!sh && i >= bp->rx_nr_rings)) {
3158                         struct bnxt_cp_ring_info *cpr2 =
3159                                 bnxt_alloc_cp_sub_ring(bp);
3160
3161                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3162                         if (!cpr2)
3163                                 return -ENOMEM;
3164                         cpr2->bnapi = bnapi;
3165                 }
3166         }
3167         return 0;
3168 }
3169
3170 static void bnxt_init_ring_struct(struct bnxt *bp)
3171 {
3172         int i;
3173
3174         for (i = 0; i < bp->cp_nr_rings; i++) {
3175                 struct bnxt_napi *bnapi = bp->bnapi[i];
3176                 struct bnxt_ring_mem_info *rmem;
3177                 struct bnxt_cp_ring_info *cpr;
3178                 struct bnxt_rx_ring_info *rxr;
3179                 struct bnxt_tx_ring_info *txr;
3180                 struct bnxt_ring_struct *ring;
3181
3182                 if (!bnapi)
3183                         continue;
3184
3185                 cpr = &bnapi->cp_ring;
3186                 ring = &cpr->cp_ring_struct;
3187                 rmem = &ring->ring_mem;
3188                 rmem->nr_pages = bp->cp_nr_pages;
3189                 rmem->page_size = HW_CMPD_RING_SIZE;
3190                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3191                 rmem->dma_arr = cpr->cp_desc_mapping;
3192                 rmem->vmem_size = 0;
3193
3194                 rxr = bnapi->rx_ring;
3195                 if (!rxr)
3196                         goto skip_rx;
3197
3198                 ring = &rxr->rx_ring_struct;
3199                 rmem = &ring->ring_mem;
3200                 rmem->nr_pages = bp->rx_nr_pages;
3201                 rmem->page_size = HW_RXBD_RING_SIZE;
3202                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3203                 rmem->dma_arr = rxr->rx_desc_mapping;
3204                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3205                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3206
3207                 ring = &rxr->rx_agg_ring_struct;
3208                 rmem = &ring->ring_mem;
3209                 rmem->nr_pages = bp->rx_agg_nr_pages;
3210                 rmem->page_size = HW_RXBD_RING_SIZE;
3211                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3212                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3213                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3214                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3215
3216 skip_rx:
3217                 txr = bnapi->tx_ring;
3218                 if (!txr)
3219                         continue;
3220
3221                 ring = &txr->tx_ring_struct;
3222                 rmem = &ring->ring_mem;
3223                 rmem->nr_pages = bp->tx_nr_pages;
3224                 rmem->page_size = HW_RXBD_RING_SIZE;
3225                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3226                 rmem->dma_arr = txr->tx_desc_mapping;
3227                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3228                 rmem->vmem = (void **)&txr->tx_buf_ring;
3229         }
3230 }
3231
3232 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3233 {
3234         int i;
3235         u32 prod;
3236         struct rx_bd **rx_buf_ring;
3237
3238         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3239         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3240                 int j;
3241                 struct rx_bd *rxbd;
3242
3243                 rxbd = rx_buf_ring[i];
3244                 if (!rxbd)
3245                         continue;
3246
3247                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3248                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3249                         rxbd->rx_bd_opaque = prod;
3250                 }
3251         }
3252 }
3253
3254 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3255 {
3256         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3257         struct net_device *dev = bp->dev;
3258         u32 prod;
3259         int i;
3260
3261         prod = rxr->rx_prod;
3262         for (i = 0; i < bp->rx_ring_size; i++) {
3263                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3264                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3265                                     ring_nr, i, bp->rx_ring_size);
3266                         break;
3267                 }
3268                 prod = NEXT_RX(prod);
3269         }
3270         rxr->rx_prod = prod;
3271
3272         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3273                 return 0;
3274
3275         prod = rxr->rx_agg_prod;
3276         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3277                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3278                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3279                                     ring_nr, i, bp->rx_ring_size);
3280                         break;
3281                 }
3282                 prod = NEXT_RX_AGG(prod);
3283         }
3284         rxr->rx_agg_prod = prod;
3285
3286         if (rxr->rx_tpa) {
3287                 dma_addr_t mapping;
3288                 u8 *data;
3289
3290                 for (i = 0; i < bp->max_tpa; i++) {
3291                         data = __bnxt_alloc_rx_data(bp, &mapping, GFP_KERNEL);
3292                         if (!data)
3293                                 return -ENOMEM;
3294
3295                         rxr->rx_tpa[i].data = data;
3296                         rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3297                         rxr->rx_tpa[i].mapping = mapping;
3298                 }
3299         }
3300         return 0;
3301 }
3302
3303 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3304 {
3305         struct bnxt_rx_ring_info *rxr;
3306         struct bnxt_ring_struct *ring;
3307         u32 type;
3308
3309         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3310                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3311
3312         if (NET_IP_ALIGN == 2)
3313                 type |= RX_BD_FLAGS_SOP;
3314
3315         rxr = &bp->rx_ring[ring_nr];
3316         ring = &rxr->rx_ring_struct;
3317         bnxt_init_rxbd_pages(ring, type);
3318
3319         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3320                 bpf_prog_add(bp->xdp_prog, 1);
3321                 rxr->xdp_prog = bp->xdp_prog;
3322         }
3323         ring->fw_ring_id = INVALID_HW_RING_ID;
3324
3325         ring = &rxr->rx_agg_ring_struct;
3326         ring->fw_ring_id = INVALID_HW_RING_ID;
3327
3328         if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3329                 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3330                         RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3331
3332                 bnxt_init_rxbd_pages(ring, type);
3333         }
3334
3335         return bnxt_alloc_one_rx_ring(bp, ring_nr);
3336 }
3337
3338 static void bnxt_init_cp_rings(struct bnxt *bp)
3339 {
3340         int i, j;
3341
3342         for (i = 0; i < bp->cp_nr_rings; i++) {
3343                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3344                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3345
3346                 ring->fw_ring_id = INVALID_HW_RING_ID;
3347                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3348                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3349                 for (j = 0; j < 2; j++) {
3350                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3351
3352                         if (!cpr2)
3353                                 continue;
3354
3355                         ring = &cpr2->cp_ring_struct;
3356                         ring->fw_ring_id = INVALID_HW_RING_ID;
3357                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3358                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3359                 }
3360         }
3361 }
3362
3363 static int bnxt_init_rx_rings(struct bnxt *bp)
3364 {
3365         int i, rc = 0;
3366
3367         if (BNXT_RX_PAGE_MODE(bp)) {
3368                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3369                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3370         } else {
3371                 bp->rx_offset = BNXT_RX_OFFSET;
3372                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3373         }
3374
3375         for (i = 0; i < bp->rx_nr_rings; i++) {
3376                 rc = bnxt_init_one_rx_ring(bp, i);
3377                 if (rc)
3378                         break;
3379         }
3380
3381         return rc;
3382 }
3383
3384 static int bnxt_init_tx_rings(struct bnxt *bp)
3385 {
3386         u16 i;
3387
3388         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3389                                    MAX_SKB_FRAGS + 1);
3390
3391         for (i = 0; i < bp->tx_nr_rings; i++) {
3392                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3393                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3394
3395                 ring->fw_ring_id = INVALID_HW_RING_ID;
3396         }
3397
3398         return 0;
3399 }
3400
3401 static void bnxt_free_ring_grps(struct bnxt *bp)
3402 {
3403         kfree(bp->grp_info);
3404         bp->grp_info = NULL;
3405 }
3406
3407 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3408 {
3409         int i;
3410
3411         if (irq_re_init) {
3412                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3413                                        sizeof(struct bnxt_ring_grp_info),
3414                                        GFP_KERNEL);
3415                 if (!bp->grp_info)
3416                         return -ENOMEM;
3417         }
3418         for (i = 0; i < bp->cp_nr_rings; i++) {
3419                 if (irq_re_init)
3420                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3421                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3422                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3423                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3424                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3425         }
3426         return 0;
3427 }
3428
3429 static void bnxt_free_vnics(struct bnxt *bp)
3430 {
3431         kfree(bp->vnic_info);
3432         bp->vnic_info = NULL;
3433         bp->nr_vnics = 0;
3434 }
3435
3436 static int bnxt_alloc_vnics(struct bnxt *bp)
3437 {
3438         int num_vnics = 1;
3439
3440 #ifdef CONFIG_RFS_ACCEL
3441         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3442                 num_vnics += bp->rx_nr_rings;
3443 #endif
3444
3445         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3446                 num_vnics++;
3447
3448         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3449                                 GFP_KERNEL);
3450         if (!bp->vnic_info)
3451                 return -ENOMEM;
3452
3453         bp->nr_vnics = num_vnics;
3454         return 0;
3455 }
3456
3457 static void bnxt_init_vnics(struct bnxt *bp)
3458 {
3459         int i;
3460
3461         for (i = 0; i < bp->nr_vnics; i++) {
3462                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3463                 int j;
3464
3465                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3466                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3467                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3468
3469                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3470
3471                 if (bp->vnic_info[i].rss_hash_key) {
3472                         if (i == 0)
3473                                 prandom_bytes(vnic->rss_hash_key,
3474                                               HW_HASH_KEY_SIZE);
3475                         else
3476                                 memcpy(vnic->rss_hash_key,
3477                                        bp->vnic_info[0].rss_hash_key,
3478                                        HW_HASH_KEY_SIZE);
3479                 }
3480         }
3481 }
3482
3483 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3484 {
3485         int pages;
3486
3487         pages = ring_size / desc_per_pg;
3488
3489         if (!pages)
3490                 return 1;
3491
3492         pages++;
3493
3494         while (pages & (pages - 1))
3495                 pages++;
3496
3497         return pages;
3498 }
3499
3500 void bnxt_set_tpa_flags(struct bnxt *bp)
3501 {
3502         bp->flags &= ~BNXT_FLAG_TPA;
3503         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3504                 return;
3505         if (bp->dev->features & NETIF_F_LRO)
3506                 bp->flags |= BNXT_FLAG_LRO;
3507         else if (bp->dev->features & NETIF_F_GRO_HW)
3508                 bp->flags |= BNXT_FLAG_GRO;
3509 }
3510
3511 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3512  * be set on entry.
3513  */
3514 void bnxt_set_ring_params(struct bnxt *bp)
3515 {
3516         u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3517         u32 agg_factor = 0, agg_ring_size = 0;
3518
3519         /* 8 for CRC and VLAN */
3520         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3521
3522         rx_space = rx_size + NET_SKB_PAD +
3523                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3524
3525         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3526         ring_size = bp->rx_ring_size;
3527         bp->rx_agg_ring_size = 0;
3528         bp->rx_agg_nr_pages = 0;
3529
3530         if (bp->flags & BNXT_FLAG_TPA)
3531                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3532
3533         bp->flags &= ~BNXT_FLAG_JUMBO;
3534         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3535                 u32 jumbo_factor;
3536
3537                 bp->flags |= BNXT_FLAG_JUMBO;
3538                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3539                 if (jumbo_factor > agg_factor)
3540                         agg_factor = jumbo_factor;
3541         }
3542         agg_ring_size = ring_size * agg_factor;
3543
3544         if (agg_ring_size) {
3545                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3546                                                         RX_DESC_CNT);
3547                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3548                         u32 tmp = agg_ring_size;
3549
3550                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3551                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3552                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3553                                     tmp, agg_ring_size);
3554                 }
3555                 bp->rx_agg_ring_size = agg_ring_size;
3556                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3557                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3558                 rx_space = rx_size + NET_SKB_PAD +
3559                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3560         }
3561
3562         bp->rx_buf_use_size = rx_size;
3563         bp->rx_buf_size = rx_space;
3564
3565         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3566         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3567
3568         ring_size = bp->tx_ring_size;
3569         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3570         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3571
3572         max_rx_cmpl = bp->rx_ring_size;
3573         /* MAX TPA needs to be added because TPA_START completions are
3574          * immediately recycled, so the TPA completions are not bound by
3575          * the RX ring size.
3576          */
3577         if (bp->flags & BNXT_FLAG_TPA)
3578                 max_rx_cmpl += bp->max_tpa;
3579         /* RX and TPA completions are 32-byte, all others are 16-byte */
3580         ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3581         bp->cp_ring_size = ring_size;
3582
3583         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3584         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3585                 bp->cp_nr_pages = MAX_CP_PAGES;
3586                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3587                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3588                             ring_size, bp->cp_ring_size);
3589         }
3590         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3591         bp->cp_ring_mask = bp->cp_bit - 1;
3592 }
3593
3594 /* Changing allocation mode of RX rings.
3595  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3596  */
3597 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3598 {
3599         if (page_mode) {
3600                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3601                         return -EOPNOTSUPP;
3602                 bp->dev->max_mtu =
3603                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3604                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3605                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3606                 bp->rx_dir = DMA_BIDIRECTIONAL;
3607                 bp->rx_skb_func = bnxt_rx_page_skb;
3608                 /* Disable LRO or GRO_HW */
3609                 netdev_update_features(bp->dev);
3610         } else {
3611                 bp->dev->max_mtu = bp->max_mtu;
3612                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3613                 bp->rx_dir = DMA_FROM_DEVICE;
3614                 bp->rx_skb_func = bnxt_rx_skb;
3615         }
3616         return 0;
3617 }
3618
3619 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3620 {
3621         int i;
3622         struct bnxt_vnic_info *vnic;
3623         struct pci_dev *pdev = bp->pdev;
3624
3625         if (!bp->vnic_info)
3626                 return;
3627
3628         for (i = 0; i < bp->nr_vnics; i++) {
3629                 vnic = &bp->vnic_info[i];
3630
3631                 kfree(vnic->fw_grp_ids);
3632                 vnic->fw_grp_ids = NULL;
3633
3634                 kfree(vnic->uc_list);
3635                 vnic->uc_list = NULL;
3636
3637                 if (vnic->mc_list) {
3638                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3639                                           vnic->mc_list, vnic->mc_list_mapping);
3640                         vnic->mc_list = NULL;
3641                 }
3642
3643                 if (vnic->rss_table) {
3644                         dma_free_coherent(&pdev->dev, vnic->rss_table_size,
3645                                           vnic->rss_table,
3646                                           vnic->rss_table_dma_addr);
3647                         vnic->rss_table = NULL;
3648                 }
3649
3650                 vnic->rss_hash_key = NULL;
3651                 vnic->flags = 0;
3652         }
3653 }
3654
3655 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3656 {
3657         int i, rc = 0, size;
3658         struct bnxt_vnic_info *vnic;
3659         struct pci_dev *pdev = bp->pdev;
3660         int max_rings;
3661
3662         for (i = 0; i < bp->nr_vnics; i++) {
3663                 vnic = &bp->vnic_info[i];
3664
3665                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3666                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3667
3668                         if (mem_size > 0) {
3669                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3670                                 if (!vnic->uc_list) {
3671                                         rc = -ENOMEM;
3672                                         goto out;
3673                                 }
3674                         }
3675                 }
3676
3677                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3678                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3679                         vnic->mc_list =
3680                                 dma_alloc_coherent(&pdev->dev,
3681                                                    vnic->mc_list_size,
3682                                                    &vnic->mc_list_mapping,
3683                                                    GFP_KERNEL);
3684                         if (!vnic->mc_list) {
3685                                 rc = -ENOMEM;
3686                                 goto out;
3687                         }
3688                 }
3689
3690                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3691                         goto vnic_skip_grps;
3692
3693                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3694                         max_rings = bp->rx_nr_rings;
3695                 else
3696                         max_rings = 1;
3697
3698                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3699                 if (!vnic->fw_grp_ids) {
3700                         rc = -ENOMEM;
3701                         goto out;
3702                 }
3703 vnic_skip_grps:
3704                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3705                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3706                         continue;
3707
3708                 /* Allocate rss table and hash key */
3709                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3710                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3711                         size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
3712
3713                 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
3714                 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
3715                                                      vnic->rss_table_size,
3716                                                      &vnic->rss_table_dma_addr,
3717                                                      GFP_KERNEL);
3718                 if (!vnic->rss_table) {
3719                         rc = -ENOMEM;
3720                         goto out;
3721                 }
3722
3723                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3724                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3725         }
3726         return 0;
3727
3728 out:
3729         return rc;
3730 }
3731
3732 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3733 {
3734         struct pci_dev *pdev = bp->pdev;
3735
3736         if (bp->hwrm_cmd_resp_addr) {
3737                 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3738                                   bp->hwrm_cmd_resp_dma_addr);
3739                 bp->hwrm_cmd_resp_addr = NULL;
3740         }
3741
3742         if (bp->hwrm_cmd_kong_resp_addr) {
3743                 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3744                                   bp->hwrm_cmd_kong_resp_addr,
3745                                   bp->hwrm_cmd_kong_resp_dma_addr);
3746                 bp->hwrm_cmd_kong_resp_addr = NULL;
3747         }
3748 }
3749
3750 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3751 {
3752         struct pci_dev *pdev = bp->pdev;
3753
3754         if (bp->hwrm_cmd_kong_resp_addr)
3755                 return 0;
3756
3757         bp->hwrm_cmd_kong_resp_addr =
3758                 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3759                                    &bp->hwrm_cmd_kong_resp_dma_addr,
3760                                    GFP_KERNEL);
3761         if (!bp->hwrm_cmd_kong_resp_addr)
3762                 return -ENOMEM;
3763
3764         return 0;
3765 }
3766
3767 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3768 {
3769         struct pci_dev *pdev = bp->pdev;
3770
3771         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3772                                                    &bp->hwrm_cmd_resp_dma_addr,
3773                                                    GFP_KERNEL);
3774         if (!bp->hwrm_cmd_resp_addr)
3775                 return -ENOMEM;
3776
3777         return 0;
3778 }
3779
3780 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3781 {
3782         if (bp->hwrm_short_cmd_req_addr) {
3783                 struct pci_dev *pdev = bp->pdev;
3784
3785                 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3786                                   bp->hwrm_short_cmd_req_addr,
3787                                   bp->hwrm_short_cmd_req_dma_addr);
3788                 bp->hwrm_short_cmd_req_addr = NULL;
3789         }
3790 }
3791
3792 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3793 {
3794         struct pci_dev *pdev = bp->pdev;
3795
3796         if (bp->hwrm_short_cmd_req_addr)
3797                 return 0;
3798
3799         bp->hwrm_short_cmd_req_addr =
3800                 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3801                                    &bp->hwrm_short_cmd_req_dma_addr,
3802                                    GFP_KERNEL);
3803         if (!bp->hwrm_short_cmd_req_addr)
3804                 return -ENOMEM;
3805
3806         return 0;
3807 }
3808
3809 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
3810 {
3811         kfree(stats->hw_masks);
3812         stats->hw_masks = NULL;
3813         kfree(stats->sw_stats);
3814         stats->sw_stats = NULL;
3815         if (stats->hw_stats) {
3816                 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
3817                                   stats->hw_stats_map);
3818                 stats->hw_stats = NULL;
3819         }
3820 }
3821
3822 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
3823                                 bool alloc_masks)
3824 {
3825         stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
3826                                              &stats->hw_stats_map, GFP_KERNEL);
3827         if (!stats->hw_stats)
3828                 return -ENOMEM;
3829
3830         stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
3831         if (!stats->sw_stats)
3832                 goto stats_mem_err;
3833
3834         if (alloc_masks) {
3835                 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
3836                 if (!stats->hw_masks)
3837                         goto stats_mem_err;
3838         }
3839         return 0;
3840
3841 stats_mem_err:
3842         bnxt_free_stats_mem(bp, stats);
3843         return -ENOMEM;
3844 }
3845
3846 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
3847 {
3848         int i;
3849
3850         for (i = 0; i < count; i++)
3851                 mask_arr[i] = mask;
3852 }
3853
3854 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
3855 {
3856         int i;
3857
3858         for (i = 0; i < count; i++)
3859                 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
3860 }
3861
3862 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
3863                                     struct bnxt_stats_mem *stats)
3864 {
3865         struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
3866         struct hwrm_func_qstats_ext_input req = {0};
3867         __le64 *hw_masks;
3868         int rc;
3869
3870         if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
3871             !(bp->flags & BNXT_FLAG_CHIP_P5))
3872                 return -EOPNOTSUPP;
3873
3874         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1);
3875         req.fid = cpu_to_le16(0xffff);
3876         req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3877         mutex_lock(&bp->hwrm_cmd_lock);
3878         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3879         if (rc)
3880                 goto qstat_exit;
3881
3882         hw_masks = &resp->rx_ucast_pkts;
3883         bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
3884
3885 qstat_exit:
3886         mutex_unlock(&bp->hwrm_cmd_lock);
3887         return rc;
3888 }
3889
3890 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
3891 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
3892
3893 static void bnxt_init_stats(struct bnxt *bp)
3894 {
3895         struct bnxt_napi *bnapi = bp->bnapi[0];
3896         struct bnxt_cp_ring_info *cpr;
3897         struct bnxt_stats_mem *stats;
3898         __le64 *rx_stats, *tx_stats;
3899         int rc, rx_count, tx_count;
3900         u64 *rx_masks, *tx_masks;
3901         u64 mask;
3902         u8 flags;
3903
3904         cpr = &bnapi->cp_ring;
3905         stats = &cpr->stats;
3906         rc = bnxt_hwrm_func_qstat_ext(bp, stats);
3907         if (rc) {
3908                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3909                         mask = (1ULL << 48) - 1;
3910                 else
3911                         mask = -1ULL;
3912                 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
3913         }
3914         if (bp->flags & BNXT_FLAG_PORT_STATS) {
3915                 stats = &bp->port_stats;
3916                 rx_stats = stats->hw_stats;
3917                 rx_masks = stats->hw_masks;
3918                 rx_count = sizeof(struct rx_port_stats) / 8;
3919                 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3920                 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3921                 tx_count = sizeof(struct tx_port_stats) / 8;
3922
3923                 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
3924                 rc = bnxt_hwrm_port_qstats(bp, flags);
3925                 if (rc) {
3926                         mask = (1ULL << 40) - 1;
3927
3928                         bnxt_fill_masks(rx_masks, mask, rx_count);
3929                         bnxt_fill_masks(tx_masks, mask, tx_count);
3930                 } else {
3931                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3932                         bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
3933                         bnxt_hwrm_port_qstats(bp, 0);
3934                 }
3935         }
3936         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
3937                 stats = &bp->rx_port_stats_ext;
3938                 rx_stats = stats->hw_stats;
3939                 rx_masks = stats->hw_masks;
3940                 rx_count = sizeof(struct rx_port_stats_ext) / 8;
3941                 stats = &bp->tx_port_stats_ext;
3942                 tx_stats = stats->hw_stats;
3943                 tx_masks = stats->hw_masks;
3944                 tx_count = sizeof(struct tx_port_stats_ext) / 8;
3945
3946                 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3947                 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
3948                 if (rc) {
3949                         mask = (1ULL << 40) - 1;
3950
3951                         bnxt_fill_masks(rx_masks, mask, rx_count);
3952                         if (tx_stats)
3953                                 bnxt_fill_masks(tx_masks, mask, tx_count);
3954                 } else {
3955                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3956                         if (tx_stats)
3957                                 bnxt_copy_hw_masks(tx_masks, tx_stats,
3958                                                    tx_count);
3959                         bnxt_hwrm_port_qstats_ext(bp, 0);
3960                 }
3961         }
3962 }
3963
3964 static void bnxt_free_port_stats(struct bnxt *bp)
3965 {
3966         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3967         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3968
3969         bnxt_free_stats_mem(bp, &bp->port_stats);
3970         bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
3971         bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
3972 }
3973
3974 static void bnxt_free_ring_stats(struct bnxt *bp)
3975 {
3976         int i;
3977
3978         if (!bp->bnapi)
3979                 return;
3980
3981         for (i = 0; i < bp->cp_nr_rings; i++) {
3982                 struct bnxt_napi *bnapi = bp->bnapi[i];
3983                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3984
3985                 bnxt_free_stats_mem(bp, &cpr->stats);
3986         }
3987 }
3988
3989 static int bnxt_alloc_stats(struct bnxt *bp)
3990 {
3991         u32 size, i;
3992         int rc;
3993
3994         size = bp->hw_ring_stats_size;
3995
3996         for (i = 0; i < bp->cp_nr_rings; i++) {
3997                 struct bnxt_napi *bnapi = bp->bnapi[i];
3998                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3999
4000                 cpr->stats.len = size;
4001                 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4002                 if (rc)
4003                         return rc;
4004
4005                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4006         }
4007
4008         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4009                 return 0;
4010
4011         if (bp->port_stats.hw_stats)
4012                 goto alloc_ext_stats;
4013
4014         bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4015         rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4016         if (rc)
4017                 return rc;
4018
4019         bp->flags |= BNXT_FLAG_PORT_STATS;
4020
4021 alloc_ext_stats:
4022         /* Display extended statistics only if FW supports it */
4023         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4024                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4025                         return 0;
4026
4027         if (bp->rx_port_stats_ext.hw_stats)
4028                 goto alloc_tx_ext_stats;
4029
4030         bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4031         rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4032         /* Extended stats are optional */
4033         if (rc)
4034                 return 0;
4035
4036 alloc_tx_ext_stats:
4037         if (bp->tx_port_stats_ext.hw_stats)
4038                 return 0;
4039
4040         if (bp->hwrm_spec_code >= 0x10902 ||
4041             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4042                 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4043                 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4044                 /* Extended stats are optional */
4045                 if (rc)
4046                         return 0;
4047         }
4048         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4049         return 0;
4050 }
4051
4052 static void bnxt_clear_ring_indices(struct bnxt *bp)
4053 {
4054         int i;
4055
4056         if (!bp->bnapi)
4057                 return;
4058
4059         for (i = 0; i < bp->cp_nr_rings; i++) {
4060                 struct bnxt_napi *bnapi = bp->bnapi[i];
4061                 struct bnxt_cp_ring_info *cpr;
4062                 struct bnxt_rx_ring_info *rxr;
4063                 struct bnxt_tx_ring_info *txr;
4064
4065                 if (!bnapi)
4066                         continue;
4067
4068                 cpr = &bnapi->cp_ring;
4069                 cpr->cp_raw_cons = 0;
4070
4071                 txr = bnapi->tx_ring;
4072                 if (txr) {
4073                         txr->tx_prod = 0;
4074                         txr->tx_cons = 0;
4075                 }
4076
4077                 rxr = bnapi->rx_ring;
4078                 if (rxr) {
4079                         rxr->rx_prod = 0;
4080                         rxr->rx_agg_prod = 0;
4081                         rxr->rx_sw_agg_prod = 0;
4082                         rxr->rx_next_cons = 0;
4083                 }
4084         }
4085 }
4086
4087 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4088 {
4089 #ifdef CONFIG_RFS_ACCEL
4090         int i;
4091
4092         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
4093          * safe to delete the hash table.
4094          */
4095         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4096                 struct hlist_head *head;
4097                 struct hlist_node *tmp;
4098                 struct bnxt_ntuple_filter *fltr;
4099
4100                 head = &bp->ntp_fltr_hash_tbl[i];
4101                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4102                         hlist_del(&fltr->hash);
4103                         kfree(fltr);
4104                 }
4105         }
4106         if (irq_reinit) {
4107                 kfree(bp->ntp_fltr_bmap);
4108                 bp->ntp_fltr_bmap = NULL;
4109         }
4110         bp->ntp_fltr_count = 0;
4111 #endif
4112 }
4113
4114 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4115 {
4116 #ifdef CONFIG_RFS_ACCEL
4117         int i, rc = 0;
4118
4119         if (!(bp->flags & BNXT_FLAG_RFS))
4120                 return 0;
4121
4122         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4123                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4124
4125         bp->ntp_fltr_count = 0;
4126         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
4127                                     sizeof(long),
4128                                     GFP_KERNEL);
4129
4130         if (!bp->ntp_fltr_bmap)
4131                 rc = -ENOMEM;
4132
4133         return rc;
4134 #else
4135         return 0;
4136 #endif
4137 }
4138
4139 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4140 {
4141         bnxt_free_vnic_attributes(bp);
4142         bnxt_free_tx_rings(bp);
4143         bnxt_free_rx_rings(bp);
4144         bnxt_free_cp_rings(bp);
4145         bnxt_free_ntp_fltrs(bp, irq_re_init);
4146         if (irq_re_init) {
4147                 bnxt_free_ring_stats(bp);
4148                 if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET) ||
4149                     test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4150                         bnxt_free_port_stats(bp);
4151                 bnxt_free_ring_grps(bp);
4152                 bnxt_free_vnics(bp);
4153                 kfree(bp->tx_ring_map);
4154                 bp->tx_ring_map = NULL;
4155                 kfree(bp->tx_ring);
4156                 bp->tx_ring = NULL;
4157                 kfree(bp->rx_ring);
4158                 bp->rx_ring = NULL;
4159                 kfree(bp->bnapi);
4160                 bp->bnapi = NULL;
4161         } else {
4162                 bnxt_clear_ring_indices(bp);
4163         }
4164 }
4165
4166 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4167 {
4168         int i, j, rc, size, arr_size;
4169         void *bnapi;
4170
4171         if (irq_re_init) {
4172                 /* Allocate bnapi mem pointer array and mem block for
4173                  * all queues
4174                  */
4175                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4176                                 bp->cp_nr_rings);
4177                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4178                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4179                 if (!bnapi)
4180                         return -ENOMEM;
4181
4182                 bp->bnapi = bnapi;
4183                 bnapi += arr_size;
4184                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4185                         bp->bnapi[i] = bnapi;
4186                         bp->bnapi[i]->index = i;
4187                         bp->bnapi[i]->bp = bp;
4188                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4189                                 struct bnxt_cp_ring_info *cpr =
4190                                         &bp->bnapi[i]->cp_ring;
4191
4192                                 cpr->cp_ring_struct.ring_mem.flags =
4193                                         BNXT_RMEM_RING_PTE_FLAG;
4194                         }
4195                 }
4196
4197                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4198                                       sizeof(struct bnxt_rx_ring_info),
4199                                       GFP_KERNEL);
4200                 if (!bp->rx_ring)
4201                         return -ENOMEM;
4202
4203                 for (i = 0; i < bp->rx_nr_rings; i++) {
4204                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4205
4206                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4207                                 rxr->rx_ring_struct.ring_mem.flags =
4208                                         BNXT_RMEM_RING_PTE_FLAG;
4209                                 rxr->rx_agg_ring_struct.ring_mem.flags =
4210                                         BNXT_RMEM_RING_PTE_FLAG;
4211                         }
4212                         rxr->bnapi = bp->bnapi[i];
4213                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4214                 }
4215
4216                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4217                                       sizeof(struct bnxt_tx_ring_info),
4218                                       GFP_KERNEL);
4219                 if (!bp->tx_ring)
4220                         return -ENOMEM;
4221
4222                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4223                                           GFP_KERNEL);
4224
4225                 if (!bp->tx_ring_map)
4226                         return -ENOMEM;
4227
4228                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4229                         j = 0;
4230                 else
4231                         j = bp->rx_nr_rings;
4232
4233                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4234                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4235
4236                         if (bp->flags & BNXT_FLAG_CHIP_P5)
4237                                 txr->tx_ring_struct.ring_mem.flags =
4238                                         BNXT_RMEM_RING_PTE_FLAG;
4239                         txr->bnapi = bp->bnapi[j];
4240                         bp->bnapi[j]->tx_ring = txr;
4241                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4242                         if (i >= bp->tx_nr_rings_xdp) {
4243                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4244                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
4245                         } else {
4246                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4247                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4248                         }
4249                 }
4250
4251                 rc = bnxt_alloc_stats(bp);
4252                 if (rc)
4253                         goto alloc_mem_err;
4254                 bnxt_init_stats(bp);
4255
4256                 rc = bnxt_alloc_ntp_fltrs(bp);
4257                 if (rc)
4258                         goto alloc_mem_err;
4259
4260                 rc = bnxt_alloc_vnics(bp);
4261                 if (rc)
4262                         goto alloc_mem_err;
4263         }
4264
4265         bnxt_init_ring_struct(bp);
4266
4267         rc = bnxt_alloc_rx_rings(bp);
4268         if (rc)
4269                 goto alloc_mem_err;
4270
4271         rc = bnxt_alloc_tx_rings(bp);
4272         if (rc)
4273                 goto alloc_mem_err;
4274
4275         rc = bnxt_alloc_cp_rings(bp);
4276         if (rc)
4277                 goto alloc_mem_err;
4278
4279         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4280                                   BNXT_VNIC_UCAST_FLAG;
4281         rc = bnxt_alloc_vnic_attributes(bp);
4282         if (rc)
4283                 goto alloc_mem_err;
4284         return 0;
4285
4286 alloc_mem_err:
4287         bnxt_free_mem(bp, true);
4288         return rc;
4289 }
4290
4291 static void bnxt_disable_int(struct bnxt *bp)
4292 {
4293         int i;
4294
4295         if (!bp->bnapi)
4296                 return;
4297
4298         for (i = 0; i < bp->cp_nr_rings; i++) {
4299                 struct bnxt_napi *bnapi = bp->bnapi[i];
4300                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4301                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4302
4303                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4304                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4305         }
4306 }
4307
4308 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4309 {
4310         struct bnxt_napi *bnapi = bp->bnapi[n];
4311         struct bnxt_cp_ring_info *cpr;
4312
4313         cpr = &bnapi->cp_ring;
4314         return cpr->cp_ring_struct.map_idx;
4315 }
4316
4317 static void bnxt_disable_int_sync(struct bnxt *bp)
4318 {
4319         int i;
4320
4321         if (!bp->irq_tbl)
4322                 return;
4323
4324         atomic_inc(&bp->intr_sem);
4325
4326         bnxt_disable_int(bp);
4327         for (i = 0; i < bp->cp_nr_rings; i++) {
4328                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4329
4330                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4331         }
4332 }
4333
4334 static void bnxt_enable_int(struct bnxt *bp)
4335 {
4336         int i;
4337
4338         atomic_set(&bp->intr_sem, 0);
4339         for (i = 0; i < bp->cp_nr_rings; i++) {
4340                 struct bnxt_napi *bnapi = bp->bnapi[i];
4341                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4342
4343                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4344         }
4345 }
4346
4347 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4348                             u16 cmpl_ring, u16 target_id)
4349 {
4350         struct input *req = request;
4351
4352         req->req_type = cpu_to_le16(req_type);
4353         req->cmpl_ring = cpu_to_le16(cmpl_ring);
4354         req->target_id = cpu_to_le16(target_id);
4355         if (bnxt_kong_hwrm_message(bp, req))
4356                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4357         else
4358                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4359 }
4360
4361 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4362 {
4363         switch (hwrm_err) {
4364         case HWRM_ERR_CODE_SUCCESS:
4365                 return 0;
4366         case HWRM_ERR_CODE_RESOURCE_LOCKED:
4367                 return -EROFS;
4368         case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4369                 return -EACCES;
4370         case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4371                 return -ENOSPC;
4372         case HWRM_ERR_CODE_INVALID_PARAMS:
4373         case HWRM_ERR_CODE_INVALID_FLAGS:
4374         case HWRM_ERR_CODE_INVALID_ENABLES:
4375         case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4376         case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4377                 return -EINVAL;
4378         case HWRM_ERR_CODE_NO_BUFFER:
4379                 return -ENOMEM;
4380         case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4381         case HWRM_ERR_CODE_BUSY:
4382                 return -EAGAIN;
4383         case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4384                 return -EOPNOTSUPP;
4385         default:
4386                 return -EIO;
4387         }
4388 }
4389
4390 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4391                                  int timeout, bool silent)
4392 {
4393         int i, intr_process, rc, tmo_count;
4394         struct input *req = msg;
4395         u32 *data = msg;
4396         u8 *valid;
4397         u16 cp_ring_id, len = 0;
4398         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4399         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4400         struct hwrm_short_input short_input = {0};
4401         u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4402         u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4403         u16 dst = BNXT_HWRM_CHNL_CHIMP;
4404
4405         if (BNXT_NO_FW_ACCESS(bp) &&
4406             le16_to_cpu(req->req_type) != HWRM_FUNC_RESET)
4407                 return -EBUSY;
4408
4409         if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4410                 if (msg_len > bp->hwrm_max_ext_req_len ||
4411                     !bp->hwrm_short_cmd_req_addr)
4412                         return -EINVAL;
4413         }
4414
4415         if (bnxt_hwrm_kong_chnl(bp, req)) {
4416                 dst = BNXT_HWRM_CHNL_KONG;
4417                 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4418                 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4419                 resp = bp->hwrm_cmd_kong_resp_addr;
4420         }
4421
4422         memset(resp, 0, PAGE_SIZE);
4423         cp_ring_id = le16_to_cpu(req->cmpl_ring);
4424         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4425
4426         req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4427         /* currently supports only one outstanding message */
4428         if (intr_process)
4429                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4430
4431         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4432             msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4433                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4434                 u16 max_msg_len;
4435
4436                 /* Set boundary for maximum extended request length for short
4437                  * cmd format. If passed up from device use the max supported
4438                  * internal req length.
4439                  */
4440                 max_msg_len = bp->hwrm_max_ext_req_len;
4441
4442                 memcpy(short_cmd_req, req, msg_len);
4443                 if (msg_len < max_msg_len)
4444                         memset(short_cmd_req + msg_len, 0,
4445                                max_msg_len - msg_len);
4446
4447                 short_input.req_type = req->req_type;
4448                 short_input.signature =
4449                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4450                 short_input.size = cpu_to_le16(msg_len);
4451                 short_input.req_addr =
4452                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4453
4454                 data = (u32 *)&short_input;
4455                 msg_len = sizeof(short_input);
4456
4457                 /* Sync memory write before updating doorbell */
4458                 wmb();
4459
4460                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4461         }
4462
4463         /* Write request msg to hwrm channel */
4464         __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4465
4466         for (i = msg_len; i < max_req_len; i += 4)
4467                 writel(0, bp->bar0 + bar_offset + i);
4468
4469         /* Ring channel doorbell */
4470         writel(1, bp->bar0 + doorbell_offset);
4471
4472         if (!pci_is_enabled(bp->pdev))
4473                 return 0;
4474
4475         if (!timeout)
4476                 timeout = DFLT_HWRM_CMD_TIMEOUT;
4477         /* Limit timeout to an upper limit */
4478         timeout = min(timeout, HWRM_CMD_MAX_TIMEOUT);
4479         /* convert timeout to usec */
4480         timeout *= 1000;
4481
4482         i = 0;
4483         /* Short timeout for the first few iterations:
4484          * number of loops = number of loops for short timeout +
4485          * number of loops for standard timeout.
4486          */
4487         tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4488         timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4489         tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4490
4491         if (intr_process) {
4492                 u16 seq_id = bp->hwrm_intr_seq_id;
4493
4494                 /* Wait until hwrm response cmpl interrupt is processed */
4495                 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4496                        i++ < tmo_count) {
4497                         /* Abort the wait for completion if the FW health
4498                          * check has failed.
4499                          */
4500                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4501                                 return -EBUSY;
4502                         /* on first few passes, just barely sleep */
4503                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4504                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4505                                              HWRM_SHORT_MAX_TIMEOUT);
4506                         else
4507                                 usleep_range(HWRM_MIN_TIMEOUT,
4508                                              HWRM_MAX_TIMEOUT);
4509                 }
4510
4511                 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4512                         if (!silent)
4513                                 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4514                                            le16_to_cpu(req->req_type));
4515                         return -EBUSY;
4516                 }
4517                 len = le16_to_cpu(resp->resp_len);
4518                 valid = ((u8 *)resp) + len - 1;
4519         } else {
4520                 int j;
4521
4522                 /* Check if response len is updated */
4523                 for (i = 0; i < tmo_count; i++) {
4524                         /* Abort the wait for completion if the FW health
4525                          * check has failed.
4526                          */
4527                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4528                                 return -EBUSY;
4529                         len = le16_to_cpu(resp->resp_len);
4530                         if (len)
4531                                 break;
4532                         /* on first few passes, just barely sleep */
4533                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4534                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4535                                              HWRM_SHORT_MAX_TIMEOUT);
4536                         else
4537                                 usleep_range(HWRM_MIN_TIMEOUT,
4538                                              HWRM_MAX_TIMEOUT);
4539                 }
4540
4541                 if (i >= tmo_count) {
4542                         if (!silent)
4543                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4544                                            HWRM_TOTAL_TIMEOUT(i),
4545                                            le16_to_cpu(req->req_type),
4546                                            le16_to_cpu(req->seq_id), len);
4547                         return -EBUSY;
4548                 }
4549
4550                 /* Last byte of resp contains valid bit */
4551                 valid = ((u8 *)resp) + len - 1;
4552                 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4553                         /* make sure we read from updated DMA memory */
4554                         dma_rmb();
4555                         if (*valid)
4556                                 break;
4557                         usleep_range(1, 5);
4558                 }
4559
4560                 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4561                         if (!silent)
4562                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4563                                            HWRM_TOTAL_TIMEOUT(i),
4564                                            le16_to_cpu(req->req_type),
4565                                            le16_to_cpu(req->seq_id), len,
4566                                            *valid);
4567                         return -EBUSY;
4568                 }
4569         }
4570
4571         /* Zero valid bit for compatibility.  Valid bit in an older spec
4572          * may become a new field in a newer spec.  We must make sure that
4573          * a new field not implemented by old spec will read zero.
4574          */
4575         *valid = 0;
4576         rc = le16_to_cpu(resp->error_code);
4577         if (rc && !silent)
4578                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4579                            le16_to_cpu(resp->req_type),
4580                            le16_to_cpu(resp->seq_id), rc);
4581         return bnxt_hwrm_to_stderr(rc);
4582 }
4583
4584 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4585 {
4586         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4587 }
4588
4589 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4590                               int timeout)
4591 {
4592         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4593 }
4594
4595 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4596 {
4597         int rc;
4598
4599         mutex_lock(&bp->hwrm_cmd_lock);
4600         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4601         mutex_unlock(&bp->hwrm_cmd_lock);
4602         return rc;
4603 }
4604
4605 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4606                              int timeout)
4607 {
4608         int rc;
4609
4610         mutex_lock(&bp->hwrm_cmd_lock);
4611         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4612         mutex_unlock(&bp->hwrm_cmd_lock);
4613         return rc;
4614 }
4615
4616 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4617                             bool async_only)
4618 {
4619         struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4620         struct hwrm_func_drv_rgtr_input req = {0};
4621         DECLARE_BITMAP(async_events_bmap, 256);
4622         u32 *events = (u32 *)async_events_bmap;
4623         u32 flags;
4624         int rc, i;
4625
4626         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4627
4628         req.enables =
4629                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4630                             FUNC_DRV_RGTR_REQ_ENABLES_VER |
4631                             FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4632
4633         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4634         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4635         if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4636                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4637         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4638                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4639                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4640         req.flags = cpu_to_le32(flags);
4641         req.ver_maj_8b = DRV_VER_MAJ;
4642         req.ver_min_8b = DRV_VER_MIN;
4643         req.ver_upd_8b = DRV_VER_UPD;
4644         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4645         req.ver_min = cpu_to_le16(DRV_VER_MIN);
4646         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4647
4648         if (BNXT_PF(bp)) {
4649                 u32 data[8];
4650                 int i;
4651
4652                 memset(data, 0, sizeof(data));
4653                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4654                         u16 cmd = bnxt_vf_req_snif[i];
4655                         unsigned int bit, idx;
4656
4657                         idx = cmd / 32;
4658                         bit = cmd % 32;
4659                         data[idx] |= 1 << bit;
4660                 }
4661
4662                 for (i = 0; i < 8; i++)
4663                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4664
4665                 req.enables |=
4666                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4667         }
4668
4669         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4670                 req.flags |= cpu_to_le32(
4671                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4672
4673         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4674         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4675                 u16 event_id = bnxt_async_events_arr[i];
4676
4677                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4678                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4679                         continue;
4680                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4681         }
4682         if (bmap && bmap_size) {
4683                 for (i = 0; i < bmap_size; i++) {
4684                         if (test_bit(i, bmap))
4685                                 __set_bit(i, async_events_bmap);
4686                 }
4687         }
4688         for (i = 0; i < 8; i++)
4689                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4690
4691         if (async_only)
4692                 req.enables =
4693                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4694
4695         mutex_lock(&bp->hwrm_cmd_lock);
4696         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4697         if (!rc) {
4698                 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4699                 if (resp->flags &
4700                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4701                         bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4702         }
4703         mutex_unlock(&bp->hwrm_cmd_lock);
4704         return rc;
4705 }
4706
4707 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4708 {
4709         struct hwrm_func_drv_unrgtr_input req = {0};
4710
4711         if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4712                 return 0;
4713
4714         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4715         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4716 }
4717
4718 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4719 {
4720         u32 rc = 0;
4721         struct hwrm_tunnel_dst_port_free_input req = {0};
4722
4723         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4724         req.tunnel_type = tunnel_type;
4725
4726         switch (tunnel_type) {
4727         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4728                 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4729                 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4730                 break;
4731         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4732                 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4733                 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4734                 break;
4735         default:
4736                 break;
4737         }
4738
4739         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4740         if (rc)
4741                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4742                            rc);
4743         return rc;
4744 }
4745
4746 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4747                                            u8 tunnel_type)
4748 {
4749         u32 rc = 0;
4750         struct hwrm_tunnel_dst_port_alloc_input req = {0};
4751         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4752
4753         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4754
4755         req.tunnel_type = tunnel_type;
4756         req.tunnel_dst_port_val = port;
4757
4758         mutex_lock(&bp->hwrm_cmd_lock);
4759         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4760         if (rc) {
4761                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4762                            rc);
4763                 goto err_out;
4764         }
4765
4766         switch (tunnel_type) {
4767         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4768                 bp->vxlan_fw_dst_port_id =
4769                         le16_to_cpu(resp->tunnel_dst_port_id);
4770                 break;
4771         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4772                 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4773                 break;
4774         default:
4775                 break;
4776         }
4777
4778 err_out:
4779         mutex_unlock(&bp->hwrm_cmd_lock);
4780         return rc;
4781 }
4782
4783 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4784 {
4785         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4786         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4787
4788         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4789         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4790
4791         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4792         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4793         req.mask = cpu_to_le32(vnic->rx_mask);
4794         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4795 }
4796
4797 #ifdef CONFIG_RFS_ACCEL
4798 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4799                                             struct bnxt_ntuple_filter *fltr)
4800 {
4801         struct hwrm_cfa_ntuple_filter_free_input req = {0};
4802
4803         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4804         req.ntuple_filter_id = fltr->filter_id;
4805         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4806 }
4807
4808 #define BNXT_NTP_FLTR_FLAGS                                     \
4809         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4810          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4811          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4812          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4813          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4814          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4815          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4816          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4817          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4818          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4819          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4820          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4821          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4822          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4823
4824 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4825                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4826
4827 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4828                                              struct bnxt_ntuple_filter *fltr)
4829 {
4830         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4831         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4832         struct flow_keys *keys = &fltr->fkeys;
4833         struct bnxt_vnic_info *vnic;
4834         u32 flags = 0;
4835         int rc = 0;
4836
4837         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4838         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4839
4840         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4841                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4842                 req.dst_id = cpu_to_le16(fltr->rxq);
4843         } else {
4844                 vnic = &bp->vnic_info[fltr->rxq + 1];
4845                 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4846         }
4847         req.flags = cpu_to_le32(flags);
4848         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4849
4850         req.ethertype = htons(ETH_P_IP);
4851         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4852         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4853         req.ip_protocol = keys->basic.ip_proto;
4854
4855         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4856                 int i;
4857
4858                 req.ethertype = htons(ETH_P_IPV6);
4859                 req.ip_addr_type =
4860                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4861                 *(struct in6_addr *)&req.src_ipaddr[0] =
4862                         keys->addrs.v6addrs.src;
4863                 *(struct in6_addr *)&req.dst_ipaddr[0] =
4864                         keys->addrs.v6addrs.dst;
4865                 for (i = 0; i < 4; i++) {
4866                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4867                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4868                 }
4869         } else {
4870                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4871                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4872                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4873                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4874         }
4875         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4876                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4877                 req.tunnel_type =
4878                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4879         }
4880
4881         req.src_port = keys->ports.src;
4882         req.src_port_mask = cpu_to_be16(0xffff);
4883         req.dst_port = keys->ports.dst;
4884         req.dst_port_mask = cpu_to_be16(0xffff);
4885
4886         mutex_lock(&bp->hwrm_cmd_lock);
4887         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4888         if (!rc) {
4889                 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4890                 fltr->filter_id = resp->ntuple_filter_id;
4891         }
4892         mutex_unlock(&bp->hwrm_cmd_lock);
4893         return rc;
4894 }
4895 #endif
4896
4897 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4898                                      u8 *mac_addr)
4899 {
4900         u32 rc = 0;
4901         struct hwrm_cfa_l2_filter_alloc_input req = {0};
4902         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4903
4904         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4905         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4906         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4907                 req.flags |=
4908                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4909         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4910         req.enables =
4911                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4912                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4913                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4914         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4915         req.l2_addr_mask[0] = 0xff;
4916         req.l2_addr_mask[1] = 0xff;
4917         req.l2_addr_mask[2] = 0xff;
4918         req.l2_addr_mask[3] = 0xff;
4919         req.l2_addr_mask[4] = 0xff;
4920         req.l2_addr_mask[5] = 0xff;
4921
4922         mutex_lock(&bp->hwrm_cmd_lock);
4923         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4924         if (!rc)
4925                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4926                                                         resp->l2_filter_id;
4927         mutex_unlock(&bp->hwrm_cmd_lock);
4928         return rc;
4929 }
4930
4931 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4932 {
4933         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4934         int rc = 0;
4935
4936         /* Any associated ntuple filters will also be cleared by firmware. */
4937         mutex_lock(&bp->hwrm_cmd_lock);
4938         for (i = 0; i < num_of_vnics; i++) {
4939                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4940
4941                 for (j = 0; j < vnic->uc_filter_count; j++) {
4942                         struct hwrm_cfa_l2_filter_free_input req = {0};
4943
4944                         bnxt_hwrm_cmd_hdr_init(bp, &req,
4945                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
4946
4947                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
4948
4949                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4950                                                 HWRM_CMD_TIMEOUT);
4951                 }
4952                 vnic->uc_filter_count = 0;
4953         }
4954         mutex_unlock(&bp->hwrm_cmd_lock);
4955
4956         return rc;
4957 }
4958
4959 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4960 {
4961         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4962         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4963         struct hwrm_vnic_tpa_cfg_input req = {0};
4964
4965         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4966                 return 0;
4967
4968         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4969
4970         if (tpa_flags) {
4971                 u16 mss = bp->dev->mtu - 40;
4972                 u32 nsegs, n, segs = 0, flags;
4973
4974                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4975                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4976                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4977                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4978                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4979                 if (tpa_flags & BNXT_FLAG_GRO)
4980                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4981
4982                 req.flags = cpu_to_le32(flags);
4983
4984                 req.enables =
4985                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4986                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4987                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4988
4989                 /* Number of segs are log2 units, and first packet is not
4990                  * included as part of this units.
4991                  */
4992                 if (mss <= BNXT_RX_PAGE_SIZE) {
4993                         n = BNXT_RX_PAGE_SIZE / mss;
4994                         nsegs = (MAX_SKB_FRAGS - 1) * n;
4995                 } else {
4996                         n = mss / BNXT_RX_PAGE_SIZE;
4997                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
4998                                 n++;
4999                         nsegs = (MAX_SKB_FRAGS - n) / n;
5000                 }
5001
5002                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5003                         segs = MAX_TPA_SEGS_P5;
5004                         max_aggs = bp->max_tpa;
5005                 } else {
5006                         segs = ilog2(nsegs);
5007                 }
5008                 req.max_agg_segs = cpu_to_le16(segs);
5009                 req.max_aggs = cpu_to_le16(max_aggs);
5010
5011                 req.min_agg_len = cpu_to_le32(512);
5012         }
5013         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5014
5015         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5016 }
5017
5018 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
5019 {
5020         struct bnxt_ring_grp_info *grp_info;
5021
5022         grp_info = &bp->grp_info[ring->grp_idx];
5023         return grp_info->cp_fw_ring_id;
5024 }
5025
5026 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
5027 {
5028         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5029                 struct bnxt_napi *bnapi = rxr->bnapi;
5030                 struct bnxt_cp_ring_info *cpr;
5031
5032                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
5033                 return cpr->cp_ring_struct.fw_ring_id;
5034         } else {
5035                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
5036         }
5037 }
5038
5039 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
5040 {
5041         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5042                 struct bnxt_napi *bnapi = txr->bnapi;
5043                 struct bnxt_cp_ring_info *cpr;
5044
5045                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
5046                 return cpr->cp_ring_struct.fw_ring_id;
5047         } else {
5048                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
5049         }
5050 }
5051
5052 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5053 {
5054         int entries;
5055
5056         if (bp->flags & BNXT_FLAG_CHIP_P5)
5057                 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5058         else
5059                 entries = HW_HASH_INDEX_SIZE;
5060
5061         bp->rss_indir_tbl_entries = entries;
5062         bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5063                                           GFP_KERNEL);
5064         if (!bp->rss_indir_tbl)
5065                 return -ENOMEM;
5066         return 0;
5067 }
5068
5069 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5070 {
5071         u16 max_rings, max_entries, pad, i;
5072
5073         if (!bp->rx_nr_rings)
5074                 return;
5075
5076         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5077                 max_rings = bp->rx_nr_rings - 1;
5078         else
5079                 max_rings = bp->rx_nr_rings;
5080
5081         max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5082
5083         for (i = 0; i < max_entries; i++)
5084                 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5085
5086         pad = bp->rss_indir_tbl_entries - max_entries;
5087         if (pad)
5088                 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5089 }
5090
5091 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5092 {
5093         u16 i, tbl_size, max_ring = 0;
5094
5095         if (!bp->rss_indir_tbl)
5096                 return 0;
5097
5098         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5099         for (i = 0; i < tbl_size; i++)
5100                 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5101         return max_ring;
5102 }
5103
5104 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5105 {
5106         if (bp->flags & BNXT_FLAG_CHIP_P5)
5107                 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5108         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5109                 return 2;
5110         return 1;
5111 }
5112
5113 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5114 {
5115         bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5116         u16 i, j;
5117
5118         /* Fill the RSS indirection table with ring group ids */
5119         for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5120                 if (!no_rss)
5121                         j = bp->rss_indir_tbl[i];
5122                 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5123         }
5124 }
5125
5126 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5127                                       struct bnxt_vnic_info *vnic)
5128 {
5129         __le16 *ring_tbl = vnic->rss_table;
5130         struct bnxt_rx_ring_info *rxr;
5131         u16 tbl_size, i;
5132
5133         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5134
5135         for (i = 0; i < tbl_size; i++) {
5136                 u16 ring_id, j;
5137
5138                 j = bp->rss_indir_tbl[i];
5139                 rxr = &bp->rx_ring[j];
5140
5141                 ring_id = rxr->rx_ring_struct.fw_ring_id;
5142                 *ring_tbl++ = cpu_to_le16(ring_id);
5143                 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5144                 *ring_tbl++ = cpu_to_le16(ring_id);
5145         }
5146 }
5147
5148 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5149 {
5150         if (bp->flags & BNXT_FLAG_CHIP_P5)
5151                 __bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5152         else
5153                 __bnxt_fill_hw_rss_tbl(bp, vnic);
5154 }
5155
5156 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5157 {
5158         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5159         struct hwrm_vnic_rss_cfg_input req = {0};
5160
5161         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5162             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5163                 return 0;
5164
5165         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5166         if (set_rss) {
5167                 bnxt_fill_hw_rss_tbl(bp, vnic);
5168                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5169                 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5170                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5171                 req.hash_key_tbl_addr =
5172                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
5173         }
5174         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5175         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5176 }
5177
5178 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5179 {
5180         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5181         struct hwrm_vnic_rss_cfg_input req = {0};
5182         dma_addr_t ring_tbl_map;
5183         u32 i, nr_ctxs;
5184
5185         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5186         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5187         if (!set_rss) {
5188                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5189                 return 0;
5190         }
5191         bnxt_fill_hw_rss_tbl(bp, vnic);
5192         req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5193         req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5194         req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5195         ring_tbl_map = vnic->rss_table_dma_addr;
5196         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5197         for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5198                 int rc;
5199
5200                 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5201                 req.ring_table_pair_index = i;
5202                 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5203                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5204                 if (rc)
5205                         return rc;
5206         }
5207         return 0;
5208 }
5209
5210 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5211 {
5212         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5213         struct hwrm_vnic_plcmodes_cfg_input req = {0};
5214
5215         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
5216         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
5217                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5218                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5219         req.enables =
5220                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
5221                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5222         /* thresholds not implemented in firmware yet */
5223         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5224         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5225         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5226         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5227 }
5228
5229 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5230                                         u16 ctx_idx)
5231 {
5232         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
5233
5234         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
5235         req.rss_cos_lb_ctx_id =
5236                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5237
5238         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5239         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5240 }
5241
5242 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5243 {
5244         int i, j;
5245
5246         for (i = 0; i < bp->nr_vnics; i++) {
5247                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5248
5249                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5250                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5251                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5252                 }
5253         }
5254         bp->rsscos_nr_ctxs = 0;
5255 }
5256
5257 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5258 {
5259         int rc;
5260         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
5261         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
5262                                                 bp->hwrm_cmd_resp_addr;
5263
5264         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
5265                                -1);
5266
5267         mutex_lock(&bp->hwrm_cmd_lock);
5268         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5269         if (!rc)
5270                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5271                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
5272         mutex_unlock(&bp->hwrm_cmd_lock);
5273
5274         return rc;
5275 }
5276
5277 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5278 {
5279         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5280                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5281         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5282 }
5283
5284 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5285 {
5286         unsigned int ring = 0, grp_idx;
5287         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5288         struct hwrm_vnic_cfg_input req = {0};
5289         u16 def_vlan = 0;
5290
5291         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
5292
5293         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5294                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5295
5296                 req.default_rx_ring_id =
5297                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5298                 req.default_cmpl_ring_id =
5299                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5300                 req.enables =
5301                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5302                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5303                 goto vnic_mru;
5304         }
5305         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5306         /* Only RSS support for now TBD: COS & LB */
5307         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5308                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5309                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5310                                            VNIC_CFG_REQ_ENABLES_MRU);
5311         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5312                 req.rss_rule =
5313                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5314                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5315                                            VNIC_CFG_REQ_ENABLES_MRU);
5316                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5317         } else {
5318                 req.rss_rule = cpu_to_le16(0xffff);
5319         }
5320
5321         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5322             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5323                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5324                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5325         } else {
5326                 req.cos_rule = cpu_to_le16(0xffff);
5327         }
5328
5329         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5330                 ring = 0;
5331         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5332                 ring = vnic_id - 1;
5333         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5334                 ring = bp->rx_nr_rings - 1;
5335
5336         grp_idx = bp->rx_ring[ring].bnapi->index;
5337         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5338         req.lb_rule = cpu_to_le16(0xffff);
5339 vnic_mru:
5340         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5341
5342         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5343 #ifdef CONFIG_BNXT_SRIOV
5344         if (BNXT_VF(bp))
5345                 def_vlan = bp->vf.vlan;
5346 #endif
5347         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5348                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5349         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5350                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5351
5352         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5353 }
5354
5355 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5356 {
5357         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5358                 struct hwrm_vnic_free_input req = {0};
5359
5360                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5361                 req.vnic_id =
5362                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5363
5364                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5365                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5366         }
5367 }
5368
5369 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5370 {
5371         u16 i;
5372
5373         for (i = 0; i < bp->nr_vnics; i++)
5374                 bnxt_hwrm_vnic_free_one(bp, i);
5375 }
5376
5377 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5378                                 unsigned int start_rx_ring_idx,
5379                                 unsigned int nr_rings)
5380 {
5381         int rc = 0;
5382         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5383         struct hwrm_vnic_alloc_input req = {0};
5384         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5385         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5386
5387         if (bp->flags & BNXT_FLAG_CHIP_P5)
5388                 goto vnic_no_ring_grps;
5389
5390         /* map ring groups to this vnic */
5391         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5392                 grp_idx = bp->rx_ring[i].bnapi->index;
5393                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5394                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5395                                    j, nr_rings);
5396                         break;
5397                 }
5398                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5399         }
5400
5401 vnic_no_ring_grps:
5402         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5403                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5404         if (vnic_id == 0)
5405                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5406
5407         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5408
5409         mutex_lock(&bp->hwrm_cmd_lock);
5410         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5411         if (!rc)
5412                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5413         mutex_unlock(&bp->hwrm_cmd_lock);
5414         return rc;
5415 }
5416
5417 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5418 {
5419         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5420         struct hwrm_vnic_qcaps_input req = {0};
5421         int rc;
5422
5423         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5424         bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5425         if (bp->hwrm_spec_code < 0x10600)
5426                 return 0;
5427
5428         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5429         mutex_lock(&bp->hwrm_cmd_lock);
5430         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5431         if (!rc) {
5432                 u32 flags = le32_to_cpu(resp->flags);
5433
5434                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5435                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5436                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5437                 if (flags &
5438                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5439                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5440
5441                 /* Older P5 fw before EXT_HW_STATS support did not set
5442                  * VLAN_STRIP_CAP properly.
5443                  */
5444                 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5445                     (BNXT_CHIP_P5_THOR(bp) &&
5446                      !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5447                         bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5448                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5449                 if (bp->max_tpa_v2) {
5450                         if (BNXT_CHIP_P5_THOR(bp))
5451                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5452                         else
5453                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5454                 }
5455         }
5456         mutex_unlock(&bp->hwrm_cmd_lock);
5457         return rc;
5458 }
5459
5460 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5461 {
5462         u16 i;
5463         u32 rc = 0;
5464
5465         if (bp->flags & BNXT_FLAG_CHIP_P5)
5466                 return 0;
5467
5468         mutex_lock(&bp->hwrm_cmd_lock);
5469         for (i = 0; i < bp->rx_nr_rings; i++) {
5470                 struct hwrm_ring_grp_alloc_input req = {0};
5471                 struct hwrm_ring_grp_alloc_output *resp =
5472                                         bp->hwrm_cmd_resp_addr;
5473                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5474
5475                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5476
5477                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5478                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5479                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5480                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5481
5482                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5483                                         HWRM_CMD_TIMEOUT);
5484                 if (rc)
5485                         break;
5486
5487                 bp->grp_info[grp_idx].fw_grp_id =
5488                         le32_to_cpu(resp->ring_group_id);
5489         }
5490         mutex_unlock(&bp->hwrm_cmd_lock);
5491         return rc;
5492 }
5493
5494 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5495 {
5496         u16 i;
5497         struct hwrm_ring_grp_free_input req = {0};
5498
5499         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5500                 return;
5501
5502         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5503
5504         mutex_lock(&bp->hwrm_cmd_lock);
5505         for (i = 0; i < bp->cp_nr_rings; i++) {
5506                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5507                         continue;
5508                 req.ring_group_id =
5509                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5510
5511                 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5512                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5513         }
5514         mutex_unlock(&bp->hwrm_cmd_lock);
5515 }
5516
5517 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5518                                     struct bnxt_ring_struct *ring,
5519                                     u32 ring_type, u32 map_index)
5520 {
5521         int rc = 0, err = 0;
5522         struct hwrm_ring_alloc_input req = {0};
5523         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5524         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5525         struct bnxt_ring_grp_info *grp_info;
5526         u16 ring_id;
5527
5528         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5529
5530         req.enables = 0;
5531         if (rmem->nr_pages > 1) {
5532                 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5533                 /* Page size is in log2 units */
5534                 req.page_size = BNXT_PAGE_SHIFT;
5535                 req.page_tbl_depth = 1;
5536         } else {
5537                 req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5538         }
5539         req.fbo = 0;
5540         /* Association of ring index with doorbell index and MSIX number */
5541         req.logical_id = cpu_to_le16(map_index);
5542
5543         switch (ring_type) {
5544         case HWRM_RING_ALLOC_TX: {
5545                 struct bnxt_tx_ring_info *txr;
5546
5547                 txr = container_of(ring, struct bnxt_tx_ring_info,
5548                                    tx_ring_struct);
5549                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5550                 /* Association of transmit ring with completion ring */
5551                 grp_info = &bp->grp_info[ring->grp_idx];
5552                 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5553                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5554                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5555                 req.queue_id = cpu_to_le16(ring->queue_id);
5556                 break;
5557         }
5558         case HWRM_RING_ALLOC_RX:
5559                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5560                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5561                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5562                         u16 flags = 0;
5563
5564                         /* Association of rx ring with stats context */
5565                         grp_info = &bp->grp_info[ring->grp_idx];
5566                         req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5567                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5568                         req.enables |= cpu_to_le32(
5569                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5570                         if (NET_IP_ALIGN == 2)
5571                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5572                         req.flags = cpu_to_le16(flags);
5573                 }
5574                 break;
5575         case HWRM_RING_ALLOC_AGG:
5576                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5577                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5578                         /* Association of agg ring with rx ring */
5579                         grp_info = &bp->grp_info[ring->grp_idx];
5580                         req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5581                         req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5582                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5583                         req.enables |= cpu_to_le32(
5584                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5585                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5586                 } else {
5587                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5588                 }
5589                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5590                 break;
5591         case HWRM_RING_ALLOC_CMPL:
5592                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5593                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5594                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5595                         /* Association of cp ring with nq */
5596                         grp_info = &bp->grp_info[map_index];
5597                         req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5598                         req.cq_handle = cpu_to_le64(ring->handle);
5599                         req.enables |= cpu_to_le32(
5600                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5601                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5602                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5603                 }
5604                 break;
5605         case HWRM_RING_ALLOC_NQ:
5606                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5607                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5608                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5609                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5610                 break;
5611         default:
5612                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5613                            ring_type);
5614                 return -1;
5615         }
5616
5617         mutex_lock(&bp->hwrm_cmd_lock);
5618         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5619         err = le16_to_cpu(resp->error_code);
5620         ring_id = le16_to_cpu(resp->ring_id);
5621         mutex_unlock(&bp->hwrm_cmd_lock);
5622
5623         if (rc || err) {
5624                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5625                            ring_type, rc, err);
5626                 return -EIO;
5627         }
5628         ring->fw_ring_id = ring_id;
5629         return rc;
5630 }
5631
5632 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5633 {
5634         int rc;
5635
5636         if (BNXT_PF(bp)) {
5637                 struct hwrm_func_cfg_input req = {0};
5638
5639                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5640                 req.fid = cpu_to_le16(0xffff);
5641                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5642                 req.async_event_cr = cpu_to_le16(idx);
5643                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5644         } else {
5645                 struct hwrm_func_vf_cfg_input req = {0};
5646
5647                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5648                 req.enables =
5649                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5650                 req.async_event_cr = cpu_to_le16(idx);
5651                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5652         }
5653         return rc;
5654 }
5655
5656 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5657                         u32 map_idx, u32 xid)
5658 {
5659         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5660                 if (BNXT_PF(bp))
5661                         db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5662                 else
5663                         db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5664                 switch (ring_type) {
5665                 case HWRM_RING_ALLOC_TX:
5666                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5667                         break;
5668                 case HWRM_RING_ALLOC_RX:
5669                 case HWRM_RING_ALLOC_AGG:
5670                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5671                         break;
5672                 case HWRM_RING_ALLOC_CMPL:
5673                         db->db_key64 = DBR_PATH_L2;
5674                         break;
5675                 case HWRM_RING_ALLOC_NQ:
5676                         db->db_key64 = DBR_PATH_L2;
5677                         break;
5678                 }
5679                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5680         } else {
5681                 db->doorbell = bp->bar1 + map_idx * 0x80;
5682                 switch (ring_type) {
5683                 case HWRM_RING_ALLOC_TX:
5684                         db->db_key32 = DB_KEY_TX;
5685                         break;
5686                 case HWRM_RING_ALLOC_RX:
5687                 case HWRM_RING_ALLOC_AGG:
5688                         db->db_key32 = DB_KEY_RX;
5689                         break;
5690                 case HWRM_RING_ALLOC_CMPL:
5691                         db->db_key32 = DB_KEY_CP;
5692                         break;
5693                 }
5694         }
5695 }
5696
5697 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5698 {
5699         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5700         int i, rc = 0;
5701         u32 type;
5702
5703         if (bp->flags & BNXT_FLAG_CHIP_P5)
5704                 type = HWRM_RING_ALLOC_NQ;
5705         else
5706                 type = HWRM_RING_ALLOC_CMPL;
5707         for (i = 0; i < bp->cp_nr_rings; i++) {
5708                 struct bnxt_napi *bnapi = bp->bnapi[i];
5709                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5710                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5711                 u32 map_idx = ring->map_idx;
5712                 unsigned int vector;
5713
5714                 vector = bp->irq_tbl[map_idx].vector;
5715                 disable_irq_nosync(vector);
5716                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5717                 if (rc) {
5718                         enable_irq(vector);
5719                         goto err_out;
5720                 }
5721                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5722                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5723                 enable_irq(vector);
5724                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5725
5726                 if (!i) {
5727                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5728                         if (rc)
5729                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5730                 }
5731         }
5732
5733         type = HWRM_RING_ALLOC_TX;
5734         for (i = 0; i < bp->tx_nr_rings; i++) {
5735                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5736                 struct bnxt_ring_struct *ring;
5737                 u32 map_idx;
5738
5739                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5740                         struct bnxt_napi *bnapi = txr->bnapi;
5741                         struct bnxt_cp_ring_info *cpr, *cpr2;
5742                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5743
5744                         cpr = &bnapi->cp_ring;
5745                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5746                         ring = &cpr2->cp_ring_struct;
5747                         ring->handle = BNXT_TX_HDL;
5748                         map_idx = bnapi->index;
5749                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5750                         if (rc)
5751                                 goto err_out;
5752                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5753                                     ring->fw_ring_id);
5754                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5755                 }
5756                 ring = &txr->tx_ring_struct;
5757                 map_idx = i;
5758                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5759                 if (rc)
5760                         goto err_out;
5761                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5762         }
5763
5764         type = HWRM_RING_ALLOC_RX;
5765         for (i = 0; i < bp->rx_nr_rings; i++) {
5766                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5767                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5768                 struct bnxt_napi *bnapi = rxr->bnapi;
5769                 u32 map_idx = bnapi->index;
5770
5771                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5772                 if (rc)
5773                         goto err_out;
5774                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5775                 /* If we have agg rings, post agg buffers first. */
5776                 if (!agg_rings)
5777                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5778                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5779                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5780                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5781                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5782                         struct bnxt_cp_ring_info *cpr2;
5783
5784                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5785                         ring = &cpr2->cp_ring_struct;
5786                         ring->handle = BNXT_RX_HDL;
5787                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5788                         if (rc)
5789                                 goto err_out;
5790                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5791                                     ring->fw_ring_id);
5792                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5793                 }
5794         }
5795
5796         if (agg_rings) {
5797                 type = HWRM_RING_ALLOC_AGG;
5798                 for (i = 0; i < bp->rx_nr_rings; i++) {
5799                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5800                         struct bnxt_ring_struct *ring =
5801                                                 &rxr->rx_agg_ring_struct;
5802                         u32 grp_idx = ring->grp_idx;
5803                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5804
5805                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5806                         if (rc)
5807                                 goto err_out;
5808
5809                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5810                                     ring->fw_ring_id);
5811                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5812                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5813                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5814                 }
5815         }
5816 err_out:
5817         return rc;
5818 }
5819
5820 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5821                                    struct bnxt_ring_struct *ring,
5822                                    u32 ring_type, int cmpl_ring_id)
5823 {
5824         int rc;
5825         struct hwrm_ring_free_input req = {0};
5826         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5827         u16 error_code;
5828
5829         if (BNXT_NO_FW_ACCESS(bp))
5830                 return 0;
5831
5832         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5833         req.ring_type = ring_type;
5834         req.ring_id = cpu_to_le16(ring->fw_ring_id);
5835
5836         mutex_lock(&bp->hwrm_cmd_lock);
5837         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5838         error_code = le16_to_cpu(resp->error_code);
5839         mutex_unlock(&bp->hwrm_cmd_lock);
5840
5841         if (rc || error_code) {
5842                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5843                            ring_type, rc, error_code);
5844                 return -EIO;
5845         }
5846         return 0;
5847 }
5848
5849 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5850 {
5851         u32 type;
5852         int i;
5853
5854         if (!bp->bnapi)
5855                 return;
5856
5857         for (i = 0; i < bp->tx_nr_rings; i++) {
5858                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5859                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5860
5861                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5862                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5863
5864                         hwrm_ring_free_send_msg(bp, ring,
5865                                                 RING_FREE_REQ_RING_TYPE_TX,
5866                                                 close_path ? cmpl_ring_id :
5867                                                 INVALID_HW_RING_ID);
5868                         ring->fw_ring_id = INVALID_HW_RING_ID;
5869                 }
5870         }
5871
5872         for (i = 0; i < bp->rx_nr_rings; i++) {
5873                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5874                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5875                 u32 grp_idx = rxr->bnapi->index;
5876
5877                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5878                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5879
5880                         hwrm_ring_free_send_msg(bp, ring,
5881                                                 RING_FREE_REQ_RING_TYPE_RX,
5882                                                 close_path ? cmpl_ring_id :
5883                                                 INVALID_HW_RING_ID);
5884                         ring->fw_ring_id = INVALID_HW_RING_ID;
5885                         bp->grp_info[grp_idx].rx_fw_ring_id =
5886                                 INVALID_HW_RING_ID;
5887                 }
5888         }
5889
5890         if (bp->flags & BNXT_FLAG_CHIP_P5)
5891                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5892         else
5893                 type = RING_FREE_REQ_RING_TYPE_RX;
5894         for (i = 0; i < bp->rx_nr_rings; i++) {
5895                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5896                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5897                 u32 grp_idx = rxr->bnapi->index;
5898
5899                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5900                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5901
5902                         hwrm_ring_free_send_msg(bp, ring, type,
5903                                                 close_path ? cmpl_ring_id :
5904                                                 INVALID_HW_RING_ID);
5905                         ring->fw_ring_id = INVALID_HW_RING_ID;
5906                         bp->grp_info[grp_idx].agg_fw_ring_id =
5907                                 INVALID_HW_RING_ID;
5908                 }
5909         }
5910
5911         /* The completion rings are about to be freed.  After that the
5912          * IRQ doorbell will not work anymore.  So we need to disable
5913          * IRQ here.
5914          */
5915         bnxt_disable_int_sync(bp);
5916
5917         if (bp->flags & BNXT_FLAG_CHIP_P5)
5918                 type = RING_FREE_REQ_RING_TYPE_NQ;
5919         else
5920                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5921         for (i = 0; i < bp->cp_nr_rings; i++) {
5922                 struct bnxt_napi *bnapi = bp->bnapi[i];
5923                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5924                 struct bnxt_ring_struct *ring;
5925                 int j;
5926
5927                 for (j = 0; j < 2; j++) {
5928                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5929
5930                         if (cpr2) {
5931                                 ring = &cpr2->cp_ring_struct;
5932                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5933                                         continue;
5934                                 hwrm_ring_free_send_msg(bp, ring,
5935                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
5936                                         INVALID_HW_RING_ID);
5937                                 ring->fw_ring_id = INVALID_HW_RING_ID;
5938                         }
5939                 }
5940                 ring = &cpr->cp_ring_struct;
5941                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5942                         hwrm_ring_free_send_msg(bp, ring, type,
5943                                                 INVALID_HW_RING_ID);
5944                         ring->fw_ring_id = INVALID_HW_RING_ID;
5945                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5946                 }
5947         }
5948 }
5949
5950 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5951                            bool shared);
5952
5953 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5954 {
5955         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5956         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5957         struct hwrm_func_qcfg_input req = {0};
5958         int rc;
5959
5960         if (bp->hwrm_spec_code < 0x10601)
5961                 return 0;
5962
5963         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5964         req.fid = cpu_to_le16(0xffff);
5965         mutex_lock(&bp->hwrm_cmd_lock);
5966         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5967         if (rc) {
5968                 mutex_unlock(&bp->hwrm_cmd_lock);
5969                 return rc;
5970         }
5971
5972         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5973         if (BNXT_NEW_RM(bp)) {
5974                 u16 cp, stats;
5975
5976                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5977                 hw_resc->resv_hw_ring_grps =
5978                         le32_to_cpu(resp->alloc_hw_ring_grps);
5979                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5980                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5981                 stats = le16_to_cpu(resp->alloc_stat_ctx);
5982                 hw_resc->resv_irqs = cp;
5983                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5984                         int rx = hw_resc->resv_rx_rings;
5985                         int tx = hw_resc->resv_tx_rings;
5986
5987                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5988                                 rx >>= 1;
5989                         if (cp < (rx + tx)) {
5990                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5991                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5992                                         rx <<= 1;
5993                                 hw_resc->resv_rx_rings = rx;
5994                                 hw_resc->resv_tx_rings = tx;
5995                         }
5996                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5997                         hw_resc->resv_hw_ring_grps = rx;
5998                 }
5999                 hw_resc->resv_cp_rings = cp;
6000                 hw_resc->resv_stat_ctxs = stats;
6001         }
6002         mutex_unlock(&bp->hwrm_cmd_lock);
6003         return 0;
6004 }
6005
6006 /* Caller must hold bp->hwrm_cmd_lock */
6007 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
6008 {
6009         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6010         struct hwrm_func_qcfg_input req = {0};
6011         int rc;
6012
6013         if (bp->hwrm_spec_code < 0x10601)
6014                 return 0;
6015
6016         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6017         req.fid = cpu_to_le16(fid);
6018         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6019         if (!rc)
6020                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6021
6022         return rc;
6023 }
6024
6025 static bool bnxt_rfs_supported(struct bnxt *bp);
6026
6027 static void
6028 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
6029                              int tx_rings, int rx_rings, int ring_grps,
6030                              int cp_rings, int stats, int vnics)
6031 {
6032         u32 enables = 0;
6033
6034         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
6035         req->fid = cpu_to_le16(0xffff);
6036         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6037         req->num_tx_rings = cpu_to_le16(tx_rings);
6038         if (BNXT_NEW_RM(bp)) {
6039                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
6040                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6041                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6042                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
6043                         enables |= tx_rings + ring_grps ?
6044                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6045                         enables |= rx_rings ?
6046                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6047                 } else {
6048                         enables |= cp_rings ?
6049                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6050                         enables |= ring_grps ?
6051                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6052                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6053                 }
6054                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6055
6056                 req->num_rx_rings = cpu_to_le16(rx_rings);
6057                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6058                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6059                         req->num_msix = cpu_to_le16(cp_rings);
6060                         req->num_rsscos_ctxs =
6061                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6062                 } else {
6063                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
6064                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6065                         req->num_rsscos_ctxs = cpu_to_le16(1);
6066                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6067                             bnxt_rfs_supported(bp))
6068                                 req->num_rsscos_ctxs =
6069                                         cpu_to_le16(ring_grps + 1);
6070                 }
6071                 req->num_stat_ctxs = cpu_to_le16(stats);
6072                 req->num_vnics = cpu_to_le16(vnics);
6073         }
6074         req->enables = cpu_to_le32(enables);
6075 }
6076
6077 static void
6078 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
6079                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
6080                              int rx_rings, int ring_grps, int cp_rings,
6081                              int stats, int vnics)
6082 {
6083         u32 enables = 0;
6084
6085         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
6086         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6087         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6088                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6089         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6090         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6091                 enables |= tx_rings + ring_grps ?
6092                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6093         } else {
6094                 enables |= cp_rings ?
6095                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6096                 enables |= ring_grps ?
6097                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6098         }
6099         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6100         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6101
6102         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6103         req->num_tx_rings = cpu_to_le16(tx_rings);
6104         req->num_rx_rings = cpu_to_le16(rx_rings);
6105         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6106                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6107                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6108         } else {
6109                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
6110                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6111                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6112         }
6113         req->num_stat_ctxs = cpu_to_le16(stats);
6114         req->num_vnics = cpu_to_le16(vnics);
6115
6116         req->enables = cpu_to_le32(enables);
6117 }
6118
6119 static int
6120 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6121                            int ring_grps, int cp_rings, int stats, int vnics)
6122 {
6123         struct hwrm_func_cfg_input req = {0};
6124         int rc;
6125
6126         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6127                                      cp_rings, stats, vnics);
6128         if (!req.enables)
6129                 return 0;
6130
6131         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6132         if (rc)
6133                 return rc;
6134
6135         if (bp->hwrm_spec_code < 0x10601)
6136                 bp->hw_resc.resv_tx_rings = tx_rings;
6137
6138         return bnxt_hwrm_get_rings(bp);
6139 }
6140
6141 static int
6142 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6143                            int ring_grps, int cp_rings, int stats, int vnics)
6144 {
6145         struct hwrm_func_vf_cfg_input req = {0};
6146         int rc;
6147
6148         if (!BNXT_NEW_RM(bp)) {
6149                 bp->hw_resc.resv_tx_rings = tx_rings;
6150                 return 0;
6151         }
6152
6153         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6154                                      cp_rings, stats, vnics);
6155         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6156         if (rc)
6157                 return rc;
6158
6159         return bnxt_hwrm_get_rings(bp);
6160 }
6161
6162 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6163                                    int cp, int stat, int vnic)
6164 {
6165         if (BNXT_PF(bp))
6166                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6167                                                   vnic);
6168         else
6169                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6170                                                   vnic);
6171 }
6172
6173 int bnxt_nq_rings_in_use(struct bnxt *bp)
6174 {
6175         int cp = bp->cp_nr_rings;
6176         int ulp_msix, ulp_base;
6177
6178         ulp_msix = bnxt_get_ulp_msix_num(bp);
6179         if (ulp_msix) {
6180                 ulp_base = bnxt_get_ulp_msix_base(bp);
6181                 cp += ulp_msix;
6182                 if ((ulp_base + ulp_msix) > cp)
6183                         cp = ulp_base + ulp_msix;
6184         }
6185         return cp;
6186 }
6187
6188 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6189 {
6190         int cp;
6191
6192         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6193                 return bnxt_nq_rings_in_use(bp);
6194
6195         cp = bp->tx_nr_rings + bp->rx_nr_rings;
6196         return cp;
6197 }
6198
6199 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6200 {
6201         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6202         int cp = bp->cp_nr_rings;
6203
6204         if (!ulp_stat)
6205                 return cp;
6206
6207         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6208                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6209
6210         return cp + ulp_stat;
6211 }
6212
6213 /* Check if a default RSS map needs to be setup.  This function is only
6214  * used on older firmware that does not require reserving RX rings.
6215  */
6216 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6217 {
6218         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6219
6220         /* The RSS map is valid for RX rings set to resv_rx_rings */
6221         if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6222                 hw_resc->resv_rx_rings = bp->rx_nr_rings;
6223                 if (!netif_is_rxfh_configured(bp->dev))
6224                         bnxt_set_dflt_rss_indir_tbl(bp);
6225         }
6226 }
6227
6228 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6229 {
6230         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6231         int cp = bnxt_cp_rings_in_use(bp);
6232         int nq = bnxt_nq_rings_in_use(bp);
6233         int rx = bp->rx_nr_rings, stat;
6234         int vnic = 1, grp = rx;
6235
6236         if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6237             bp->hwrm_spec_code >= 0x10601)
6238                 return true;
6239
6240         /* Old firmware does not need RX ring reservations but we still
6241          * need to setup a default RSS map when needed.  With new firmware
6242          * we go through RX ring reservations first and then set up the
6243          * RSS map for the successfully reserved RX rings when needed.
6244          */
6245         if (!BNXT_NEW_RM(bp)) {
6246                 bnxt_check_rss_tbl_no_rmgr(bp);
6247                 return false;
6248         }
6249         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6250                 vnic = rx + 1;
6251         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6252                 rx <<= 1;
6253         stat = bnxt_get_func_stat_ctxs(bp);
6254         if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6255             hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6256             (hw_resc->resv_hw_ring_grps != grp &&
6257              !(bp->flags & BNXT_FLAG_CHIP_P5)))
6258                 return true;
6259         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6260             hw_resc->resv_irqs != nq)
6261                 return true;
6262         return false;
6263 }
6264
6265 static int __bnxt_reserve_rings(struct bnxt *bp)
6266 {
6267         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6268         int cp = bnxt_nq_rings_in_use(bp);
6269         int tx = bp->tx_nr_rings;
6270         int rx = bp->rx_nr_rings;
6271         int grp, rx_rings, rc;
6272         int vnic = 1, stat;
6273         bool sh = false;
6274
6275         if (!bnxt_need_reserve_rings(bp))
6276                 return 0;
6277
6278         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6279                 sh = true;
6280         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6281                 vnic = rx + 1;
6282         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6283                 rx <<= 1;
6284         grp = bp->rx_nr_rings;
6285         stat = bnxt_get_func_stat_ctxs(bp);
6286
6287         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6288         if (rc)
6289                 return rc;
6290
6291         tx = hw_resc->resv_tx_rings;
6292         if (BNXT_NEW_RM(bp)) {
6293                 rx = hw_resc->resv_rx_rings;
6294                 cp = hw_resc->resv_irqs;
6295                 grp = hw_resc->resv_hw_ring_grps;
6296                 vnic = hw_resc->resv_vnics;
6297                 stat = hw_resc->resv_stat_ctxs;
6298         }
6299
6300         rx_rings = rx;
6301         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6302                 if (rx >= 2) {
6303                         rx_rings = rx >> 1;
6304                 } else {
6305                         if (netif_running(bp->dev))
6306                                 return -ENOMEM;
6307
6308                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6309                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6310                         bp->dev->hw_features &= ~NETIF_F_LRO;
6311                         bp->dev->features &= ~NETIF_F_LRO;
6312                         bnxt_set_ring_params(bp);
6313                 }
6314         }
6315         rx_rings = min_t(int, rx_rings, grp);
6316         cp = min_t(int, cp, bp->cp_nr_rings);
6317         if (stat > bnxt_get_ulp_stat_ctxs(bp))
6318                 stat -= bnxt_get_ulp_stat_ctxs(bp);
6319         cp = min_t(int, cp, stat);
6320         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6321         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6322                 rx = rx_rings << 1;
6323         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6324         bp->tx_nr_rings = tx;
6325
6326         /* If we cannot reserve all the RX rings, reset the RSS map only
6327          * if absolutely necessary
6328          */
6329         if (rx_rings != bp->rx_nr_rings) {
6330                 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6331                             rx_rings, bp->rx_nr_rings);
6332                 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) &&
6333                     (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6334                      bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6335                      bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6336                         netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6337                         bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6338                 }
6339         }
6340         bp->rx_nr_rings = rx_rings;
6341         bp->cp_nr_rings = cp;
6342
6343         if (!tx || !rx || !cp || !grp || !vnic || !stat)
6344                 return -ENOMEM;
6345
6346         if (!netif_is_rxfh_configured(bp->dev))
6347                 bnxt_set_dflt_rss_indir_tbl(bp);
6348
6349         return rc;
6350 }
6351
6352 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6353                                     int ring_grps, int cp_rings, int stats,
6354                                     int vnics)
6355 {
6356         struct hwrm_func_vf_cfg_input req = {0};
6357         u32 flags;
6358
6359         if (!BNXT_NEW_RM(bp))
6360                 return 0;
6361
6362         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6363                                      cp_rings, stats, vnics);
6364         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6365                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6366                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6367                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6368                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6369                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6370         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6371                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6372
6373         req.flags = cpu_to_le32(flags);
6374         return hwrm_send_message_silent(bp, &req, sizeof(req),
6375                                         HWRM_CMD_TIMEOUT);
6376 }
6377
6378 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6379                                     int ring_grps, int cp_rings, int stats,
6380                                     int vnics)
6381 {
6382         struct hwrm_func_cfg_input req = {0};
6383         u32 flags;
6384
6385         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6386                                      cp_rings, stats, vnics);
6387         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6388         if (BNXT_NEW_RM(bp)) {
6389                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6390                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6391                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6392                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6393                 if (bp->flags & BNXT_FLAG_CHIP_P5)
6394                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6395                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6396                 else
6397                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6398         }
6399
6400         req.flags = cpu_to_le32(flags);
6401         return hwrm_send_message_silent(bp, &req, sizeof(req),
6402                                         HWRM_CMD_TIMEOUT);
6403 }
6404
6405 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6406                                  int ring_grps, int cp_rings, int stats,
6407                                  int vnics)
6408 {
6409         if (bp->hwrm_spec_code < 0x10801)
6410                 return 0;
6411
6412         if (BNXT_PF(bp))
6413                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6414                                                 ring_grps, cp_rings, stats,
6415                                                 vnics);
6416
6417         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6418                                         cp_rings, stats, vnics);
6419 }
6420
6421 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6422 {
6423         struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6424         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6425         struct hwrm_ring_aggint_qcaps_input req = {0};
6426         int rc;
6427
6428         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6429         coal_cap->num_cmpl_dma_aggr_max = 63;
6430         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6431         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6432         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6433         coal_cap->int_lat_tmr_min_max = 65535;
6434         coal_cap->int_lat_tmr_max_max = 65535;
6435         coal_cap->num_cmpl_aggr_int_max = 65535;
6436         coal_cap->timer_units = 80;
6437
6438         if (bp->hwrm_spec_code < 0x10902)
6439                 return;
6440
6441         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6442         mutex_lock(&bp->hwrm_cmd_lock);
6443         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6444         if (!rc) {
6445                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6446                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6447                 coal_cap->num_cmpl_dma_aggr_max =
6448                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6449                 coal_cap->num_cmpl_dma_aggr_during_int_max =
6450                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6451                 coal_cap->cmpl_aggr_dma_tmr_max =
6452                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6453                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6454                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6455                 coal_cap->int_lat_tmr_min_max =
6456                         le16_to_cpu(resp->int_lat_tmr_min_max);
6457                 coal_cap->int_lat_tmr_max_max =
6458                         le16_to_cpu(resp->int_lat_tmr_max_max);
6459                 coal_cap->num_cmpl_aggr_int_max =
6460                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
6461                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6462         }
6463         mutex_unlock(&bp->hwrm_cmd_lock);
6464 }
6465
6466 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6467 {
6468         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6469
6470         return usec * 1000 / coal_cap->timer_units;
6471 }
6472
6473 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6474         struct bnxt_coal *hw_coal,
6475         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6476 {
6477         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6478         u32 cmpl_params = coal_cap->cmpl_params;
6479         u16 val, tmr, max, flags = 0;
6480
6481         max = hw_coal->bufs_per_record * 128;
6482         if (hw_coal->budget)
6483                 max = hw_coal->bufs_per_record * hw_coal->budget;
6484         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6485
6486         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6487         req->num_cmpl_aggr_int = cpu_to_le16(val);
6488
6489         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6490         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6491
6492         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6493                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6494         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6495
6496         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6497         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6498         req->int_lat_tmr_max = cpu_to_le16(tmr);
6499
6500         /* min timer set to 1/2 of interrupt timer */
6501         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6502                 val = tmr / 2;
6503                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6504                 req->int_lat_tmr_min = cpu_to_le16(val);
6505                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6506         }
6507
6508         /* buf timer set to 1/4 of interrupt timer */
6509         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6510         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6511
6512         if (cmpl_params &
6513             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6514                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6515                 val = clamp_t(u16, tmr, 1,
6516                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6517                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6518                 req->enables |=
6519                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6520         }
6521
6522         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6523                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6524         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6525             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6526                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6527         req->flags = cpu_to_le16(flags);
6528         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6529 }
6530
6531 /* Caller holds bp->hwrm_cmd_lock */
6532 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6533                                    struct bnxt_coal *hw_coal)
6534 {
6535         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6536         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6537         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6538         u32 nq_params = coal_cap->nq_params;
6539         u16 tmr;
6540
6541         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6542                 return 0;
6543
6544         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6545                                -1, -1);
6546         req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6547         req.flags =
6548                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6549
6550         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6551         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6552         req.int_lat_tmr_min = cpu_to_le16(tmr);
6553         req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6554         return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6555 }
6556
6557 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6558 {
6559         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6560         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6561         struct bnxt_coal coal;
6562
6563         /* Tick values in micro seconds.
6564          * 1 coal_buf x bufs_per_record = 1 completion record.
6565          */
6566         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6567
6568         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6569         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6570
6571         if (!bnapi->rx_ring)
6572                 return -ENODEV;
6573
6574         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6575                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6576
6577         bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6578
6579         req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6580
6581         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6582                                  HWRM_CMD_TIMEOUT);
6583 }
6584
6585 int bnxt_hwrm_set_coal(struct bnxt *bp)
6586 {
6587         int i, rc = 0;
6588         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6589                                                            req_tx = {0}, *req;
6590
6591         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6592                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6593         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6594                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6595
6596         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6597         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6598
6599         mutex_lock(&bp->hwrm_cmd_lock);
6600         for (i = 0; i < bp->cp_nr_rings; i++) {
6601                 struct bnxt_napi *bnapi = bp->bnapi[i];
6602                 struct bnxt_coal *hw_coal;
6603                 u16 ring_id;
6604
6605                 req = &req_rx;
6606                 if (!bnapi->rx_ring) {
6607                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6608                         req = &req_tx;
6609                 } else {
6610                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6611                 }
6612                 req->ring_id = cpu_to_le16(ring_id);
6613
6614                 rc = _hwrm_send_message(bp, req, sizeof(*req),
6615                                         HWRM_CMD_TIMEOUT);
6616                 if (rc)
6617                         break;
6618
6619                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6620                         continue;
6621
6622                 if (bnapi->rx_ring && bnapi->tx_ring) {
6623                         req = &req_tx;
6624                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6625                         req->ring_id = cpu_to_le16(ring_id);
6626                         rc = _hwrm_send_message(bp, req, sizeof(*req),
6627                                                 HWRM_CMD_TIMEOUT);
6628                         if (rc)
6629                                 break;
6630                 }
6631                 if (bnapi->rx_ring)
6632                         hw_coal = &bp->rx_coal;
6633                 else
6634                         hw_coal = &bp->tx_coal;
6635                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6636         }
6637         mutex_unlock(&bp->hwrm_cmd_lock);
6638         return rc;
6639 }
6640
6641 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6642 {
6643         struct hwrm_stat_ctx_clr_stats_input req0 = {0};
6644         struct hwrm_stat_ctx_free_input req = {0};
6645         int i;
6646
6647         if (!bp->bnapi)
6648                 return;
6649
6650         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6651                 return;
6652
6653         bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1);
6654         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6655
6656         mutex_lock(&bp->hwrm_cmd_lock);
6657         for (i = 0; i < bp->cp_nr_rings; i++) {
6658                 struct bnxt_napi *bnapi = bp->bnapi[i];
6659                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6660
6661                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6662                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6663                         if (BNXT_FW_MAJ(bp) <= 20) {
6664                                 req0.stat_ctx_id = req.stat_ctx_id;
6665                                 _hwrm_send_message(bp, &req0, sizeof(req0),
6666                                                    HWRM_CMD_TIMEOUT);
6667                         }
6668                         _hwrm_send_message(bp, &req, sizeof(req),
6669                                            HWRM_CMD_TIMEOUT);
6670
6671                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6672                 }
6673         }
6674         mutex_unlock(&bp->hwrm_cmd_lock);
6675 }
6676
6677 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6678 {
6679         int rc = 0, i;
6680         struct hwrm_stat_ctx_alloc_input req = {0};
6681         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6682
6683         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6684                 return 0;
6685
6686         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6687
6688         req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6689         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6690
6691         mutex_lock(&bp->hwrm_cmd_lock);
6692         for (i = 0; i < bp->cp_nr_rings; i++) {
6693                 struct bnxt_napi *bnapi = bp->bnapi[i];
6694                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6695
6696                 req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6697
6698                 rc = _hwrm_send_message(bp, &req, sizeof(req),
6699                                         HWRM_CMD_TIMEOUT);
6700                 if (rc)
6701                         break;
6702
6703                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6704
6705                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6706         }
6707         mutex_unlock(&bp->hwrm_cmd_lock);
6708         return rc;
6709 }
6710
6711 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6712 {
6713         struct hwrm_func_qcfg_input req = {0};
6714         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6715         u32 min_db_offset = 0;
6716         u16 flags;
6717         int rc;
6718
6719         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6720         req.fid = cpu_to_le16(0xffff);
6721         mutex_lock(&bp->hwrm_cmd_lock);
6722         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6723         if (rc)
6724                 goto func_qcfg_exit;
6725
6726 #ifdef CONFIG_BNXT_SRIOV
6727         if (BNXT_VF(bp)) {
6728                 struct bnxt_vf_info *vf = &bp->vf;
6729
6730                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6731         } else {
6732                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6733         }
6734 #endif
6735         flags = le16_to_cpu(resp->flags);
6736         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6737                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6738                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6739                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6740                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6741         }
6742         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6743                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6744         if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6745                 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6746
6747         switch (resp->port_partition_type) {
6748         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6749         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6750         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6751                 bp->port_partition_type = resp->port_partition_type;
6752                 break;
6753         }
6754         if (bp->hwrm_spec_code < 0x10707 ||
6755             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6756                 bp->br_mode = BRIDGE_MODE_VEB;
6757         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6758                 bp->br_mode = BRIDGE_MODE_VEPA;
6759         else
6760                 bp->br_mode = BRIDGE_MODE_UNDEF;
6761
6762         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6763         if (!bp->max_mtu)
6764                 bp->max_mtu = BNXT_MAX_MTU;
6765
6766         if (bp->db_size)
6767                 goto func_qcfg_exit;
6768
6769         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6770                 if (BNXT_PF(bp))
6771                         min_db_offset = DB_PF_OFFSET_P5;
6772                 else
6773                         min_db_offset = DB_VF_OFFSET_P5;
6774         }
6775         bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
6776                                  1024);
6777         if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
6778             bp->db_size <= min_db_offset)
6779                 bp->db_size = pci_resource_len(bp->pdev, 2);
6780
6781 func_qcfg_exit:
6782         mutex_unlock(&bp->hwrm_cmd_lock);
6783         return rc;
6784 }
6785
6786 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx,
6787                         struct hwrm_func_backing_store_qcaps_output *resp)
6788 {
6789         struct bnxt_mem_init *mem_init;
6790         u16 init_mask;
6791         u8 init_val;
6792         u8 *offset;
6793         int i;
6794
6795         init_val = resp->ctx_kind_initializer;
6796         init_mask = le16_to_cpu(resp->ctx_init_mask);
6797         offset = &resp->qp_init_offset;
6798         mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
6799         for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) {
6800                 mem_init->init_val = init_val;
6801                 mem_init->offset = BNXT_MEM_INVALID_OFFSET;
6802                 if (!init_mask)
6803                         continue;
6804                 if (i == BNXT_CTX_MEM_INIT_STAT)
6805                         offset = &resp->stat_init_offset;
6806                 if (init_mask & (1 << i))
6807                         mem_init->offset = *offset * 4;
6808                 else
6809                         mem_init->init_val = 0;
6810         }
6811         ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size;
6812         ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size;
6813         ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size;
6814         ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size;
6815         ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size;
6816         ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size;
6817 }
6818
6819 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6820 {
6821         struct hwrm_func_backing_store_qcaps_input req = {0};
6822         struct hwrm_func_backing_store_qcaps_output *resp =
6823                 bp->hwrm_cmd_resp_addr;
6824         int rc;
6825
6826         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6827                 return 0;
6828
6829         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6830         mutex_lock(&bp->hwrm_cmd_lock);
6831         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6832         if (!rc) {
6833                 struct bnxt_ctx_pg_info *ctx_pg;
6834                 struct bnxt_ctx_mem_info *ctx;
6835                 int i, tqm_rings;
6836
6837                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6838                 if (!ctx) {
6839                         rc = -ENOMEM;
6840                         goto ctx_err;
6841                 }
6842                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6843                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6844                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6845                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6846                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6847                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6848                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6849                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6850                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6851                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6852                 ctx->vnic_max_vnic_entries =
6853                         le16_to_cpu(resp->vnic_max_vnic_entries);
6854                 ctx->vnic_max_ring_table_entries =
6855                         le16_to_cpu(resp->vnic_max_ring_table_entries);
6856                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6857                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6858                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6859                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6860                 ctx->tqm_min_entries_per_ring =
6861                         le32_to_cpu(resp->tqm_min_entries_per_ring);
6862                 ctx->tqm_max_entries_per_ring =
6863                         le32_to_cpu(resp->tqm_max_entries_per_ring);
6864                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6865                 if (!ctx->tqm_entries_multiple)
6866                         ctx->tqm_entries_multiple = 1;
6867                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6868                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6869                 ctx->mrav_num_entries_units =
6870                         le16_to_cpu(resp->mrav_num_entries_units);
6871                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6872                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6873
6874                 bnxt_init_ctx_initializer(ctx, resp);
6875
6876                 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
6877                 if (!ctx->tqm_fp_rings_count)
6878                         ctx->tqm_fp_rings_count = bp->max_q;
6879                 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
6880                         ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
6881
6882                 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS;
6883                 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
6884                 if (!ctx_pg) {
6885                         kfree(ctx);
6886                         rc = -ENOMEM;
6887                         goto ctx_err;
6888                 }
6889                 for (i = 0; i < tqm_rings; i++, ctx_pg++)
6890                         ctx->tqm_mem[i] = ctx_pg;
6891                 bp->ctx = ctx;
6892         } else {
6893                 rc = 0;
6894         }
6895 ctx_err:
6896         mutex_unlock(&bp->hwrm_cmd_lock);
6897         return rc;
6898 }
6899
6900 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6901                                   __le64 *pg_dir)
6902 {
6903         u8 pg_size = 0;
6904
6905         if (!rmem->nr_pages)
6906                 return;
6907
6908         if (BNXT_PAGE_SHIFT == 13)
6909                 pg_size = 1 << 4;
6910         else if (BNXT_PAGE_SIZE == 16)
6911                 pg_size = 2 << 4;
6912
6913         *pg_attr = pg_size;
6914         if (rmem->depth >= 1) {
6915                 if (rmem->depth == 2)
6916                         *pg_attr |= 2;
6917                 else
6918                         *pg_attr |= 1;
6919                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6920         } else {
6921                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6922         }
6923 }
6924
6925 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
6926         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
6927          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
6928          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
6929          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
6930          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6931
6932 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6933 {
6934         struct hwrm_func_backing_store_cfg_input req = {0};
6935         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6936         struct bnxt_ctx_pg_info *ctx_pg;
6937         u32 req_len = sizeof(req);
6938         __le32 *num_entries;
6939         __le64 *pg_dir;
6940         u32 flags = 0;
6941         u8 *pg_attr;
6942         u32 ena;
6943         int i;
6944
6945         if (!ctx)
6946                 return 0;
6947
6948         if (req_len > bp->hwrm_max_ext_req_len)
6949                 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
6950         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6951         req.enables = cpu_to_le32(enables);
6952
6953         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6954                 ctx_pg = &ctx->qp_mem;
6955                 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6956                 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6957                 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6958                 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6959                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6960                                       &req.qpc_pg_size_qpc_lvl,
6961                                       &req.qpc_page_dir);
6962         }
6963         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6964                 ctx_pg = &ctx->srq_mem;
6965                 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6966                 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6967                 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6968                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6969                                       &req.srq_pg_size_srq_lvl,
6970                                       &req.srq_page_dir);
6971         }
6972         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6973                 ctx_pg = &ctx->cq_mem;
6974                 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6975                 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6976                 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6977                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6978                                       &req.cq_page_dir);
6979         }
6980         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6981                 ctx_pg = &ctx->vnic_mem;
6982                 req.vnic_num_vnic_entries =
6983                         cpu_to_le16(ctx->vnic_max_vnic_entries);
6984                 req.vnic_num_ring_table_entries =
6985                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
6986                 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6987                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6988                                       &req.vnic_pg_size_vnic_lvl,
6989                                       &req.vnic_page_dir);
6990         }
6991         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6992                 ctx_pg = &ctx->stat_mem;
6993                 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6994                 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6995                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6996                                       &req.stat_pg_size_stat_lvl,
6997                                       &req.stat_page_dir);
6998         }
6999         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
7000                 ctx_pg = &ctx->mrav_mem;
7001                 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
7002                 if (ctx->mrav_num_entries_units)
7003                         flags |=
7004                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
7005                 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
7006                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7007                                       &req.mrav_pg_size_mrav_lvl,
7008                                       &req.mrav_page_dir);
7009         }
7010         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
7011                 ctx_pg = &ctx->tim_mem;
7012                 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
7013                 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
7014                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7015                                       &req.tim_pg_size_tim_lvl,
7016                                       &req.tim_page_dir);
7017         }
7018         for (i = 0, num_entries = &req.tqm_sp_num_entries,
7019              pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
7020              pg_dir = &req.tqm_sp_page_dir,
7021              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
7022              i < BNXT_MAX_TQM_RINGS;
7023              i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
7024                 if (!(enables & ena))
7025                         continue;
7026
7027                 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
7028                 ctx_pg = ctx->tqm_mem[i];
7029                 *num_entries = cpu_to_le32(ctx_pg->entries);
7030                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
7031         }
7032         req.flags = cpu_to_le32(flags);
7033         return hwrm_send_message(bp, &req, req_len, HWRM_CMD_TIMEOUT);
7034 }
7035
7036 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
7037                                   struct bnxt_ctx_pg_info *ctx_pg)
7038 {
7039         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7040
7041         rmem->page_size = BNXT_PAGE_SIZE;
7042         rmem->pg_arr = ctx_pg->ctx_pg_arr;
7043         rmem->dma_arr = ctx_pg->ctx_dma_arr;
7044         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
7045         if (rmem->depth >= 1)
7046                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
7047         return bnxt_alloc_ring(bp, rmem);
7048 }
7049
7050 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
7051                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
7052                                   u8 depth, struct bnxt_mem_init *mem_init)
7053 {
7054         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7055         int rc;
7056
7057         if (!mem_size)
7058                 return -EINVAL;
7059
7060         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7061         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
7062                 ctx_pg->nr_pages = 0;
7063                 return -EINVAL;
7064         }
7065         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
7066                 int nr_tbls, i;
7067
7068                 rmem->depth = 2;
7069                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
7070                                              GFP_KERNEL);
7071                 if (!ctx_pg->ctx_pg_tbl)
7072                         return -ENOMEM;
7073                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
7074                 rmem->nr_pages = nr_tbls;
7075                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7076                 if (rc)
7077                         return rc;
7078                 for (i = 0; i < nr_tbls; i++) {
7079                         struct bnxt_ctx_pg_info *pg_tbl;
7080
7081                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
7082                         if (!pg_tbl)
7083                                 return -ENOMEM;
7084                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
7085                         rmem = &pg_tbl->ring_mem;
7086                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
7087                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
7088                         rmem->depth = 1;
7089                         rmem->nr_pages = MAX_CTX_PAGES;
7090                         rmem->mem_init = mem_init;
7091                         if (i == (nr_tbls - 1)) {
7092                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
7093
7094                                 if (rem)
7095                                         rmem->nr_pages = rem;
7096                         }
7097                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7098                         if (rc)
7099                                 break;
7100                 }
7101         } else {
7102                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7103                 if (rmem->nr_pages > 1 || depth)
7104                         rmem->depth = 1;
7105                 rmem->mem_init = mem_init;
7106                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7107         }
7108         return rc;
7109 }
7110
7111 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7112                                   struct bnxt_ctx_pg_info *ctx_pg)
7113 {
7114         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7115
7116         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7117             ctx_pg->ctx_pg_tbl) {
7118                 int i, nr_tbls = rmem->nr_pages;
7119
7120                 for (i = 0; i < nr_tbls; i++) {
7121                         struct bnxt_ctx_pg_info *pg_tbl;
7122                         struct bnxt_ring_mem_info *rmem2;
7123
7124                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
7125                         if (!pg_tbl)
7126                                 continue;
7127                         rmem2 = &pg_tbl->ring_mem;
7128                         bnxt_free_ring(bp, rmem2);
7129                         ctx_pg->ctx_pg_arr[i] = NULL;
7130                         kfree(pg_tbl);
7131                         ctx_pg->ctx_pg_tbl[i] = NULL;
7132                 }
7133                 kfree(ctx_pg->ctx_pg_tbl);
7134                 ctx_pg->ctx_pg_tbl = NULL;
7135         }
7136         bnxt_free_ring(bp, rmem);
7137         ctx_pg->nr_pages = 0;
7138 }
7139
7140 static void bnxt_free_ctx_mem(struct bnxt *bp)
7141 {
7142         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7143         int i;
7144
7145         if (!ctx)
7146                 return;
7147
7148         if (ctx->tqm_mem[0]) {
7149                 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7150                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7151                 kfree(ctx->tqm_mem[0]);
7152                 ctx->tqm_mem[0] = NULL;
7153         }
7154
7155         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7156         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7157         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7158         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7159         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7160         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7161         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7162         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7163 }
7164
7165 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7166 {
7167         struct bnxt_ctx_pg_info *ctx_pg;
7168         struct bnxt_ctx_mem_info *ctx;
7169         struct bnxt_mem_init *init;
7170         u32 mem_size, ena, entries;
7171         u32 entries_sp, min;
7172         u32 num_mr, num_ah;
7173         u32 extra_srqs = 0;
7174         u32 extra_qps = 0;
7175         u8 pg_lvl = 1;
7176         int i, rc;
7177
7178         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7179         if (rc) {
7180                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7181                            rc);
7182                 return rc;
7183         }
7184         ctx = bp->ctx;
7185         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7186                 return 0;
7187
7188         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7189                 pg_lvl = 2;
7190                 extra_qps = 65536;
7191                 extra_srqs = 8192;
7192         }
7193
7194         ctx_pg = &ctx->qp_mem;
7195         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7196                           extra_qps;
7197         if (ctx->qp_entry_size) {
7198                 mem_size = ctx->qp_entry_size * ctx_pg->entries;
7199                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7200                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7201                 if (rc)
7202                         return rc;
7203         }
7204
7205         ctx_pg = &ctx->srq_mem;
7206         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7207         if (ctx->srq_entry_size) {
7208                 mem_size = ctx->srq_entry_size * ctx_pg->entries;
7209                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ];
7210                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7211                 if (rc)
7212                         return rc;
7213         }
7214
7215         ctx_pg = &ctx->cq_mem;
7216         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7217         if (ctx->cq_entry_size) {
7218                 mem_size = ctx->cq_entry_size * ctx_pg->entries;
7219                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ];
7220                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7221                 if (rc)
7222                         return rc;
7223         }
7224
7225         ctx_pg = &ctx->vnic_mem;
7226         ctx_pg->entries = ctx->vnic_max_vnic_entries +
7227                           ctx->vnic_max_ring_table_entries;
7228         if (ctx->vnic_entry_size) {
7229                 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7230                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC];
7231                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7232                 if (rc)
7233                         return rc;
7234         }
7235
7236         ctx_pg = &ctx->stat_mem;
7237         ctx_pg->entries = ctx->stat_max_entries;
7238         if (ctx->stat_entry_size) {
7239                 mem_size = ctx->stat_entry_size * ctx_pg->entries;
7240                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT];
7241                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7242                 if (rc)
7243                         return rc;
7244         }
7245
7246         ena = 0;
7247         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7248                 goto skip_rdma;
7249
7250         ctx_pg = &ctx->mrav_mem;
7251         /* 128K extra is needed to accommodate static AH context
7252          * allocation by f/w.
7253          */
7254         num_mr = 1024 * 256;
7255         num_ah = 1024 * 128;
7256         ctx_pg->entries = num_mr + num_ah;
7257         if (ctx->mrav_entry_size) {
7258                 mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7259                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV];
7260                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init);
7261                 if (rc)
7262                         return rc;
7263         }
7264         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7265         if (ctx->mrav_num_entries_units)
7266                 ctx_pg->entries =
7267                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
7268                          (num_ah / ctx->mrav_num_entries_units);
7269
7270         ctx_pg = &ctx->tim_mem;
7271         ctx_pg->entries = ctx->qp_mem.entries;
7272         if (ctx->tim_entry_size) {
7273                 mem_size = ctx->tim_entry_size * ctx_pg->entries;
7274                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL);
7275                 if (rc)
7276                         return rc;
7277         }
7278         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7279
7280 skip_rdma:
7281         min = ctx->tqm_min_entries_per_ring;
7282         entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7283                      2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7284         entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7285         entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries;
7286         entries = roundup(entries, ctx->tqm_entries_multiple);
7287         entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7288         for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7289                 ctx_pg = ctx->tqm_mem[i];
7290                 ctx_pg->entries = i ? entries : entries_sp;
7291                 if (ctx->tqm_entry_size) {
7292                         mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7293                         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1,
7294                                                     NULL);
7295                         if (rc)
7296                                 return rc;
7297                 }
7298                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7299         }
7300         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7301         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7302         if (rc) {
7303                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7304                            rc);
7305                 return rc;
7306         }
7307         ctx->flags |= BNXT_CTX_FLAG_INITED;
7308         return 0;
7309 }
7310
7311 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7312 {
7313         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7314         struct hwrm_func_resource_qcaps_input req = {0};
7315         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7316         int rc;
7317
7318         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
7319         req.fid = cpu_to_le16(0xffff);
7320
7321         mutex_lock(&bp->hwrm_cmd_lock);
7322         rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
7323                                        HWRM_CMD_TIMEOUT);
7324         if (rc)
7325                 goto hwrm_func_resc_qcaps_exit;
7326
7327         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7328         if (!all)
7329                 goto hwrm_func_resc_qcaps_exit;
7330
7331         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7332         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7333         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7334         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7335         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7336         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7337         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7338         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7339         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7340         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7341         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7342         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7343         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7344         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7345         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7346         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7347
7348         if (bp->flags & BNXT_FLAG_CHIP_P5) {
7349                 u16 max_msix = le16_to_cpu(resp->max_msix);
7350
7351                 hw_resc->max_nqs = max_msix;
7352                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7353         }
7354
7355         if (BNXT_PF(bp)) {
7356                 struct bnxt_pf_info *pf = &bp->pf;
7357
7358                 pf->vf_resv_strategy =
7359                         le16_to_cpu(resp->vf_reservation_strategy);
7360                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7361                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7362         }
7363 hwrm_func_resc_qcaps_exit:
7364         mutex_unlock(&bp->hwrm_cmd_lock);
7365         return rc;
7366 }
7367
7368 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7369 {
7370         int rc = 0;
7371         struct hwrm_func_qcaps_input req = {0};
7372         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7373         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7374         u32 flags, flags_ext;
7375
7376         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
7377         req.fid = cpu_to_le16(0xffff);
7378
7379         mutex_lock(&bp->hwrm_cmd_lock);
7380         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7381         if (rc)
7382                 goto hwrm_func_qcaps_exit;
7383
7384         flags = le32_to_cpu(resp->flags);
7385         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7386                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7387         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7388                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7389         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7390                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7391         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7392                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7393         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7394                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7395         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7396                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7397         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7398                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7399         if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7400                 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7401
7402         flags_ext = le32_to_cpu(resp->flags_ext);
7403         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7404                 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7405
7406         bp->tx_push_thresh = 0;
7407         if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7408             BNXT_FW_MAJ(bp) > 217)
7409                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7410
7411         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7412         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7413         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7414         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7415         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7416         if (!hw_resc->max_hw_ring_grps)
7417                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7418         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7419         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7420         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7421
7422         if (BNXT_PF(bp)) {
7423                 struct bnxt_pf_info *pf = &bp->pf;
7424
7425                 pf->fw_fid = le16_to_cpu(resp->fid);
7426                 pf->port_id = le16_to_cpu(resp->port_id);
7427                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7428                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7429                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7430                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7431                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7432                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7433                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7434                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7435                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7436                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7437                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7438                         bp->flags |= BNXT_FLAG_WOL_CAP;
7439         } else {
7440 #ifdef CONFIG_BNXT_SRIOV
7441                 struct bnxt_vf_info *vf = &bp->vf;
7442
7443                 vf->fw_fid = le16_to_cpu(resp->fid);
7444                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7445 #endif
7446         }
7447
7448 hwrm_func_qcaps_exit:
7449         mutex_unlock(&bp->hwrm_cmd_lock);
7450         return rc;
7451 }
7452
7453 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7454
7455 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7456 {
7457         int rc;
7458
7459         rc = __bnxt_hwrm_func_qcaps(bp);
7460         if (rc)
7461                 return rc;
7462         rc = bnxt_hwrm_queue_qportcfg(bp);
7463         if (rc) {
7464                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7465                 return rc;
7466         }
7467         if (bp->hwrm_spec_code >= 0x10803) {
7468                 rc = bnxt_alloc_ctx_mem(bp);
7469                 if (rc)
7470                         return rc;
7471                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7472                 if (!rc)
7473                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7474         }
7475         return 0;
7476 }
7477
7478 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7479 {
7480         struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7481         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7482         int rc = 0;
7483         u32 flags;
7484
7485         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7486                 return 0;
7487
7488         resp = bp->hwrm_cmd_resp_addr;
7489         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7490
7491         mutex_lock(&bp->hwrm_cmd_lock);
7492         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7493         if (rc)
7494                 goto hwrm_cfa_adv_qcaps_exit;
7495
7496         flags = le32_to_cpu(resp->flags);
7497         if (flags &
7498             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7499                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7500
7501 hwrm_cfa_adv_qcaps_exit:
7502         mutex_unlock(&bp->hwrm_cmd_lock);
7503         return rc;
7504 }
7505
7506 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7507 {
7508         if (bp->fw_health)
7509                 return 0;
7510
7511         bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7512         if (!bp->fw_health)
7513                 return -ENOMEM;
7514
7515         return 0;
7516 }
7517
7518 static int bnxt_alloc_fw_health(struct bnxt *bp)
7519 {
7520         int rc;
7521
7522         if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7523             !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7524                 return 0;
7525
7526         rc = __bnxt_alloc_fw_health(bp);
7527         if (rc) {
7528                 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7529                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7530                 return rc;
7531         }
7532
7533         return 0;
7534 }
7535
7536 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7537 {
7538         writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7539                                          BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7540                                          BNXT_FW_HEALTH_WIN_MAP_OFF);
7541 }
7542
7543 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7544 {
7545         void __iomem *hs;
7546         u32 status_loc;
7547         u32 reg_type;
7548         u32 sig;
7549
7550         __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7551         hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7552
7553         sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7554         if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7555                 if (!bp->chip_num) {
7556                         __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
7557                         bp->chip_num = readl(bp->bar0 +
7558                                              BNXT_FW_HEALTH_WIN_BASE +
7559                                              BNXT_GRC_REG_CHIP_NUM);
7560                 }
7561                 if (!BNXT_CHIP_P5(bp)) {
7562                         if (bp->fw_health)
7563                                 bp->fw_health->status_reliable = false;
7564                         return;
7565                 }
7566                 status_loc = BNXT_GRC_REG_STATUS_P5 |
7567                              BNXT_FW_HEALTH_REG_TYPE_BAR0;
7568         } else {
7569                 status_loc = readl(hs + offsetof(struct hcomm_status,
7570                                                  fw_status_loc));
7571         }
7572
7573         if (__bnxt_alloc_fw_health(bp)) {
7574                 netdev_warn(bp->dev, "no memory for firmware status checks\n");
7575                 return;
7576         }
7577
7578         bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7579         reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7580         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7581                 __bnxt_map_fw_health_reg(bp, status_loc);
7582                 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7583                         BNXT_FW_HEALTH_WIN_OFF(status_loc);
7584         }
7585
7586         bp->fw_health->status_reliable = true;
7587 }
7588
7589 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7590 {
7591         struct bnxt_fw_health *fw_health = bp->fw_health;
7592         u32 reg_base = 0xffffffff;
7593         int i;
7594
7595         /* Only pre-map the monitoring GRC registers using window 3 */
7596         for (i = 0; i < 4; i++) {
7597                 u32 reg = fw_health->regs[i];
7598
7599                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7600                         continue;
7601                 if (reg_base == 0xffffffff)
7602                         reg_base = reg & BNXT_GRC_BASE_MASK;
7603                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7604                         return -ERANGE;
7605                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7606         }
7607         if (reg_base == 0xffffffff)
7608                 return 0;
7609
7610         __bnxt_map_fw_health_reg(bp, reg_base);
7611         return 0;
7612 }
7613
7614 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7615 {
7616         struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7617         struct bnxt_fw_health *fw_health = bp->fw_health;
7618         struct hwrm_error_recovery_qcfg_input req = {0};
7619         int rc, i;
7620
7621         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7622                 return 0;
7623
7624         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7625         mutex_lock(&bp->hwrm_cmd_lock);
7626         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7627         if (rc)
7628                 goto err_recovery_out;
7629         fw_health->flags = le32_to_cpu(resp->flags);
7630         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7631             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7632                 rc = -EINVAL;
7633                 goto err_recovery_out;
7634         }
7635         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7636         fw_health->master_func_wait_dsecs =
7637                 le32_to_cpu(resp->master_func_wait_period);
7638         fw_health->normal_func_wait_dsecs =
7639                 le32_to_cpu(resp->normal_func_wait_period);
7640         fw_health->post_reset_wait_dsecs =
7641                 le32_to_cpu(resp->master_func_wait_period_after_reset);
7642         fw_health->post_reset_max_wait_dsecs =
7643                 le32_to_cpu(resp->max_bailout_time_after_reset);
7644         fw_health->regs[BNXT_FW_HEALTH_REG] =
7645                 le32_to_cpu(resp->fw_health_status_reg);
7646         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7647                 le32_to_cpu(resp->fw_heartbeat_reg);
7648         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7649                 le32_to_cpu(resp->fw_reset_cnt_reg);
7650         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7651                 le32_to_cpu(resp->reset_inprogress_reg);
7652         fw_health->fw_reset_inprog_reg_mask =
7653                 le32_to_cpu(resp->reset_inprogress_reg_mask);
7654         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7655         if (fw_health->fw_reset_seq_cnt >= 16) {
7656                 rc = -EINVAL;
7657                 goto err_recovery_out;
7658         }
7659         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7660                 fw_health->fw_reset_seq_regs[i] =
7661                         le32_to_cpu(resp->reset_reg[i]);
7662                 fw_health->fw_reset_seq_vals[i] =
7663                         le32_to_cpu(resp->reset_reg_val[i]);
7664                 fw_health->fw_reset_seq_delay_msec[i] =
7665                         resp->delay_after_reset[i];
7666         }
7667 err_recovery_out:
7668         mutex_unlock(&bp->hwrm_cmd_lock);
7669         if (!rc)
7670                 rc = bnxt_map_fw_health_regs(bp);
7671         if (rc)
7672                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7673         return rc;
7674 }
7675
7676 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7677 {
7678         struct hwrm_func_reset_input req = {0};
7679
7680         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7681         req.enables = 0;
7682
7683         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7684 }
7685
7686 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
7687 {
7688         struct hwrm_nvm_get_dev_info_output nvm_info;
7689
7690         if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
7691                 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
7692                          nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
7693                          nvm_info.nvm_cfg_ver_upd);
7694 }
7695
7696 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7697 {
7698         int rc = 0;
7699         struct hwrm_queue_qportcfg_input req = {0};
7700         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7701         u8 i, j, *qptr;
7702         bool no_rdma;
7703
7704         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7705
7706         mutex_lock(&bp->hwrm_cmd_lock);
7707         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7708         if (rc)
7709                 goto qportcfg_exit;
7710
7711         if (!resp->max_configurable_queues) {
7712                 rc = -EINVAL;
7713                 goto qportcfg_exit;
7714         }
7715         bp->max_tc = resp->max_configurable_queues;
7716         bp->max_lltc = resp->max_configurable_lossless_queues;
7717         if (bp->max_tc > BNXT_MAX_QUEUE)
7718                 bp->max_tc = BNXT_MAX_QUEUE;
7719
7720         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7721         qptr = &resp->queue_id0;
7722         for (i = 0, j = 0; i < bp->max_tc; i++) {
7723                 bp->q_info[j].queue_id = *qptr;
7724                 bp->q_ids[i] = *qptr++;
7725                 bp->q_info[j].queue_profile = *qptr++;
7726                 bp->tc_to_qidx[j] = j;
7727                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7728                     (no_rdma && BNXT_PF(bp)))
7729                         j++;
7730         }
7731         bp->max_q = bp->max_tc;
7732         bp->max_tc = max_t(u8, j, 1);
7733
7734         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7735                 bp->max_tc = 1;
7736
7737         if (bp->max_lltc > bp->max_tc)
7738                 bp->max_lltc = bp->max_tc;
7739
7740 qportcfg_exit:
7741         mutex_unlock(&bp->hwrm_cmd_lock);
7742         return rc;
7743 }
7744
7745 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7746 {
7747         struct hwrm_ver_get_input req = {0};
7748         int rc;
7749
7750         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7751         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7752         req.hwrm_intf_min = HWRM_VERSION_MINOR;
7753         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7754
7755         rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7756                                    silent);
7757         return rc;
7758 }
7759
7760 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7761 {
7762         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7763         u16 fw_maj, fw_min, fw_bld, fw_rsv;
7764         u32 dev_caps_cfg, hwrm_ver;
7765         int rc, len;
7766
7767         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7768         mutex_lock(&bp->hwrm_cmd_lock);
7769         rc = __bnxt_hwrm_ver_get(bp, false);
7770         if (rc)
7771                 goto hwrm_ver_get_exit;
7772
7773         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7774
7775         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7776                              resp->hwrm_intf_min_8b << 8 |
7777                              resp->hwrm_intf_upd_8b;
7778         if (resp->hwrm_intf_maj_8b < 1) {
7779                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7780                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7781                             resp->hwrm_intf_upd_8b);
7782                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7783         }
7784
7785         hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
7786                         HWRM_VERSION_UPDATE;
7787
7788         if (bp->hwrm_spec_code > hwrm_ver)
7789                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7790                          HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
7791                          HWRM_VERSION_UPDATE);
7792         else
7793                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7794                          resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7795                          resp->hwrm_intf_upd_8b);
7796
7797         fw_maj = le16_to_cpu(resp->hwrm_fw_major);
7798         if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
7799                 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
7800                 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
7801                 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
7802                 len = FW_VER_STR_LEN;
7803         } else {
7804                 fw_maj = resp->hwrm_fw_maj_8b;
7805                 fw_min = resp->hwrm_fw_min_8b;
7806                 fw_bld = resp->hwrm_fw_bld_8b;
7807                 fw_rsv = resp->hwrm_fw_rsvd_8b;
7808                 len = BC_HWRM_STR_LEN;
7809         }
7810         bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
7811         snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
7812                  fw_rsv);
7813
7814         if (strlen(resp->active_pkg_name)) {
7815                 int fw_ver_len = strlen(bp->fw_ver_str);
7816
7817                 snprintf(bp->fw_ver_str + fw_ver_len,
7818                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7819                          resp->active_pkg_name);
7820                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7821         }
7822
7823         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7824         if (!bp->hwrm_cmd_timeout)
7825                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7826
7827         if (resp->hwrm_intf_maj_8b >= 1) {
7828                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7829                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7830         }
7831         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7832                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7833
7834         bp->chip_num = le16_to_cpu(resp->chip_num);
7835         bp->chip_rev = resp->chip_rev;
7836         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7837             !resp->chip_metal)
7838                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7839
7840         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7841         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7842             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7843                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7844
7845         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7846                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7847
7848         if (dev_caps_cfg &
7849             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7850                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7851
7852         if (dev_caps_cfg &
7853             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7854                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7855
7856         if (dev_caps_cfg &
7857             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7858                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7859
7860 hwrm_ver_get_exit:
7861         mutex_unlock(&bp->hwrm_cmd_lock);
7862         return rc;
7863 }
7864
7865 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7866 {
7867         struct hwrm_fw_set_time_input req = {0};
7868         struct tm tm;
7869         time64_t now = ktime_get_real_seconds();
7870
7871         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7872             bp->hwrm_spec_code < 0x10400)
7873                 return -EOPNOTSUPP;
7874
7875         time64_to_tm(now, 0, &tm);
7876         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7877         req.year = cpu_to_le16(1900 + tm.tm_year);
7878         req.month = 1 + tm.tm_mon;
7879         req.day = tm.tm_mday;
7880         req.hour = tm.tm_hour;
7881         req.minute = tm.tm_min;
7882         req.second = tm.tm_sec;
7883         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7884 }
7885
7886 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
7887 {
7888         u64 sw_tmp;
7889
7890         hw &= mask;
7891         sw_tmp = (*sw & ~mask) | hw;
7892         if (hw < (*sw & mask))
7893                 sw_tmp += mask + 1;
7894         WRITE_ONCE(*sw, sw_tmp);
7895 }
7896
7897 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
7898                                     int count, bool ignore_zero)
7899 {
7900         int i;
7901
7902         for (i = 0; i < count; i++) {
7903                 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
7904
7905                 if (ignore_zero && !hw)
7906                         continue;
7907
7908                 if (masks[i] == -1ULL)
7909                         sw_stats[i] = hw;
7910                 else
7911                         bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
7912         }
7913 }
7914
7915 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
7916 {
7917         if (!stats->hw_stats)
7918                 return;
7919
7920         __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7921                                 stats->hw_masks, stats->len / 8, false);
7922 }
7923
7924 static void bnxt_accumulate_all_stats(struct bnxt *bp)
7925 {
7926         struct bnxt_stats_mem *ring0_stats;
7927         bool ignore_zero = false;
7928         int i;
7929
7930         /* Chip bug.  Counter intermittently becomes 0. */
7931         if (bp->flags & BNXT_FLAG_CHIP_P5)
7932                 ignore_zero = true;
7933
7934         for (i = 0; i < bp->cp_nr_rings; i++) {
7935                 struct bnxt_napi *bnapi = bp->bnapi[i];
7936                 struct bnxt_cp_ring_info *cpr;
7937                 struct bnxt_stats_mem *stats;
7938
7939                 cpr = &bnapi->cp_ring;
7940                 stats = &cpr->stats;
7941                 if (!i)
7942                         ring0_stats = stats;
7943                 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7944                                         ring0_stats->hw_masks,
7945                                         ring0_stats->len / 8, ignore_zero);
7946         }
7947         if (bp->flags & BNXT_FLAG_PORT_STATS) {
7948                 struct bnxt_stats_mem *stats = &bp->port_stats;
7949                 __le64 *hw_stats = stats->hw_stats;
7950                 u64 *sw_stats = stats->sw_stats;
7951                 u64 *masks = stats->hw_masks;
7952                 int cnt;
7953
7954                 cnt = sizeof(struct rx_port_stats) / 8;
7955                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7956
7957                 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7958                 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7959                 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7960                 cnt = sizeof(struct tx_port_stats) / 8;
7961                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7962         }
7963         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
7964                 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
7965                 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
7966         }
7967 }
7968
7969 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
7970 {
7971         struct bnxt_pf_info *pf = &bp->pf;
7972         struct hwrm_port_qstats_input req = {0};
7973
7974         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7975                 return 0;
7976
7977         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7978                 return -EOPNOTSUPP;
7979
7980         req.flags = flags;
7981         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7982         req.port_id = cpu_to_le16(pf->port_id);
7983         req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
7984                                             BNXT_TX_PORT_STATS_BYTE_OFFSET);
7985         req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
7986         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7987 }
7988
7989 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
7990 {
7991         struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7992         struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7993         struct hwrm_port_qstats_ext_input req = {0};
7994         struct bnxt_pf_info *pf = &bp->pf;
7995         u32 tx_stat_size;
7996         int rc;
7997
7998         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7999                 return 0;
8000
8001         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8002                 return -EOPNOTSUPP;
8003
8004         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
8005         req.flags = flags;
8006         req.port_id = cpu_to_le16(pf->port_id);
8007         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
8008         req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
8009         tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
8010                        sizeof(struct tx_port_stats_ext) : 0;
8011         req.tx_stat_size = cpu_to_le16(tx_stat_size);
8012         req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
8013         mutex_lock(&bp->hwrm_cmd_lock);
8014         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8015         if (!rc) {
8016                 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
8017                 bp->fw_tx_stats_ext_size = tx_stat_size ?
8018                         le16_to_cpu(resp->tx_stat_size) / 8 : 0;
8019         } else {
8020                 bp->fw_rx_stats_ext_size = 0;
8021                 bp->fw_tx_stats_ext_size = 0;
8022         }
8023         if (flags)
8024                 goto qstats_done;
8025
8026         if (bp->fw_tx_stats_ext_size <=
8027             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
8028                 mutex_unlock(&bp->hwrm_cmd_lock);
8029                 bp->pri2cos_valid = 0;
8030                 return rc;
8031         }
8032
8033         bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
8034         req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
8035
8036         rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
8037         if (!rc) {
8038                 struct hwrm_queue_pri2cos_qcfg_output *resp2;
8039                 u8 *pri2cos;
8040                 int i, j;
8041
8042                 resp2 = bp->hwrm_cmd_resp_addr;
8043                 pri2cos = &resp2->pri0_cos_queue_id;
8044                 for (i = 0; i < 8; i++) {
8045                         u8 queue_id = pri2cos[i];
8046                         u8 queue_idx;
8047
8048                         /* Per port queue IDs start from 0, 10, 20, etc */
8049                         queue_idx = queue_id % 10;
8050                         if (queue_idx > BNXT_MAX_QUEUE) {
8051                                 bp->pri2cos_valid = false;
8052                                 goto qstats_done;
8053                         }
8054                         for (j = 0; j < bp->max_q; j++) {
8055                                 if (bp->q_ids[j] == queue_id)
8056                                         bp->pri2cos_idx[i] = queue_idx;
8057                         }
8058                 }
8059                 bp->pri2cos_valid = 1;
8060         }
8061 qstats_done:
8062         mutex_unlock(&bp->hwrm_cmd_lock);
8063         return rc;
8064 }
8065
8066 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
8067 {
8068         if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID)
8069                 bnxt_hwrm_tunnel_dst_port_free(
8070                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
8071         if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID)
8072                 bnxt_hwrm_tunnel_dst_port_free(
8073                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
8074 }
8075
8076 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
8077 {
8078         int rc, i;
8079         u32 tpa_flags = 0;
8080
8081         if (set_tpa)
8082                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
8083         else if (BNXT_NO_FW_ACCESS(bp))
8084                 return 0;
8085         for (i = 0; i < bp->nr_vnics; i++) {
8086                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
8087                 if (rc) {
8088                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
8089                                    i, rc);
8090                         return rc;
8091                 }
8092         }
8093         return 0;
8094 }
8095
8096 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
8097 {
8098         int i;
8099
8100         for (i = 0; i < bp->nr_vnics; i++)
8101                 bnxt_hwrm_vnic_set_rss(bp, i, false);
8102 }
8103
8104 static void bnxt_clear_vnic(struct bnxt *bp)
8105 {
8106         if (!bp->vnic_info)
8107                 return;
8108
8109         bnxt_hwrm_clear_vnic_filter(bp);
8110         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
8111                 /* clear all RSS setting before free vnic ctx */
8112                 bnxt_hwrm_clear_vnic_rss(bp);
8113                 bnxt_hwrm_vnic_ctx_free(bp);
8114         }
8115         /* before free the vnic, undo the vnic tpa settings */
8116         if (bp->flags & BNXT_FLAG_TPA)
8117                 bnxt_set_tpa(bp, false);
8118         bnxt_hwrm_vnic_free(bp);
8119         if (bp->flags & BNXT_FLAG_CHIP_P5)
8120                 bnxt_hwrm_vnic_ctx_free(bp);
8121 }
8122
8123 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
8124                                     bool irq_re_init)
8125 {
8126         bnxt_clear_vnic(bp);
8127         bnxt_hwrm_ring_free(bp, close_path);
8128         bnxt_hwrm_ring_grp_free(bp);
8129         if (irq_re_init) {
8130                 bnxt_hwrm_stat_ctx_free(bp);
8131                 bnxt_hwrm_free_tunnel_ports(bp);
8132         }
8133 }
8134
8135 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8136 {
8137         struct hwrm_func_cfg_input req = {0};
8138
8139         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8140         req.fid = cpu_to_le16(0xffff);
8141         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8142         if (br_mode == BRIDGE_MODE_VEB)
8143                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8144         else if (br_mode == BRIDGE_MODE_VEPA)
8145                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8146         else
8147                 return -EINVAL;
8148         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8149 }
8150
8151 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8152 {
8153         struct hwrm_func_cfg_input req = {0};
8154
8155         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8156                 return 0;
8157
8158         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8159         req.fid = cpu_to_le16(0xffff);
8160         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8161         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8162         if (size == 128)
8163                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8164
8165         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8166 }
8167
8168 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8169 {
8170         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8171         int rc;
8172
8173         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8174                 goto skip_rss_ctx;
8175
8176         /* allocate context for vnic */
8177         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8178         if (rc) {
8179                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8180                            vnic_id, rc);
8181                 goto vnic_setup_err;
8182         }
8183         bp->rsscos_nr_ctxs++;
8184
8185         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8186                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8187                 if (rc) {
8188                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8189                                    vnic_id, rc);
8190                         goto vnic_setup_err;
8191                 }
8192                 bp->rsscos_nr_ctxs++;
8193         }
8194
8195 skip_rss_ctx:
8196         /* configure default vnic, ring grp */
8197         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8198         if (rc) {
8199                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8200                            vnic_id, rc);
8201                 goto vnic_setup_err;
8202         }
8203
8204         /* Enable RSS hashing on vnic */
8205         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8206         if (rc) {
8207                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8208                            vnic_id, rc);
8209                 goto vnic_setup_err;
8210         }
8211
8212         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8213                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8214                 if (rc) {
8215                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8216                                    vnic_id, rc);
8217                 }
8218         }
8219
8220 vnic_setup_err:
8221         return rc;
8222 }
8223
8224 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8225 {
8226         int rc, i, nr_ctxs;
8227
8228         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8229         for (i = 0; i < nr_ctxs; i++) {
8230                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8231                 if (rc) {
8232                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8233                                    vnic_id, i, rc);
8234                         break;
8235                 }
8236                 bp->rsscos_nr_ctxs++;
8237         }
8238         if (i < nr_ctxs)
8239                 return -ENOMEM;
8240
8241         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8242         if (rc) {
8243                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8244                            vnic_id, rc);
8245                 return rc;
8246         }
8247         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8248         if (rc) {
8249                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8250                            vnic_id, rc);
8251                 return rc;
8252         }
8253         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8254                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8255                 if (rc) {
8256                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8257                                    vnic_id, rc);
8258                 }
8259         }
8260         return rc;
8261 }
8262
8263 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8264 {
8265         if (bp->flags & BNXT_FLAG_CHIP_P5)
8266                 return __bnxt_setup_vnic_p5(bp, vnic_id);
8267         else
8268                 return __bnxt_setup_vnic(bp, vnic_id);
8269 }
8270
8271 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8272 {
8273 #ifdef CONFIG_RFS_ACCEL
8274         int i, rc = 0;
8275
8276         if (bp->flags & BNXT_FLAG_CHIP_P5)
8277                 return 0;
8278
8279         for (i = 0; i < bp->rx_nr_rings; i++) {
8280                 struct bnxt_vnic_info *vnic;
8281                 u16 vnic_id = i + 1;
8282                 u16 ring_id = i;
8283
8284                 if (vnic_id >= bp->nr_vnics)
8285                         break;
8286
8287                 vnic = &bp->vnic_info[vnic_id];
8288                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
8289                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8290                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8291                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8292                 if (rc) {
8293                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8294                                    vnic_id, rc);
8295                         break;
8296                 }
8297                 rc = bnxt_setup_vnic(bp, vnic_id);
8298                 if (rc)
8299                         break;
8300         }
8301         return rc;
8302 #else
8303         return 0;
8304 #endif
8305 }
8306
8307 /* Allow PF and VF with default VLAN to be in promiscuous mode */
8308 static bool bnxt_promisc_ok(struct bnxt *bp)
8309 {
8310 #ifdef CONFIG_BNXT_SRIOV
8311         if (BNXT_VF(bp) && !bp->vf.vlan)
8312                 return false;
8313 #endif
8314         return true;
8315 }
8316
8317 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8318 {
8319         unsigned int rc = 0;
8320
8321         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8322         if (rc) {
8323                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8324                            rc);
8325                 return rc;
8326         }
8327
8328         rc = bnxt_hwrm_vnic_cfg(bp, 1);
8329         if (rc) {
8330                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8331                            rc);
8332                 return rc;
8333         }
8334         return rc;
8335 }
8336
8337 static int bnxt_cfg_rx_mode(struct bnxt *);
8338 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8339
8340 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8341 {
8342         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8343         int rc = 0;
8344         unsigned int rx_nr_rings = bp->rx_nr_rings;
8345
8346         if (irq_re_init) {
8347                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
8348                 if (rc) {
8349                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8350                                    rc);
8351                         goto err_out;
8352                 }
8353         }
8354
8355         rc = bnxt_hwrm_ring_alloc(bp);
8356         if (rc) {
8357                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8358                 goto err_out;
8359         }
8360
8361         rc = bnxt_hwrm_ring_grp_alloc(bp);
8362         if (rc) {
8363                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8364                 goto err_out;
8365         }
8366
8367         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8368                 rx_nr_rings--;
8369
8370         /* default vnic 0 */
8371         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8372         if (rc) {
8373                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8374                 goto err_out;
8375         }
8376
8377         rc = bnxt_setup_vnic(bp, 0);
8378         if (rc)
8379                 goto err_out;
8380
8381         if (bp->flags & BNXT_FLAG_RFS) {
8382                 rc = bnxt_alloc_rfs_vnics(bp);
8383                 if (rc)
8384                         goto err_out;
8385         }
8386
8387         if (bp->flags & BNXT_FLAG_TPA) {
8388                 rc = bnxt_set_tpa(bp, true);
8389                 if (rc)
8390                         goto err_out;
8391         }
8392
8393         if (BNXT_VF(bp))
8394                 bnxt_update_vf_mac(bp);
8395
8396         /* Filter for default vnic 0 */
8397         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8398         if (rc) {
8399                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8400                 goto err_out;
8401         }
8402         vnic->uc_filter_count = 1;
8403
8404         vnic->rx_mask = 0;
8405         if (bp->dev->flags & IFF_BROADCAST)
8406                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8407
8408         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
8409                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8410
8411         if (bp->dev->flags & IFF_ALLMULTI) {
8412                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8413                 vnic->mc_list_count = 0;
8414         } else {
8415                 u32 mask = 0;
8416
8417                 bnxt_mc_list_updated(bp, &mask);
8418                 vnic->rx_mask |= mask;
8419         }
8420
8421         rc = bnxt_cfg_rx_mode(bp);
8422         if (rc)
8423                 goto err_out;
8424
8425         rc = bnxt_hwrm_set_coal(bp);
8426         if (rc)
8427                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8428                                 rc);
8429
8430         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8431                 rc = bnxt_setup_nitroa0_vnic(bp);
8432                 if (rc)
8433                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8434                                    rc);
8435         }
8436
8437         if (BNXT_VF(bp)) {
8438                 bnxt_hwrm_func_qcfg(bp);
8439                 netdev_update_features(bp->dev);
8440         }
8441
8442         return 0;
8443
8444 err_out:
8445         bnxt_hwrm_resource_free(bp, 0, true);
8446
8447         return rc;
8448 }
8449
8450 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8451 {
8452         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8453         return 0;
8454 }
8455
8456 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8457 {
8458         bnxt_init_cp_rings(bp);
8459         bnxt_init_rx_rings(bp);
8460         bnxt_init_tx_rings(bp);
8461         bnxt_init_ring_grps(bp, irq_re_init);
8462         bnxt_init_vnics(bp);
8463
8464         return bnxt_init_chip(bp, irq_re_init);
8465 }
8466
8467 static int bnxt_set_real_num_queues(struct bnxt *bp)
8468 {
8469         int rc;
8470         struct net_device *dev = bp->dev;
8471
8472         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8473                                           bp->tx_nr_rings_xdp);
8474         if (rc)
8475                 return rc;
8476
8477         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8478         if (rc)
8479                 return rc;
8480
8481 #ifdef CONFIG_RFS_ACCEL
8482         if (bp->flags & BNXT_FLAG_RFS)
8483                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8484 #endif
8485
8486         return rc;
8487 }
8488
8489 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8490                            bool shared)
8491 {
8492         int _rx = *rx, _tx = *tx;
8493
8494         if (shared) {
8495                 *rx = min_t(int, _rx, max);
8496                 *tx = min_t(int, _tx, max);
8497         } else {
8498                 if (max < 2)
8499                         return -ENOMEM;
8500
8501                 while (_rx + _tx > max) {
8502                         if (_rx > _tx && _rx > 1)
8503                                 _rx--;
8504                         else if (_tx > 1)
8505                                 _tx--;
8506                 }
8507                 *rx = _rx;
8508                 *tx = _tx;
8509         }
8510         return 0;
8511 }
8512
8513 static void bnxt_setup_msix(struct bnxt *bp)
8514 {
8515         const int len = sizeof(bp->irq_tbl[0].name);
8516         struct net_device *dev = bp->dev;
8517         int tcs, i;
8518
8519         tcs = netdev_get_num_tc(dev);
8520         if (tcs) {
8521                 int i, off, count;
8522
8523                 for (i = 0; i < tcs; i++) {
8524                         count = bp->tx_nr_rings_per_tc;
8525                         off = i * count;
8526                         netdev_set_tc_queue(dev, i, count, off);
8527                 }
8528         }
8529
8530         for (i = 0; i < bp->cp_nr_rings; i++) {
8531                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8532                 char *attr;
8533
8534                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8535                         attr = "TxRx";
8536                 else if (i < bp->rx_nr_rings)
8537                         attr = "rx";
8538                 else
8539                         attr = "tx";
8540
8541                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8542                          attr, i);
8543                 bp->irq_tbl[map_idx].handler = bnxt_msix;
8544         }
8545 }
8546
8547 static void bnxt_setup_inta(struct bnxt *bp)
8548 {
8549         const int len = sizeof(bp->irq_tbl[0].name);
8550
8551         if (netdev_get_num_tc(bp->dev))
8552                 netdev_reset_tc(bp->dev);
8553
8554         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
8555                  0);
8556         bp->irq_tbl[0].handler = bnxt_inta;
8557 }
8558
8559 static int bnxt_setup_int_mode(struct bnxt *bp)
8560 {
8561         int rc;
8562
8563         if (bp->flags & BNXT_FLAG_USING_MSIX)
8564                 bnxt_setup_msix(bp);
8565         else
8566                 bnxt_setup_inta(bp);
8567
8568         rc = bnxt_set_real_num_queues(bp);
8569         return rc;
8570 }
8571
8572 #ifdef CONFIG_RFS_ACCEL
8573 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
8574 {
8575         return bp->hw_resc.max_rsscos_ctxs;
8576 }
8577
8578 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
8579 {
8580         return bp->hw_resc.max_vnics;
8581 }
8582 #endif
8583
8584 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
8585 {
8586         return bp->hw_resc.max_stat_ctxs;
8587 }
8588
8589 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
8590 {
8591         return bp->hw_resc.max_cp_rings;
8592 }
8593
8594 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
8595 {
8596         unsigned int cp = bp->hw_resc.max_cp_rings;
8597
8598         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8599                 cp -= bnxt_get_ulp_msix_num(bp);
8600
8601         return cp;
8602 }
8603
8604 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
8605 {
8606         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8607
8608         if (bp->flags & BNXT_FLAG_CHIP_P5)
8609                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
8610
8611         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
8612 }
8613
8614 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
8615 {
8616         bp->hw_resc.max_irqs = max_irqs;
8617 }
8618
8619 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
8620 {
8621         unsigned int cp;
8622
8623         cp = bnxt_get_max_func_cp_rings_for_en(bp);
8624         if (bp->flags & BNXT_FLAG_CHIP_P5)
8625                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
8626         else
8627                 return cp - bp->cp_nr_rings;
8628 }
8629
8630 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
8631 {
8632         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8633 }
8634
8635 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8636 {
8637         int max_cp = bnxt_get_max_func_cp_rings(bp);
8638         int max_irq = bnxt_get_max_func_irqs(bp);
8639         int total_req = bp->cp_nr_rings + num;
8640         int max_idx, avail_msix;
8641
8642         max_idx = bp->total_irqs;
8643         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8644                 max_idx = min_t(int, bp->total_irqs, max_cp);
8645         avail_msix = max_idx - bp->cp_nr_rings;
8646         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8647                 return avail_msix;
8648
8649         if (max_irq < total_req) {
8650                 num = max_irq - bp->cp_nr_rings;
8651                 if (num <= 0)
8652                         return 0;
8653         }
8654         return num;
8655 }
8656
8657 static int bnxt_get_num_msix(struct bnxt *bp)
8658 {
8659         if (!BNXT_NEW_RM(bp))
8660                 return bnxt_get_max_func_irqs(bp);
8661
8662         return bnxt_nq_rings_in_use(bp);
8663 }
8664
8665 static int bnxt_init_msix(struct bnxt *bp)
8666 {
8667         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8668         struct msix_entry *msix_ent;
8669
8670         total_vecs = bnxt_get_num_msix(bp);
8671         max = bnxt_get_max_func_irqs(bp);
8672         if (total_vecs > max)
8673                 total_vecs = max;
8674
8675         if (!total_vecs)
8676                 return 0;
8677
8678         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8679         if (!msix_ent)
8680                 return -ENOMEM;
8681
8682         for (i = 0; i < total_vecs; i++) {
8683                 msix_ent[i].entry = i;
8684                 msix_ent[i].vector = 0;
8685         }
8686
8687         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8688                 min = 2;
8689
8690         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8691         ulp_msix = bnxt_get_ulp_msix_num(bp);
8692         if (total_vecs < 0 || total_vecs < ulp_msix) {
8693                 rc = -ENODEV;
8694                 goto msix_setup_exit;
8695         }
8696
8697         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8698         if (bp->irq_tbl) {
8699                 for (i = 0; i < total_vecs; i++)
8700                         bp->irq_tbl[i].vector = msix_ent[i].vector;
8701
8702                 bp->total_irqs = total_vecs;
8703                 /* Trim rings based upon num of vectors allocated */
8704                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8705                                      total_vecs - ulp_msix, min == 1);
8706                 if (rc)
8707                         goto msix_setup_exit;
8708
8709                 bp->cp_nr_rings = (min == 1) ?
8710                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8711                                   bp->tx_nr_rings + bp->rx_nr_rings;
8712
8713         } else {
8714                 rc = -ENOMEM;
8715                 goto msix_setup_exit;
8716         }
8717         bp->flags |= BNXT_FLAG_USING_MSIX;
8718         kfree(msix_ent);
8719         return 0;
8720
8721 msix_setup_exit:
8722         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8723         kfree(bp->irq_tbl);
8724         bp->irq_tbl = NULL;
8725         pci_disable_msix(bp->pdev);
8726         kfree(msix_ent);
8727         return rc;
8728 }
8729
8730 static int bnxt_init_inta(struct bnxt *bp)
8731 {
8732         bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL);
8733         if (!bp->irq_tbl)
8734                 return -ENOMEM;
8735
8736         bp->total_irqs = 1;
8737         bp->rx_nr_rings = 1;
8738         bp->tx_nr_rings = 1;
8739         bp->cp_nr_rings = 1;
8740         bp->flags |= BNXT_FLAG_SHARED_RINGS;
8741         bp->irq_tbl[0].vector = bp->pdev->irq;
8742         return 0;
8743 }
8744
8745 static int bnxt_init_int_mode(struct bnxt *bp)
8746 {
8747         int rc = 0;
8748
8749         if (bp->flags & BNXT_FLAG_MSIX_CAP)
8750                 rc = bnxt_init_msix(bp);
8751
8752         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8753                 /* fallback to INTA */
8754                 rc = bnxt_init_inta(bp);
8755         }
8756         return rc;
8757 }
8758
8759 static void bnxt_clear_int_mode(struct bnxt *bp)
8760 {
8761         if (bp->flags & BNXT_FLAG_USING_MSIX)
8762                 pci_disable_msix(bp->pdev);
8763
8764         kfree(bp->irq_tbl);
8765         bp->irq_tbl = NULL;
8766         bp->flags &= ~BNXT_FLAG_USING_MSIX;
8767 }
8768
8769 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8770 {
8771         int tcs = netdev_get_num_tc(bp->dev);
8772         bool irq_cleared = false;
8773         int rc;
8774
8775         if (!bnxt_need_reserve_rings(bp))
8776                 return 0;
8777
8778         if (irq_re_init && BNXT_NEW_RM(bp) &&
8779             bnxt_get_num_msix(bp) != bp->total_irqs) {
8780                 bnxt_ulp_irq_stop(bp);
8781                 bnxt_clear_int_mode(bp);
8782                 irq_cleared = true;
8783         }
8784         rc = __bnxt_reserve_rings(bp);
8785         if (irq_cleared) {
8786                 if (!rc)
8787                         rc = bnxt_init_int_mode(bp);
8788                 bnxt_ulp_irq_restart(bp, rc);
8789         }
8790         if (rc) {
8791                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8792                 return rc;
8793         }
8794         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8795                 netdev_err(bp->dev, "tx ring reservation failure\n");
8796                 netdev_reset_tc(bp->dev);
8797                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8798                 return -ENOMEM;
8799         }
8800         return 0;
8801 }
8802
8803 static void bnxt_free_irq(struct bnxt *bp)
8804 {
8805         struct bnxt_irq *irq;
8806         int i;
8807
8808 #ifdef CONFIG_RFS_ACCEL
8809         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8810         bp->dev->rx_cpu_rmap = NULL;
8811 #endif
8812         if (!bp->irq_tbl || !bp->bnapi)
8813                 return;
8814
8815         for (i = 0; i < bp->cp_nr_rings; i++) {
8816                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8817
8818                 irq = &bp->irq_tbl[map_idx];
8819                 if (irq->requested) {
8820                         if (irq->have_cpumask) {
8821                                 irq_set_affinity_hint(irq->vector, NULL);
8822                                 free_cpumask_var(irq->cpu_mask);
8823                                 irq->have_cpumask = 0;
8824                         }
8825                         free_irq(irq->vector, bp->bnapi[i]);
8826                 }
8827
8828                 irq->requested = 0;
8829         }
8830 }
8831
8832 static int bnxt_request_irq(struct bnxt *bp)
8833 {
8834         int i, j, rc = 0;
8835         unsigned long flags = 0;
8836 #ifdef CONFIG_RFS_ACCEL
8837         struct cpu_rmap *rmap;
8838 #endif
8839
8840         rc = bnxt_setup_int_mode(bp);
8841         if (rc) {
8842                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8843                            rc);
8844                 return rc;
8845         }
8846 #ifdef CONFIG_RFS_ACCEL
8847         rmap = bp->dev->rx_cpu_rmap;
8848 #endif
8849         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8850                 flags = IRQF_SHARED;
8851
8852         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8853                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8854                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8855
8856 #ifdef CONFIG_RFS_ACCEL
8857                 if (rmap && bp->bnapi[i]->rx_ring) {
8858                         rc = irq_cpu_rmap_add(rmap, irq->vector);
8859                         if (rc)
8860                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8861                                             j);
8862                         j++;
8863                 }
8864 #endif
8865                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8866                                  bp->bnapi[i]);
8867                 if (rc)
8868                         break;
8869
8870                 irq->requested = 1;
8871
8872                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8873                         int numa_node = dev_to_node(&bp->pdev->dev);
8874
8875                         irq->have_cpumask = 1;
8876                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8877                                         irq->cpu_mask);
8878                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8879                         if (rc) {
8880                                 netdev_warn(bp->dev,
8881                                             "Set affinity failed, IRQ = %d\n",
8882                                             irq->vector);
8883                                 break;
8884                         }
8885                 }
8886         }
8887         return rc;
8888 }
8889
8890 static void bnxt_del_napi(struct bnxt *bp)
8891 {
8892         int i;
8893
8894         if (!bp->bnapi)
8895                 return;
8896
8897         for (i = 0; i < bp->cp_nr_rings; i++) {
8898                 struct bnxt_napi *bnapi = bp->bnapi[i];
8899
8900                 __netif_napi_del(&bnapi->napi);
8901         }
8902         /* We called __netif_napi_del(), we need
8903          * to respect an RCU grace period before freeing napi structures.
8904          */
8905         synchronize_net();
8906 }
8907
8908 static void bnxt_init_napi(struct bnxt *bp)
8909 {
8910         int i;
8911         unsigned int cp_nr_rings = bp->cp_nr_rings;
8912         struct bnxt_napi *bnapi;
8913
8914         if (bp->flags & BNXT_FLAG_USING_MSIX) {
8915                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8916
8917                 if (bp->flags & BNXT_FLAG_CHIP_P5)
8918                         poll_fn = bnxt_poll_p5;
8919                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8920                         cp_nr_rings--;
8921                 for (i = 0; i < cp_nr_rings; i++) {
8922                         bnapi = bp->bnapi[i];
8923                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8924                 }
8925                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8926                         bnapi = bp->bnapi[cp_nr_rings];
8927                         netif_napi_add(bp->dev, &bnapi->napi,
8928                                        bnxt_poll_nitroa0, 64);
8929                 }
8930         } else {
8931                 bnapi = bp->bnapi[0];
8932                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8933         }
8934 }
8935
8936 static void bnxt_disable_napi(struct bnxt *bp)
8937 {
8938         int i;
8939
8940         if (!bp->bnapi ||
8941             test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
8942                 return;
8943
8944         for (i = 0; i < bp->cp_nr_rings; i++) {
8945                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8946
8947                 if (bp->bnapi[i]->rx_ring)
8948                         cancel_work_sync(&cpr->dim.work);
8949
8950                 napi_disable(&bp->bnapi[i]->napi);
8951         }
8952 }
8953
8954 static void bnxt_enable_napi(struct bnxt *bp)
8955 {
8956         int i;
8957
8958         clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
8959         for (i = 0; i < bp->cp_nr_rings; i++) {
8960                 struct bnxt_napi *bnapi = bp->bnapi[i];
8961                 struct bnxt_cp_ring_info *cpr;
8962
8963                 cpr = &bnapi->cp_ring;
8964                 if (bnapi->in_reset)
8965                         cpr->sw_stats.rx.rx_resets++;
8966                 bnapi->in_reset = false;
8967
8968                 if (bnapi->rx_ring) {
8969                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8970                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8971                 }
8972                 napi_enable(&bnapi->napi);
8973         }
8974 }
8975
8976 void bnxt_tx_disable(struct bnxt *bp)
8977 {
8978         int i;
8979         struct bnxt_tx_ring_info *txr;
8980
8981         if (bp->tx_ring) {
8982                 for (i = 0; i < bp->tx_nr_rings; i++) {
8983                         txr = &bp->tx_ring[i];
8984                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
8985                 }
8986         }
8987         /* Drop carrier first to prevent TX timeout */
8988         netif_carrier_off(bp->dev);
8989         /* Stop all TX queues */
8990         netif_tx_disable(bp->dev);
8991 }
8992
8993 void bnxt_tx_enable(struct bnxt *bp)
8994 {
8995         int i;
8996         struct bnxt_tx_ring_info *txr;
8997
8998         for (i = 0; i < bp->tx_nr_rings; i++) {
8999                 txr = &bp->tx_ring[i];
9000                 txr->dev_state = 0;
9001         }
9002         netif_tx_wake_all_queues(bp->dev);
9003         if (bp->link_info.link_up)
9004                 netif_carrier_on(bp->dev);
9005 }
9006
9007 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
9008 {
9009         u8 active_fec = link_info->active_fec_sig_mode &
9010                         PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
9011
9012         switch (active_fec) {
9013         default:
9014         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
9015                 return "None";
9016         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
9017                 return "Clause 74 BaseR";
9018         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
9019                 return "Clause 91 RS(528,514)";
9020         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
9021                 return "Clause 91 RS544_1XN";
9022         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
9023                 return "Clause 91 RS(544,514)";
9024         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
9025                 return "Clause 91 RS272_1XN";
9026         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
9027                 return "Clause 91 RS(272,257)";
9028         }
9029 }
9030
9031 static void bnxt_report_link(struct bnxt *bp)
9032 {
9033         if (bp->link_info.link_up) {
9034                 const char *duplex;
9035                 const char *flow_ctrl;
9036                 u32 speed;
9037                 u16 fec;
9038
9039                 netif_carrier_on(bp->dev);
9040                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
9041                 if (speed == SPEED_UNKNOWN) {
9042                         netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
9043                         return;
9044                 }
9045                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
9046                         duplex = "full";
9047                 else
9048                         duplex = "half";
9049                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
9050                         flow_ctrl = "ON - receive & transmit";
9051                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
9052                         flow_ctrl = "ON - transmit";
9053                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
9054                         flow_ctrl = "ON - receive";
9055                 else
9056                         flow_ctrl = "none";
9057                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
9058                             speed, duplex, flow_ctrl);
9059                 if (bp->flags & BNXT_FLAG_EEE_CAP)
9060                         netdev_info(bp->dev, "EEE is %s\n",
9061                                     bp->eee.eee_active ? "active" :
9062                                                          "not active");
9063                 fec = bp->link_info.fec_cfg;
9064                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
9065                         netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
9066                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
9067                                     bnxt_report_fec(&bp->link_info));
9068         } else {
9069                 netif_carrier_off(bp->dev);
9070                 netdev_err(bp->dev, "NIC Link is Down\n");
9071         }
9072 }
9073
9074 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
9075 {
9076         if (!resp->supported_speeds_auto_mode &&
9077             !resp->supported_speeds_force_mode &&
9078             !resp->supported_pam4_speeds_auto_mode &&
9079             !resp->supported_pam4_speeds_force_mode)
9080                 return true;
9081         return false;
9082 }
9083
9084 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
9085 {
9086         int rc = 0;
9087         struct hwrm_port_phy_qcaps_input req = {0};
9088         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9089         struct bnxt_link_info *link_info = &bp->link_info;
9090
9091         bp->flags &= ~BNXT_FLAG_EEE_CAP;
9092         if (bp->test_info)
9093                 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK |
9094                                           BNXT_TEST_FL_AN_PHY_LPBK);
9095         if (bp->hwrm_spec_code < 0x10201)
9096                 return 0;
9097
9098         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
9099
9100         mutex_lock(&bp->hwrm_cmd_lock);
9101         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9102         if (rc)
9103                 goto hwrm_phy_qcaps_exit;
9104
9105         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
9106                 struct ethtool_eee *eee = &bp->eee;
9107                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
9108
9109                 bp->flags |= BNXT_FLAG_EEE_CAP;
9110                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9111                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
9112                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
9113                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
9114                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
9115         }
9116         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
9117                 if (bp->test_info)
9118                         bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
9119         }
9120         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) {
9121                 if (bp->test_info)
9122                         bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK;
9123         }
9124         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) {
9125                 if (BNXT_PF(bp))
9126                         bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG;
9127         }
9128         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET)
9129                 bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET;
9130
9131         if (bp->hwrm_spec_code >= 0x10a01) {
9132                 if (bnxt_phy_qcaps_no_speed(resp)) {
9133                         link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9134                         netdev_warn(bp->dev, "Ethernet link disabled\n");
9135                 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9136                         link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9137                         netdev_info(bp->dev, "Ethernet link enabled\n");
9138                         /* Phy re-enabled, reprobe the speeds */
9139                         link_info->support_auto_speeds = 0;
9140                         link_info->support_pam4_auto_speeds = 0;
9141                 }
9142         }
9143         if (resp->supported_speeds_auto_mode)
9144                 link_info->support_auto_speeds =
9145                         le16_to_cpu(resp->supported_speeds_auto_mode);
9146         if (resp->supported_pam4_speeds_auto_mode)
9147                 link_info->support_pam4_auto_speeds =
9148                         le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9149
9150         bp->port_count = resp->port_cnt;
9151
9152 hwrm_phy_qcaps_exit:
9153         mutex_unlock(&bp->hwrm_cmd_lock);
9154         return rc;
9155 }
9156
9157 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9158 {
9159         u16 diff = advertising ^ supported;
9160
9161         return ((supported | diff) != supported);
9162 }
9163
9164 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9165 {
9166         int rc = 0;
9167         struct bnxt_link_info *link_info = &bp->link_info;
9168         struct hwrm_port_phy_qcfg_input req = {0};
9169         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9170         u8 link_up = link_info->link_up;
9171         bool support_changed = false;
9172
9173         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
9174
9175         mutex_lock(&bp->hwrm_cmd_lock);
9176         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9177         if (rc) {
9178                 mutex_unlock(&bp->hwrm_cmd_lock);
9179                 return rc;
9180         }
9181
9182         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9183         link_info->phy_link_status = resp->link;
9184         link_info->duplex = resp->duplex_cfg;
9185         if (bp->hwrm_spec_code >= 0x10800)
9186                 link_info->duplex = resp->duplex_state;
9187         link_info->pause = resp->pause;
9188         link_info->auto_mode = resp->auto_mode;
9189         link_info->auto_pause_setting = resp->auto_pause;
9190         link_info->lp_pause = resp->link_partner_adv_pause;
9191         link_info->force_pause_setting = resp->force_pause;
9192         link_info->duplex_setting = resp->duplex_cfg;
9193         if (link_info->phy_link_status == BNXT_LINK_LINK)
9194                 link_info->link_speed = le16_to_cpu(resp->link_speed);
9195         else
9196                 link_info->link_speed = 0;
9197         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9198         link_info->force_pam4_link_speed =
9199                 le16_to_cpu(resp->force_pam4_link_speed);
9200         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9201         link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9202         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9203         link_info->auto_pam4_link_speeds =
9204                 le16_to_cpu(resp->auto_pam4_link_speed_mask);
9205         link_info->lp_auto_link_speeds =
9206                 le16_to_cpu(resp->link_partner_adv_speeds);
9207         link_info->lp_auto_pam4_link_speeds =
9208                 resp->link_partner_pam4_adv_speeds;
9209         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9210         link_info->phy_ver[0] = resp->phy_maj;
9211         link_info->phy_ver[1] = resp->phy_min;
9212         link_info->phy_ver[2] = resp->phy_bld;
9213         link_info->media_type = resp->media_type;
9214         link_info->phy_type = resp->phy_type;
9215         link_info->transceiver = resp->xcvr_pkg_type;
9216         link_info->phy_addr = resp->eee_config_phy_addr &
9217                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9218         link_info->module_status = resp->module_status;
9219
9220         if (bp->flags & BNXT_FLAG_EEE_CAP) {
9221                 struct ethtool_eee *eee = &bp->eee;
9222                 u16 fw_speeds;
9223
9224                 eee->eee_active = 0;
9225                 if (resp->eee_config_phy_addr &
9226                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9227                         eee->eee_active = 1;
9228                         fw_speeds = le16_to_cpu(
9229                                 resp->link_partner_adv_eee_link_speed_mask);
9230                         eee->lp_advertised =
9231                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9232                 }
9233
9234                 /* Pull initial EEE config */
9235                 if (!chng_link_state) {
9236                         if (resp->eee_config_phy_addr &
9237                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9238                                 eee->eee_enabled = 1;
9239
9240                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9241                         eee->advertised =
9242                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9243
9244                         if (resp->eee_config_phy_addr &
9245                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9246                                 __le32 tmr;
9247
9248                                 eee->tx_lpi_enabled = 1;
9249                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9250                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
9251                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9252                         }
9253                 }
9254         }
9255
9256         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9257         if (bp->hwrm_spec_code >= 0x10504) {
9258                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9259                 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9260         }
9261         /* TODO: need to add more logic to report VF link */
9262         if (chng_link_state) {
9263                 if (link_info->phy_link_status == BNXT_LINK_LINK)
9264                         link_info->link_up = 1;
9265                 else
9266                         link_info->link_up = 0;
9267                 if (link_up != link_info->link_up)
9268                         bnxt_report_link(bp);
9269         } else {
9270                 /* alwasy link down if not require to update link state */
9271                 link_info->link_up = 0;
9272         }
9273         mutex_unlock(&bp->hwrm_cmd_lock);
9274
9275         if (!BNXT_PHY_CFG_ABLE(bp))
9276                 return 0;
9277
9278         /* Check if any advertised speeds are no longer supported. The caller
9279          * holds the link_lock mutex, so we can modify link_info settings.
9280          */
9281         if (bnxt_support_dropped(link_info->advertising,
9282                                  link_info->support_auto_speeds)) {
9283                 link_info->advertising = link_info->support_auto_speeds;
9284                 support_changed = true;
9285         }
9286         if (bnxt_support_dropped(link_info->advertising_pam4,
9287                                  link_info->support_pam4_auto_speeds)) {
9288                 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9289                 support_changed = true;
9290         }
9291         if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9292                 bnxt_hwrm_set_link_setting(bp, true, false);
9293         return 0;
9294 }
9295
9296 static void bnxt_get_port_module_status(struct bnxt *bp)
9297 {
9298         struct bnxt_link_info *link_info = &bp->link_info;
9299         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9300         u8 module_status;
9301
9302         if (bnxt_update_link(bp, true))
9303                 return;
9304
9305         module_status = link_info->module_status;
9306         switch (module_status) {
9307         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9308         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9309         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9310                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9311                             bp->pf.port_id);
9312                 if (bp->hwrm_spec_code >= 0x10201) {
9313                         netdev_warn(bp->dev, "Module part number %s\n",
9314                                     resp->phy_vendor_partnumber);
9315                 }
9316                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9317                         netdev_warn(bp->dev, "TX is disabled\n");
9318                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9319                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9320         }
9321 }
9322
9323 static void
9324 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9325 {
9326         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9327                 if (bp->hwrm_spec_code >= 0x10201)
9328                         req->auto_pause =
9329                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9330                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9331                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9332                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9333                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9334                 req->enables |=
9335                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9336         } else {
9337                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9338                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9339                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9340                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9341                 req->enables |=
9342                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9343                 if (bp->hwrm_spec_code >= 0x10201) {
9344                         req->auto_pause = req->force_pause;
9345                         req->enables |= cpu_to_le32(
9346                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9347                 }
9348         }
9349 }
9350
9351 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9352 {
9353         if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9354                 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9355                 if (bp->link_info.advertising) {
9356                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9357                         req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9358                 }
9359                 if (bp->link_info.advertising_pam4) {
9360                         req->enables |=
9361                                 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9362                         req->auto_link_pam4_speed_mask =
9363                                 cpu_to_le16(bp->link_info.advertising_pam4);
9364                 }
9365                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9366                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9367         } else {
9368                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9369                 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9370                         req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9371                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9372                 } else {
9373                         req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9374                 }
9375         }
9376
9377         /* tell chimp that the setting takes effect immediately */
9378         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9379 }
9380
9381 int bnxt_hwrm_set_pause(struct bnxt *bp)
9382 {
9383         struct hwrm_port_phy_cfg_input req = {0};
9384         int rc;
9385
9386         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9387         bnxt_hwrm_set_pause_common(bp, &req);
9388
9389         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9390             bp->link_info.force_link_chng)
9391                 bnxt_hwrm_set_link_common(bp, &req);
9392
9393         mutex_lock(&bp->hwrm_cmd_lock);
9394         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9395         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9396                 /* since changing of pause setting doesn't trigger any link
9397                  * change event, the driver needs to update the current pause
9398                  * result upon successfully return of the phy_cfg command
9399                  */
9400                 bp->link_info.pause =
9401                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9402                 bp->link_info.auto_pause_setting = 0;
9403                 if (!bp->link_info.force_link_chng)
9404                         bnxt_report_link(bp);
9405         }
9406         bp->link_info.force_link_chng = false;
9407         mutex_unlock(&bp->hwrm_cmd_lock);
9408         return rc;
9409 }
9410
9411 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9412                               struct hwrm_port_phy_cfg_input *req)
9413 {
9414         struct ethtool_eee *eee = &bp->eee;
9415
9416         if (eee->eee_enabled) {
9417                 u16 eee_speeds;
9418                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9419
9420                 if (eee->tx_lpi_enabled)
9421                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9422                 else
9423                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9424
9425                 req->flags |= cpu_to_le32(flags);
9426                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9427                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9428                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9429         } else {
9430                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9431         }
9432 }
9433
9434 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9435 {
9436         struct hwrm_port_phy_cfg_input req = {0};
9437
9438         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9439         if (set_pause)
9440                 bnxt_hwrm_set_pause_common(bp, &req);
9441
9442         bnxt_hwrm_set_link_common(bp, &req);
9443
9444         if (set_eee)
9445                 bnxt_hwrm_set_eee(bp, &req);
9446         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9447 }
9448
9449 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9450 {
9451         struct hwrm_port_phy_cfg_input req = {0};
9452
9453         if (!BNXT_SINGLE_PF(bp))
9454                 return 0;
9455
9456         if (pci_num_vf(bp->pdev))
9457                 return 0;
9458
9459         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9460         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9461         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9462 }
9463
9464 static int bnxt_fw_init_one(struct bnxt *bp);
9465
9466 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
9467 {
9468 #ifdef CONFIG_TEE_BNXT_FW
9469         int rc = tee_bnxt_fw_load();
9470
9471         if (rc)
9472                 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
9473
9474         return rc;
9475 #else
9476         netdev_err(bp->dev, "OP-TEE not supported\n");
9477         return -ENODEV;
9478 #endif
9479 }
9480
9481 static int bnxt_try_recover_fw(struct bnxt *bp)
9482 {
9483         if (bp->fw_health && bp->fw_health->status_reliable) {
9484                 int retry = 0, rc;
9485                 u32 sts;
9486
9487                 mutex_lock(&bp->hwrm_cmd_lock);
9488                 do {
9489                         rc = __bnxt_hwrm_ver_get(bp, true);
9490                         sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
9491                         if (!sts || !BNXT_FW_IS_BOOTING(sts))
9492                                 break;
9493                         retry++;
9494                 } while (rc == -EBUSY && retry < BNXT_FW_RETRY);
9495                 mutex_unlock(&bp->hwrm_cmd_lock);
9496
9497                 if (!BNXT_FW_IS_HEALTHY(sts)) {
9498                         netdev_err(bp->dev,
9499                                    "Firmware not responding, status: 0x%x\n",
9500                                    sts);
9501                         rc = -ENODEV;
9502                 }
9503                 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
9504                         netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
9505                         return bnxt_fw_reset_via_optee(bp);
9506                 }
9507                 return rc;
9508         }
9509
9510         return -ENODEV;
9511 }
9512
9513 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
9514 {
9515         struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
9516         struct hwrm_func_drv_if_change_input req = {0};
9517         bool resc_reinit = false, fw_reset = false;
9518         int rc, retry = 0;
9519         u32 flags = 0;
9520
9521         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
9522                 return 0;
9523
9524         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
9525         if (up)
9526                 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
9527         mutex_lock(&bp->hwrm_cmd_lock);
9528         while (retry < BNXT_FW_IF_RETRY) {
9529                 rc = _hwrm_send_message(bp, &req, sizeof(req),
9530                                         HWRM_CMD_TIMEOUT);
9531                 if (rc != -EAGAIN)
9532                         break;
9533
9534                 msleep(50);
9535                 retry++;
9536         }
9537         if (!rc)
9538                 flags = le32_to_cpu(resp->flags);
9539         mutex_unlock(&bp->hwrm_cmd_lock);
9540
9541         if (rc == -EAGAIN)
9542                 return rc;
9543         if (rc && up) {
9544                 rc = bnxt_try_recover_fw(bp);
9545                 fw_reset = true;
9546         }
9547         if (rc)
9548                 return rc;
9549
9550         if (!up)
9551                 return 0;
9552
9553         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
9554                 resc_reinit = true;
9555         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
9556                 fw_reset = true;
9557
9558         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
9559                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
9560                 return -ENODEV;
9561         }
9562         if (resc_reinit || fw_reset) {
9563                 if (fw_reset) {
9564                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9565                                 bnxt_ulp_stop(bp);
9566                         bnxt_free_ctx_mem(bp);
9567                         kfree(bp->ctx);
9568                         bp->ctx = NULL;
9569                         bnxt_dcb_free(bp);
9570                         rc = bnxt_fw_init_one(bp);
9571                         if (rc) {
9572                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9573                                 return rc;
9574                         }
9575                         bnxt_clear_int_mode(bp);
9576                         rc = bnxt_init_int_mode(bp);
9577                         if (rc) {
9578                                 netdev_err(bp->dev, "init int mode failed\n");
9579                                 return rc;
9580                         }
9581                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9582                 }
9583                 if (BNXT_NEW_RM(bp)) {
9584                         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9585
9586                         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9587                         hw_resc->resv_cp_rings = 0;
9588                         hw_resc->resv_stat_ctxs = 0;
9589                         hw_resc->resv_irqs = 0;
9590                         hw_resc->resv_tx_rings = 0;
9591                         hw_resc->resv_rx_rings = 0;
9592                         hw_resc->resv_hw_ring_grps = 0;
9593                         hw_resc->resv_vnics = 0;
9594                         if (!fw_reset) {
9595                                 bp->tx_nr_rings = 0;
9596                                 bp->rx_nr_rings = 0;
9597                         }
9598                 }
9599         }
9600         return 0;
9601 }
9602
9603 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
9604 {
9605         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9606         struct hwrm_port_led_qcaps_input req = {0};
9607         struct bnxt_pf_info *pf = &bp->pf;
9608         int rc;
9609
9610         bp->num_leds = 0;
9611         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
9612                 return 0;
9613
9614         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
9615         req.port_id = cpu_to_le16(pf->port_id);
9616         mutex_lock(&bp->hwrm_cmd_lock);
9617         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9618         if (rc) {
9619                 mutex_unlock(&bp->hwrm_cmd_lock);
9620                 return rc;
9621         }
9622         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
9623                 int i;
9624
9625                 bp->num_leds = resp->num_leds;
9626                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
9627                                                  bp->num_leds);
9628                 for (i = 0; i < bp->num_leds; i++) {
9629                         struct bnxt_led_info *led = &bp->leds[i];
9630                         __le16 caps = led->led_state_caps;
9631
9632                         if (!led->led_group_id ||
9633                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
9634                                 bp->num_leds = 0;
9635                                 break;
9636                         }
9637                 }
9638         }
9639         mutex_unlock(&bp->hwrm_cmd_lock);
9640         return 0;
9641 }
9642
9643 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
9644 {
9645         struct hwrm_wol_filter_alloc_input req = {0};
9646         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
9647         int rc;
9648
9649         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
9650         req.port_id = cpu_to_le16(bp->pf.port_id);
9651         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
9652         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
9653         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
9654         mutex_lock(&bp->hwrm_cmd_lock);
9655         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9656         if (!rc)
9657                 bp->wol_filter_id = resp->wol_filter_id;
9658         mutex_unlock(&bp->hwrm_cmd_lock);
9659         return rc;
9660 }
9661
9662 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
9663 {
9664         struct hwrm_wol_filter_free_input req = {0};
9665
9666         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
9667         req.port_id = cpu_to_le16(bp->pf.port_id);
9668         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
9669         req.wol_filter_id = bp->wol_filter_id;
9670         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9671 }
9672
9673 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
9674 {
9675         struct hwrm_wol_filter_qcfg_input req = {0};
9676         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9677         u16 next_handle = 0;
9678         int rc;
9679
9680         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
9681         req.port_id = cpu_to_le16(bp->pf.port_id);
9682         req.handle = cpu_to_le16(handle);
9683         mutex_lock(&bp->hwrm_cmd_lock);
9684         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9685         if (!rc) {
9686                 next_handle = le16_to_cpu(resp->next_handle);
9687                 if (next_handle != 0) {
9688                         if (resp->wol_type ==
9689                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
9690                                 bp->wol = 1;
9691                                 bp->wol_filter_id = resp->wol_filter_id;
9692                         }
9693                 }
9694         }
9695         mutex_unlock(&bp->hwrm_cmd_lock);
9696         return next_handle;
9697 }
9698
9699 static void bnxt_get_wol_settings(struct bnxt *bp)
9700 {
9701         u16 handle = 0;
9702
9703         bp->wol = 0;
9704         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
9705                 return;
9706
9707         do {
9708                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
9709         } while (handle && handle != 0xffff);
9710 }
9711
9712 #ifdef CONFIG_BNXT_HWMON
9713 static ssize_t bnxt_show_temp(struct device *dev,
9714                               struct device_attribute *devattr, char *buf)
9715 {
9716         struct hwrm_temp_monitor_query_input req = {0};
9717         struct hwrm_temp_monitor_query_output *resp;
9718         struct bnxt *bp = dev_get_drvdata(dev);
9719         u32 len = 0;
9720         int rc;
9721
9722         resp = bp->hwrm_cmd_resp_addr;
9723         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9724         mutex_lock(&bp->hwrm_cmd_lock);
9725         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9726         if (!rc)
9727                 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
9728         mutex_unlock(&bp->hwrm_cmd_lock);
9729         return rc ?: len;
9730 }
9731 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
9732
9733 static struct attribute *bnxt_attrs[] = {
9734         &sensor_dev_attr_temp1_input.dev_attr.attr,
9735         NULL
9736 };
9737 ATTRIBUTE_GROUPS(bnxt);
9738
9739 static void bnxt_hwmon_close(struct bnxt *bp)
9740 {
9741         if (bp->hwmon_dev) {
9742                 hwmon_device_unregister(bp->hwmon_dev);
9743                 bp->hwmon_dev = NULL;
9744         }
9745 }
9746
9747 static void bnxt_hwmon_open(struct bnxt *bp)
9748 {
9749         struct hwrm_temp_monitor_query_input req = {0};
9750         struct pci_dev *pdev = bp->pdev;
9751         int rc;
9752
9753         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9754         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9755         if (rc == -EACCES || rc == -EOPNOTSUPP) {
9756                 bnxt_hwmon_close(bp);
9757                 return;
9758         }
9759
9760         if (bp->hwmon_dev)
9761                 return;
9762
9763         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
9764                                                           DRV_MODULE_NAME, bp,
9765                                                           bnxt_groups);
9766         if (IS_ERR(bp->hwmon_dev)) {
9767                 bp->hwmon_dev = NULL;
9768                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
9769         }
9770 }
9771 #else
9772 static void bnxt_hwmon_close(struct bnxt *bp)
9773 {
9774 }
9775
9776 static void bnxt_hwmon_open(struct bnxt *bp)
9777 {
9778 }
9779 #endif
9780
9781 static bool bnxt_eee_config_ok(struct bnxt *bp)
9782 {
9783         struct ethtool_eee *eee = &bp->eee;
9784         struct bnxt_link_info *link_info = &bp->link_info;
9785
9786         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
9787                 return true;
9788
9789         if (eee->eee_enabled) {
9790                 u32 advertising =
9791                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
9792
9793                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9794                         eee->eee_enabled = 0;
9795                         return false;
9796                 }
9797                 if (eee->advertised & ~advertising) {
9798                         eee->advertised = advertising & eee->supported;
9799                         return false;
9800                 }
9801         }
9802         return true;
9803 }
9804
9805 static int bnxt_update_phy_setting(struct bnxt *bp)
9806 {
9807         int rc;
9808         bool update_link = false;
9809         bool update_pause = false;
9810         bool update_eee = false;
9811         struct bnxt_link_info *link_info = &bp->link_info;
9812
9813         rc = bnxt_update_link(bp, true);
9814         if (rc) {
9815                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9816                            rc);
9817                 return rc;
9818         }
9819         if (!BNXT_SINGLE_PF(bp))
9820                 return 0;
9821
9822         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9823             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9824             link_info->req_flow_ctrl)
9825                 update_pause = true;
9826         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9827             link_info->force_pause_setting != link_info->req_flow_ctrl)
9828                 update_pause = true;
9829         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9830                 if (BNXT_AUTO_MODE(link_info->auto_mode))
9831                         update_link = true;
9832                 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
9833                     link_info->req_link_speed != link_info->force_link_speed)
9834                         update_link = true;
9835                 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
9836                          link_info->req_link_speed != link_info->force_pam4_link_speed)
9837                         update_link = true;
9838                 if (link_info->req_duplex != link_info->duplex_setting)
9839                         update_link = true;
9840         } else {
9841                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9842                         update_link = true;
9843                 if (link_info->advertising != link_info->auto_link_speeds ||
9844                     link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
9845                         update_link = true;
9846         }
9847
9848         /* The last close may have shutdown the link, so need to call
9849          * PHY_CFG to bring it back up.
9850          */
9851         if (!bp->link_info.link_up)
9852                 update_link = true;
9853
9854         if (!bnxt_eee_config_ok(bp))
9855                 update_eee = true;
9856
9857         if (update_link)
9858                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9859         else if (update_pause)
9860                 rc = bnxt_hwrm_set_pause(bp);
9861         if (rc) {
9862                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9863                            rc);
9864                 return rc;
9865         }
9866
9867         return rc;
9868 }
9869
9870 /* Common routine to pre-map certain register block to different GRC window.
9871  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
9872  * in PF and 3 windows in VF that can be customized to map in different
9873  * register blocks.
9874  */
9875 static void bnxt_preset_reg_win(struct bnxt *bp)
9876 {
9877         if (BNXT_PF(bp)) {
9878                 /* CAG registers map to GRC window #4 */
9879                 writel(BNXT_CAG_REG_BASE,
9880                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
9881         }
9882 }
9883
9884 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
9885
9886 static int bnxt_reinit_after_abort(struct bnxt *bp)
9887 {
9888         int rc;
9889
9890         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9891                 return -EBUSY;
9892
9893         rc = bnxt_fw_init_one(bp);
9894         if (!rc) {
9895                 bnxt_clear_int_mode(bp);
9896                 rc = bnxt_init_int_mode(bp);
9897                 if (!rc) {
9898                         clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9899                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9900                 }
9901         }
9902         return rc;
9903 }
9904
9905 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9906 {
9907         int rc = 0;
9908
9909         bnxt_preset_reg_win(bp);
9910         netif_carrier_off(bp->dev);
9911         if (irq_re_init) {
9912                 /* Reserve rings now if none were reserved at driver probe. */
9913                 rc = bnxt_init_dflt_ring_mode(bp);
9914                 if (rc) {
9915                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
9916                         return rc;
9917                 }
9918         }
9919         rc = bnxt_reserve_rings(bp, irq_re_init);
9920         if (rc)
9921                 return rc;
9922         if ((bp->flags & BNXT_FLAG_RFS) &&
9923             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
9924                 /* disable RFS if falling back to INTA */
9925                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
9926                 bp->flags &= ~BNXT_FLAG_RFS;
9927         }
9928
9929         rc = bnxt_alloc_mem(bp, irq_re_init);
9930         if (rc) {
9931                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9932                 goto open_err_free_mem;
9933         }
9934
9935         if (irq_re_init) {
9936                 bnxt_init_napi(bp);
9937                 rc = bnxt_request_irq(bp);
9938                 if (rc) {
9939                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
9940                         goto open_err_irq;
9941                 }
9942         }
9943
9944         rc = bnxt_init_nic(bp, irq_re_init);
9945         if (rc) {
9946                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9947                 goto open_err_irq;
9948         }
9949
9950         bnxt_enable_napi(bp);
9951         bnxt_debug_dev_init(bp);
9952
9953         if (link_re_init) {
9954                 mutex_lock(&bp->link_lock);
9955                 rc = bnxt_update_phy_setting(bp);
9956                 mutex_unlock(&bp->link_lock);
9957                 if (rc) {
9958                         netdev_warn(bp->dev, "failed to update phy settings\n");
9959                         if (BNXT_SINGLE_PF(bp)) {
9960                                 bp->link_info.phy_retry = true;
9961                                 bp->link_info.phy_retry_expires =
9962                                         jiffies + 5 * HZ;
9963                         }
9964                 }
9965         }
9966
9967         if (irq_re_init)
9968                 udp_tunnel_nic_reset_ntf(bp->dev);
9969
9970         set_bit(BNXT_STATE_OPEN, &bp->state);
9971         bnxt_enable_int(bp);
9972         /* Enable TX queues */
9973         bnxt_tx_enable(bp);
9974         mod_timer(&bp->timer, jiffies + bp->current_interval);
9975         /* Poll link status and check for SFP+ module status */
9976         bnxt_get_port_module_status(bp);
9977
9978         /* VF-reps may need to be re-opened after the PF is re-opened */
9979         if (BNXT_PF(bp))
9980                 bnxt_vf_reps_open(bp);
9981         return 0;
9982
9983 open_err_irq:
9984         bnxt_del_napi(bp);
9985
9986 open_err_free_mem:
9987         bnxt_free_skbs(bp);
9988         bnxt_free_irq(bp);
9989         bnxt_free_mem(bp, true);
9990         return rc;
9991 }
9992
9993 /* rtnl_lock held */
9994 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9995 {
9996         int rc = 0;
9997
9998         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
9999                 rc = -EIO;
10000         if (!rc)
10001                 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
10002         if (rc) {
10003                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
10004                 dev_close(bp->dev);
10005         }
10006         return rc;
10007 }
10008
10009 /* rtnl_lock held, open the NIC half way by allocating all resources, but
10010  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
10011  * self tests.
10012  */
10013 int bnxt_half_open_nic(struct bnxt *bp)
10014 {
10015         int rc = 0;
10016
10017         rc = bnxt_alloc_mem(bp, false);
10018         if (rc) {
10019                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10020                 goto half_open_err;
10021         }
10022         rc = bnxt_init_nic(bp, false);
10023         if (rc) {
10024                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10025                 goto half_open_err;
10026         }
10027         return 0;
10028
10029 half_open_err:
10030         bnxt_free_skbs(bp);
10031         bnxt_free_mem(bp, false);
10032         dev_close(bp->dev);
10033         return rc;
10034 }
10035
10036 /* rtnl_lock held, this call can only be made after a previous successful
10037  * call to bnxt_half_open_nic().
10038  */
10039 void bnxt_half_close_nic(struct bnxt *bp)
10040 {
10041         bnxt_hwrm_resource_free(bp, false, false);
10042         bnxt_free_skbs(bp);
10043         bnxt_free_mem(bp, false);
10044 }
10045
10046 static void bnxt_reenable_sriov(struct bnxt *bp)
10047 {
10048         if (BNXT_PF(bp)) {
10049                 struct bnxt_pf_info *pf = &bp->pf;
10050                 int n = pf->active_vfs;
10051
10052                 if (n)
10053                         bnxt_cfg_hw_sriov(bp, &n, true);
10054         }
10055 }
10056
10057 static int bnxt_open(struct net_device *dev)
10058 {
10059         struct bnxt *bp = netdev_priv(dev);
10060         int rc;
10061
10062         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10063                 rc = bnxt_reinit_after_abort(bp);
10064                 if (rc) {
10065                         if (rc == -EBUSY)
10066                                 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
10067                         else
10068                                 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
10069                         return -ENODEV;
10070                 }
10071         }
10072
10073         rc = bnxt_hwrm_if_change(bp, true);
10074         if (rc)
10075                 return rc;
10076         rc = __bnxt_open_nic(bp, true, true);
10077         if (rc) {
10078                 bnxt_hwrm_if_change(bp, false);
10079         } else {
10080                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
10081                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10082                                 bnxt_ulp_start(bp, 0);
10083                                 bnxt_reenable_sriov(bp);
10084                         }
10085                 }
10086                 bnxt_hwmon_open(bp);
10087         }
10088
10089         return rc;
10090 }
10091
10092 static bool bnxt_drv_busy(struct bnxt *bp)
10093 {
10094         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
10095                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
10096 }
10097
10098 static void bnxt_get_ring_stats(struct bnxt *bp,
10099                                 struct rtnl_link_stats64 *stats);
10100
10101 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
10102                              bool link_re_init)
10103 {
10104         /* Close the VF-reps before closing PF */
10105         if (BNXT_PF(bp))
10106                 bnxt_vf_reps_close(bp);
10107
10108         /* Change device state to avoid TX queue wake up's */
10109         bnxt_tx_disable(bp);
10110
10111         clear_bit(BNXT_STATE_OPEN, &bp->state);
10112         smp_mb__after_atomic();
10113         while (bnxt_drv_busy(bp))
10114                 msleep(20);
10115
10116         /* Flush rings and and disable interrupts */
10117         bnxt_shutdown_nic(bp, irq_re_init);
10118
10119         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
10120
10121         bnxt_debug_dev_exit(bp);
10122         bnxt_disable_napi(bp);
10123         del_timer_sync(&bp->timer);
10124         bnxt_free_skbs(bp);
10125
10126         /* Save ring stats before shutdown */
10127         if (bp->bnapi && irq_re_init)
10128                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
10129         if (irq_re_init) {
10130                 bnxt_free_irq(bp);
10131                 bnxt_del_napi(bp);
10132         }
10133         bnxt_free_mem(bp, irq_re_init);
10134 }
10135
10136 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10137 {
10138         int rc = 0;
10139
10140         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10141                 /* If we get here, it means firmware reset is in progress
10142                  * while we are trying to close.  We can safely proceed with
10143                  * the close because we are holding rtnl_lock().  Some firmware
10144                  * messages may fail as we proceed to close.  We set the
10145                  * ABORT_ERR flag here so that the FW reset thread will later
10146                  * abort when it gets the rtnl_lock() and sees the flag.
10147                  */
10148                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
10149                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10150         }
10151
10152 #ifdef CONFIG_BNXT_SRIOV
10153         if (bp->sriov_cfg) {
10154                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
10155                                                       !bp->sriov_cfg,
10156                                                       BNXT_SRIOV_CFG_WAIT_TMO);
10157                 if (rc)
10158                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
10159         }
10160 #endif
10161         __bnxt_close_nic(bp, irq_re_init, link_re_init);
10162         return rc;
10163 }
10164
10165 static int bnxt_close(struct net_device *dev)
10166 {
10167         struct bnxt *bp = netdev_priv(dev);
10168
10169         bnxt_hwmon_close(bp);
10170         bnxt_close_nic(bp, true, true);
10171         bnxt_hwrm_shutdown_link(bp);
10172         bnxt_hwrm_if_change(bp, false);
10173         return 0;
10174 }
10175
10176 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
10177                                    u16 *val)
10178 {
10179         struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
10180         struct hwrm_port_phy_mdio_read_input req = {0};
10181         int rc;
10182
10183         if (bp->hwrm_spec_code < 0x10a00)
10184                 return -EOPNOTSUPP;
10185
10186         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
10187         req.port_id = cpu_to_le16(bp->pf.port_id);
10188         req.phy_addr = phy_addr;
10189         req.reg_addr = cpu_to_le16(reg & 0x1f);
10190         if (mdio_phy_id_is_c45(phy_addr)) {
10191                 req.cl45_mdio = 1;
10192                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
10193                 req.dev_addr = mdio_phy_id_devad(phy_addr);
10194                 req.reg_addr = cpu_to_le16(reg);
10195         }
10196
10197         mutex_lock(&bp->hwrm_cmd_lock);
10198         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10199         if (!rc)
10200                 *val = le16_to_cpu(resp->reg_data);
10201         mutex_unlock(&bp->hwrm_cmd_lock);
10202         return rc;
10203 }
10204
10205 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
10206                                     u16 val)
10207 {
10208         struct hwrm_port_phy_mdio_write_input req = {0};
10209
10210         if (bp->hwrm_spec_code < 0x10a00)
10211                 return -EOPNOTSUPP;
10212
10213         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
10214         req.port_id = cpu_to_le16(bp->pf.port_id);
10215         req.phy_addr = phy_addr;
10216         req.reg_addr = cpu_to_le16(reg & 0x1f);
10217         if (mdio_phy_id_is_c45(phy_addr)) {
10218                 req.cl45_mdio = 1;
10219                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
10220                 req.dev_addr = mdio_phy_id_devad(phy_addr);
10221                 req.reg_addr = cpu_to_le16(reg);
10222         }
10223         req.reg_data = cpu_to_le16(val);
10224
10225         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10226 }
10227
10228 /* rtnl_lock held */
10229 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10230 {
10231         struct mii_ioctl_data *mdio = if_mii(ifr);
10232         struct bnxt *bp = netdev_priv(dev);
10233         int rc;
10234
10235         switch (cmd) {
10236         case SIOCGMIIPHY:
10237                 mdio->phy_id = bp->link_info.phy_addr;
10238
10239                 fallthrough;
10240         case SIOCGMIIREG: {
10241                 u16 mii_regval = 0;
10242
10243                 if (!netif_running(dev))
10244                         return -EAGAIN;
10245
10246                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10247                                              &mii_regval);
10248                 mdio->val_out = mii_regval;
10249                 return rc;
10250         }
10251
10252         case SIOCSMIIREG:
10253                 if (!netif_running(dev))
10254                         return -EAGAIN;
10255
10256                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10257                                                 mdio->val_in);
10258
10259         default:
10260                 /* do nothing */
10261                 break;
10262         }
10263         return -EOPNOTSUPP;
10264 }
10265
10266 static void bnxt_get_ring_stats(struct bnxt *bp,
10267                                 struct rtnl_link_stats64 *stats)
10268 {
10269         int i;
10270
10271         for (i = 0; i < bp->cp_nr_rings; i++) {
10272                 struct bnxt_napi *bnapi = bp->bnapi[i];
10273                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10274                 u64 *sw = cpr->stats.sw_stats;
10275
10276                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10277                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10278                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10279
10280                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10281                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10282                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10283
10284                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10285                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10286                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10287
10288                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10289                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10290                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10291
10292                 stats->rx_missed_errors +=
10293                         BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10294
10295                 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10296
10297                 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10298         }
10299 }
10300
10301 static void bnxt_add_prev_stats(struct bnxt *bp,
10302                                 struct rtnl_link_stats64 *stats)
10303 {
10304         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10305
10306         stats->rx_packets += prev_stats->rx_packets;
10307         stats->tx_packets += prev_stats->tx_packets;
10308         stats->rx_bytes += prev_stats->rx_bytes;
10309         stats->tx_bytes += prev_stats->tx_bytes;
10310         stats->rx_missed_errors += prev_stats->rx_missed_errors;
10311         stats->multicast += prev_stats->multicast;
10312         stats->tx_dropped += prev_stats->tx_dropped;
10313 }
10314
10315 static void
10316 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10317 {
10318         struct bnxt *bp = netdev_priv(dev);
10319
10320         set_bit(BNXT_STATE_READ_STATS, &bp->state);
10321         /* Make sure bnxt_close_nic() sees that we are reading stats before
10322          * we check the BNXT_STATE_OPEN flag.
10323          */
10324         smp_mb__after_atomic();
10325         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10326                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10327                 *stats = bp->net_stats_prev;
10328                 return;
10329         }
10330
10331         bnxt_get_ring_stats(bp, stats);
10332         bnxt_add_prev_stats(bp, stats);
10333
10334         if (bp->flags & BNXT_FLAG_PORT_STATS) {
10335                 u64 *rx = bp->port_stats.sw_stats;
10336                 u64 *tx = bp->port_stats.sw_stats +
10337                           BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10338
10339                 stats->rx_crc_errors =
10340                         BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10341                 stats->rx_frame_errors =
10342                         BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10343                 stats->rx_length_errors =
10344                         BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10345                         BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10346                         BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10347                 stats->rx_errors =
10348                         BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10349                         BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10350                 stats->collisions =
10351                         BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10352                 stats->tx_fifo_errors =
10353                         BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10354                 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10355         }
10356         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10357 }
10358
10359 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10360 {
10361         struct net_device *dev = bp->dev;
10362         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10363         struct netdev_hw_addr *ha;
10364         u8 *haddr;
10365         int mc_count = 0;
10366         bool update = false;
10367         int off = 0;
10368
10369         netdev_for_each_mc_addr(ha, dev) {
10370                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
10371                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10372                         vnic->mc_list_count = 0;
10373                         return false;
10374                 }
10375                 haddr = ha->addr;
10376                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
10377                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
10378                         update = true;
10379                 }
10380                 off += ETH_ALEN;
10381                 mc_count++;
10382         }
10383         if (mc_count)
10384                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
10385
10386         if (mc_count != vnic->mc_list_count) {
10387                 vnic->mc_list_count = mc_count;
10388                 update = true;
10389         }
10390         return update;
10391 }
10392
10393 static bool bnxt_uc_list_updated(struct bnxt *bp)
10394 {
10395         struct net_device *dev = bp->dev;
10396         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10397         struct netdev_hw_addr *ha;
10398         int off = 0;
10399
10400         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
10401                 return true;
10402
10403         netdev_for_each_uc_addr(ha, dev) {
10404                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
10405                         return true;
10406
10407                 off += ETH_ALEN;
10408         }
10409         return false;
10410 }
10411
10412 static void bnxt_set_rx_mode(struct net_device *dev)
10413 {
10414         struct bnxt *bp = netdev_priv(dev);
10415         struct bnxt_vnic_info *vnic;
10416         bool mc_update = false;
10417         bool uc_update;
10418         u32 mask;
10419
10420         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
10421                 return;
10422
10423         vnic = &bp->vnic_info[0];
10424         mask = vnic->rx_mask;
10425         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
10426                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
10427                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
10428                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
10429
10430         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
10431                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10432
10433         uc_update = bnxt_uc_list_updated(bp);
10434
10435         if (dev->flags & IFF_BROADCAST)
10436                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10437         if (dev->flags & IFF_ALLMULTI) {
10438                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10439                 vnic->mc_list_count = 0;
10440         } else {
10441                 mc_update = bnxt_mc_list_updated(bp, &mask);
10442         }
10443
10444         if (mask != vnic->rx_mask || uc_update || mc_update) {
10445                 vnic->rx_mask = mask;
10446
10447                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
10448                 bnxt_queue_sp_work(bp);
10449         }
10450 }
10451
10452 static int bnxt_cfg_rx_mode(struct bnxt *bp)
10453 {
10454         struct net_device *dev = bp->dev;
10455         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10456         struct netdev_hw_addr *ha;
10457         int i, off = 0, rc;
10458         bool uc_update;
10459
10460         netif_addr_lock_bh(dev);
10461         uc_update = bnxt_uc_list_updated(bp);
10462         netif_addr_unlock_bh(dev);
10463
10464         if (!uc_update)
10465                 goto skip_uc;
10466
10467         mutex_lock(&bp->hwrm_cmd_lock);
10468         for (i = 1; i < vnic->uc_filter_count; i++) {
10469                 struct hwrm_cfa_l2_filter_free_input req = {0};
10470
10471                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
10472                                        -1);
10473
10474                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
10475
10476                 rc = _hwrm_send_message(bp, &req, sizeof(req),
10477                                         HWRM_CMD_TIMEOUT);
10478         }
10479         mutex_unlock(&bp->hwrm_cmd_lock);
10480
10481         vnic->uc_filter_count = 1;
10482
10483         netif_addr_lock_bh(dev);
10484         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
10485                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10486         } else {
10487                 netdev_for_each_uc_addr(ha, dev) {
10488                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
10489                         off += ETH_ALEN;
10490                         vnic->uc_filter_count++;
10491                 }
10492         }
10493         netif_addr_unlock_bh(dev);
10494
10495         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
10496                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
10497                 if (rc) {
10498                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
10499                                    rc);
10500                         vnic->uc_filter_count = i;
10501                         return rc;
10502                 }
10503         }
10504
10505 skip_uc:
10506         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10507         if (rc && vnic->mc_list_count) {
10508                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
10509                             rc);
10510                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10511                 vnic->mc_list_count = 0;
10512                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10513         }
10514         if (rc)
10515                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
10516                            rc);
10517
10518         return rc;
10519 }
10520
10521 static bool bnxt_can_reserve_rings(struct bnxt *bp)
10522 {
10523 #ifdef CONFIG_BNXT_SRIOV
10524         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
10525                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10526
10527                 /* No minimum rings were provisioned by the PF.  Don't
10528                  * reserve rings by default when device is down.
10529                  */
10530                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
10531                         return true;
10532
10533                 if (!netif_running(bp->dev))
10534                         return false;
10535         }
10536 #endif
10537         return true;
10538 }
10539
10540 /* If the chip and firmware supports RFS */
10541 static bool bnxt_rfs_supported(struct bnxt *bp)
10542 {
10543         if (bp->flags & BNXT_FLAG_CHIP_P5) {
10544                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
10545                         return true;
10546                 return false;
10547         }
10548         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
10549                 return true;
10550         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10551                 return true;
10552         return false;
10553 }
10554
10555 /* If runtime conditions support RFS */
10556 static bool bnxt_rfs_capable(struct bnxt *bp)
10557 {
10558 #ifdef CONFIG_RFS_ACCEL
10559         int vnics, max_vnics, max_rss_ctxs;
10560
10561         if (bp->flags & BNXT_FLAG_CHIP_P5)
10562                 return bnxt_rfs_supported(bp);
10563         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
10564                 return false;
10565
10566         vnics = 1 + bp->rx_nr_rings;
10567         max_vnics = bnxt_get_max_func_vnics(bp);
10568         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
10569
10570         /* RSS contexts not a limiting factor */
10571         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10572                 max_rss_ctxs = max_vnics;
10573         if (vnics > max_vnics || vnics > max_rss_ctxs) {
10574                 if (bp->rx_nr_rings > 1)
10575                         netdev_warn(bp->dev,
10576                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
10577                                     min(max_rss_ctxs - 1, max_vnics - 1));
10578                 return false;
10579         }
10580
10581         if (!BNXT_NEW_RM(bp))
10582                 return true;
10583
10584         if (vnics == bp->hw_resc.resv_vnics)
10585                 return true;
10586
10587         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
10588         if (vnics <= bp->hw_resc.resv_vnics)
10589                 return true;
10590
10591         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
10592         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
10593         return false;
10594 #else
10595         return false;
10596 #endif
10597 }
10598
10599 static netdev_features_t bnxt_fix_features(struct net_device *dev,
10600                                            netdev_features_t features)
10601 {
10602         struct bnxt *bp = netdev_priv(dev);
10603         netdev_features_t vlan_features;
10604
10605         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
10606                 features &= ~NETIF_F_NTUPLE;
10607
10608         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10609                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10610
10611         if (!(features & NETIF_F_GRO))
10612                 features &= ~NETIF_F_GRO_HW;
10613
10614         if (features & NETIF_F_GRO_HW)
10615                 features &= ~NETIF_F_LRO;
10616
10617         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
10618          * turned on or off together.
10619          */
10620         vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
10621         if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
10622                 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10623                         features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10624                 else if (vlan_features)
10625                         features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
10626         }
10627 #ifdef CONFIG_BNXT_SRIOV
10628         if (BNXT_VF(bp) && bp->vf.vlan)
10629                 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10630 #endif
10631         return features;
10632 }
10633
10634 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
10635 {
10636         struct bnxt *bp = netdev_priv(dev);
10637         u32 flags = bp->flags;
10638         u32 changes;
10639         int rc = 0;
10640         bool re_init = false;
10641         bool update_tpa = false;
10642
10643         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
10644         if (features & NETIF_F_GRO_HW)
10645                 flags |= BNXT_FLAG_GRO;
10646         else if (features & NETIF_F_LRO)
10647                 flags |= BNXT_FLAG_LRO;
10648
10649         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10650                 flags &= ~BNXT_FLAG_TPA;
10651
10652         if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10653                 flags |= BNXT_FLAG_STRIP_VLAN;
10654
10655         if (features & NETIF_F_NTUPLE)
10656                 flags |= BNXT_FLAG_RFS;
10657
10658         changes = flags ^ bp->flags;
10659         if (changes & BNXT_FLAG_TPA) {
10660                 update_tpa = true;
10661                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
10662                     (flags & BNXT_FLAG_TPA) == 0 ||
10663                     (bp->flags & BNXT_FLAG_CHIP_P5))
10664                         re_init = true;
10665         }
10666
10667         if (changes & ~BNXT_FLAG_TPA)
10668                 re_init = true;
10669
10670         if (flags != bp->flags) {
10671                 u32 old_flags = bp->flags;
10672
10673                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10674                         bp->flags = flags;
10675                         if (update_tpa)
10676                                 bnxt_set_ring_params(bp);
10677                         return rc;
10678                 }
10679
10680                 if (re_init) {
10681                         bnxt_close_nic(bp, false, false);
10682                         bp->flags = flags;
10683                         if (update_tpa)
10684                                 bnxt_set_ring_params(bp);
10685
10686                         return bnxt_open_nic(bp, false, false);
10687                 }
10688                 if (update_tpa) {
10689                         bp->flags = flags;
10690                         rc = bnxt_set_tpa(bp,
10691                                           (flags & BNXT_FLAG_TPA) ?
10692                                           true : false);
10693                         if (rc)
10694                                 bp->flags = old_flags;
10695                 }
10696         }
10697         return rc;
10698 }
10699
10700 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
10701                          u32 *reg_buf)
10702 {
10703         struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr;
10704         struct hwrm_dbg_read_direct_input req = {0};
10705         __le32 *dbg_reg_buf;
10706         dma_addr_t mapping;
10707         int rc, i;
10708
10709         dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4,
10710                                          &mapping, GFP_KERNEL);
10711         if (!dbg_reg_buf)
10712                 return -ENOMEM;
10713         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1);
10714         req.host_dest_addr = cpu_to_le64(mapping);
10715         req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
10716         req.read_len32 = cpu_to_le32(num_words);
10717         mutex_lock(&bp->hwrm_cmd_lock);
10718         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10719         if (rc || resp->error_code) {
10720                 rc = -EIO;
10721                 goto dbg_rd_reg_exit;
10722         }
10723         for (i = 0; i < num_words; i++)
10724                 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
10725
10726 dbg_rd_reg_exit:
10727         mutex_unlock(&bp->hwrm_cmd_lock);
10728         dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping);
10729         return rc;
10730 }
10731
10732 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
10733                                        u32 ring_id, u32 *prod, u32 *cons)
10734 {
10735         struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
10736         struct hwrm_dbg_ring_info_get_input req = {0};
10737         int rc;
10738
10739         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
10740         req.ring_type = ring_type;
10741         req.fw_ring_id = cpu_to_le32(ring_id);
10742         mutex_lock(&bp->hwrm_cmd_lock);
10743         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10744         if (!rc) {
10745                 *prod = le32_to_cpu(resp->producer_index);
10746                 *cons = le32_to_cpu(resp->consumer_index);
10747         }
10748         mutex_unlock(&bp->hwrm_cmd_lock);
10749         return rc;
10750 }
10751
10752 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
10753 {
10754         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
10755         int i = bnapi->index;
10756
10757         if (!txr)
10758                 return;
10759
10760         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
10761                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
10762                     txr->tx_cons);
10763 }
10764
10765 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
10766 {
10767         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
10768         int i = bnapi->index;
10769
10770         if (!rxr)
10771                 return;
10772
10773         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
10774                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
10775                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
10776                     rxr->rx_sw_agg_prod);
10777 }
10778
10779 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
10780 {
10781         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10782         int i = bnapi->index;
10783
10784         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
10785                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
10786 }
10787
10788 static void bnxt_dbg_dump_states(struct bnxt *bp)
10789 {
10790         int i;
10791         struct bnxt_napi *bnapi;
10792
10793         for (i = 0; i < bp->cp_nr_rings; i++) {
10794                 bnapi = bp->bnapi[i];
10795                 if (netif_msg_drv(bp)) {
10796                         bnxt_dump_tx_sw_state(bnapi);
10797                         bnxt_dump_rx_sw_state(bnapi);
10798                         bnxt_dump_cp_sw_state(bnapi);
10799                 }
10800         }
10801 }
10802
10803 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
10804 {
10805         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
10806         struct hwrm_ring_reset_input req = {0};
10807         struct bnxt_napi *bnapi = rxr->bnapi;
10808         struct bnxt_cp_ring_info *cpr;
10809         u16 cp_ring_id;
10810
10811         cpr = &bnapi->cp_ring;
10812         cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
10813         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_RESET, cp_ring_id, -1);
10814         req.ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
10815         req.ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
10816         return hwrm_send_message_silent(bp, &req, sizeof(req),
10817                                         HWRM_CMD_TIMEOUT);
10818 }
10819
10820 static void bnxt_reset_task(struct bnxt *bp, bool silent)
10821 {
10822         if (!silent)
10823                 bnxt_dbg_dump_states(bp);
10824         if (netif_running(bp->dev)) {
10825                 int rc;
10826
10827                 if (silent) {
10828                         bnxt_close_nic(bp, false, false);
10829                         bnxt_open_nic(bp, false, false);
10830                 } else {
10831                         bnxt_ulp_stop(bp);
10832                         bnxt_close_nic(bp, true, false);
10833                         rc = bnxt_open_nic(bp, true, false);
10834                         bnxt_ulp_start(bp, rc);
10835                 }
10836         }
10837 }
10838
10839 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
10840 {
10841         struct bnxt *bp = netdev_priv(dev);
10842
10843         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
10844         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
10845         bnxt_queue_sp_work(bp);
10846 }
10847
10848 static void bnxt_fw_health_check(struct bnxt *bp)
10849 {
10850         struct bnxt_fw_health *fw_health = bp->fw_health;
10851         u32 val;
10852
10853         if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10854                 return;
10855
10856         if (fw_health->tmr_counter) {
10857                 fw_health->tmr_counter--;
10858                 return;
10859         }
10860
10861         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10862         if (val == fw_health->last_fw_heartbeat)
10863                 goto fw_reset;
10864
10865         fw_health->last_fw_heartbeat = val;
10866
10867         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10868         if (val != fw_health->last_fw_reset_cnt)
10869                 goto fw_reset;
10870
10871         fw_health->tmr_counter = fw_health->tmr_multiplier;
10872         return;
10873
10874 fw_reset:
10875         set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
10876         bnxt_queue_sp_work(bp);
10877 }
10878
10879 static void bnxt_timer(struct timer_list *t)
10880 {
10881         struct bnxt *bp = from_timer(bp, t, timer);
10882         struct net_device *dev = bp->dev;
10883
10884         if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
10885                 return;
10886
10887         if (atomic_read(&bp->intr_sem) != 0)
10888                 goto bnxt_restart_timer;
10889
10890         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
10891                 bnxt_fw_health_check(bp);
10892
10893         if (bp->link_info.link_up && bp->stats_coal_ticks) {
10894                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
10895                 bnxt_queue_sp_work(bp);
10896         }
10897
10898         if (bnxt_tc_flower_enabled(bp)) {
10899                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
10900                 bnxt_queue_sp_work(bp);
10901         }
10902
10903 #ifdef CONFIG_RFS_ACCEL
10904         if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
10905                 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
10906                 bnxt_queue_sp_work(bp);
10907         }
10908 #endif /*CONFIG_RFS_ACCEL*/
10909
10910         if (bp->link_info.phy_retry) {
10911                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
10912                         bp->link_info.phy_retry = false;
10913                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
10914                 } else {
10915                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
10916                         bnxt_queue_sp_work(bp);
10917                 }
10918         }
10919
10920         if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
10921             netif_carrier_ok(dev)) {
10922                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
10923                 bnxt_queue_sp_work(bp);
10924         }
10925 bnxt_restart_timer:
10926         mod_timer(&bp->timer, jiffies + bp->current_interval);
10927 }
10928
10929 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
10930 {
10931         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
10932          * set.  If the device is being closed, bnxt_close() may be holding
10933          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
10934          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
10935          */
10936         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10937         rtnl_lock();
10938 }
10939
10940 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
10941 {
10942         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10943         rtnl_unlock();
10944 }
10945
10946 /* Only called from bnxt_sp_task() */
10947 static void bnxt_reset(struct bnxt *bp, bool silent)
10948 {
10949         bnxt_rtnl_lock_sp(bp);
10950         if (test_bit(BNXT_STATE_OPEN, &bp->state))
10951                 bnxt_reset_task(bp, silent);
10952         bnxt_rtnl_unlock_sp(bp);
10953 }
10954
10955 /* Only called from bnxt_sp_task() */
10956 static void bnxt_rx_ring_reset(struct bnxt *bp)
10957 {
10958         int i;
10959
10960         bnxt_rtnl_lock_sp(bp);
10961         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10962                 bnxt_rtnl_unlock_sp(bp);
10963                 return;
10964         }
10965         /* Disable and flush TPA before resetting the RX ring */
10966         if (bp->flags & BNXT_FLAG_TPA)
10967                 bnxt_set_tpa(bp, false);
10968         for (i = 0; i < bp->rx_nr_rings; i++) {
10969                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
10970                 struct bnxt_cp_ring_info *cpr;
10971                 int rc;
10972
10973                 if (!rxr->bnapi->in_reset)
10974                         continue;
10975
10976                 rc = bnxt_hwrm_rx_ring_reset(bp, i);
10977                 if (rc) {
10978                         if (rc == -EINVAL || rc == -EOPNOTSUPP)
10979                                 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
10980                         else
10981                                 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
10982                                             rc);
10983                         bnxt_reset_task(bp, true);
10984                         break;
10985                 }
10986                 bnxt_free_one_rx_ring_skbs(bp, i);
10987                 rxr->rx_prod = 0;
10988                 rxr->rx_agg_prod = 0;
10989                 rxr->rx_sw_agg_prod = 0;
10990                 rxr->rx_next_cons = 0;
10991                 rxr->bnapi->in_reset = false;
10992                 bnxt_alloc_one_rx_ring(bp, i);
10993                 cpr = &rxr->bnapi->cp_ring;
10994                 cpr->sw_stats.rx.rx_resets++;
10995                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
10996                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
10997                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
10998         }
10999         if (bp->flags & BNXT_FLAG_TPA)
11000                 bnxt_set_tpa(bp, true);
11001         bnxt_rtnl_unlock_sp(bp);
11002 }
11003
11004 static void bnxt_fw_reset_close(struct bnxt *bp)
11005 {
11006         bnxt_ulp_stop(bp);
11007         /* When firmware is in fatal state, quiesce device and disable
11008          * bus master to prevent any potential bad DMAs before freeing
11009          * kernel memory.
11010          */
11011         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11012                 u16 val = 0;
11013
11014                 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
11015                 if (val == 0xffff)
11016                         bp->fw_reset_min_dsecs = 0;
11017                 bnxt_tx_disable(bp);
11018                 bnxt_disable_napi(bp);
11019                 bnxt_disable_int_sync(bp);
11020                 bnxt_free_irq(bp);
11021                 bnxt_clear_int_mode(bp);
11022                 pci_disable_device(bp->pdev);
11023         }
11024         __bnxt_close_nic(bp, true, false);
11025         bnxt_clear_int_mode(bp);
11026         bnxt_hwrm_func_drv_unrgtr(bp);
11027         if (pci_is_enabled(bp->pdev))
11028                 pci_disable_device(bp->pdev);
11029         bnxt_free_ctx_mem(bp);
11030         kfree(bp->ctx);
11031         bp->ctx = NULL;
11032 }
11033
11034 static bool is_bnxt_fw_ok(struct bnxt *bp)
11035 {
11036         struct bnxt_fw_health *fw_health = bp->fw_health;
11037         bool no_heartbeat = false, has_reset = false;
11038         u32 val;
11039
11040         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11041         if (val == fw_health->last_fw_heartbeat)
11042                 no_heartbeat = true;
11043
11044         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11045         if (val != fw_health->last_fw_reset_cnt)
11046                 has_reset = true;
11047
11048         if (!no_heartbeat && has_reset)
11049                 return true;
11050
11051         return false;
11052 }
11053
11054 /* rtnl_lock is acquired before calling this function */
11055 static void bnxt_force_fw_reset(struct bnxt *bp)
11056 {
11057         struct bnxt_fw_health *fw_health = bp->fw_health;
11058         u32 wait_dsecs;
11059
11060         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
11061             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11062                 return;
11063
11064         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11065         bnxt_fw_reset_close(bp);
11066         wait_dsecs = fw_health->master_func_wait_dsecs;
11067         if (fw_health->master) {
11068                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
11069                         wait_dsecs = 0;
11070                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11071         } else {
11072                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
11073                 wait_dsecs = fw_health->normal_func_wait_dsecs;
11074                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11075         }
11076
11077         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
11078         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
11079         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11080 }
11081
11082 void bnxt_fw_exception(struct bnxt *bp)
11083 {
11084         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
11085         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11086         bnxt_rtnl_lock_sp(bp);
11087         bnxt_force_fw_reset(bp);
11088         bnxt_rtnl_unlock_sp(bp);
11089 }
11090
11091 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
11092  * < 0 on error.
11093  */
11094 static int bnxt_get_registered_vfs(struct bnxt *bp)
11095 {
11096 #ifdef CONFIG_BNXT_SRIOV
11097         int rc;
11098
11099         if (!BNXT_PF(bp))
11100                 return 0;
11101
11102         rc = bnxt_hwrm_func_qcfg(bp);
11103         if (rc) {
11104                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
11105                 return rc;
11106         }
11107         if (bp->pf.registered_vfs)
11108                 return bp->pf.registered_vfs;
11109         if (bp->sriov_cfg)
11110                 return 1;
11111 #endif
11112         return 0;
11113 }
11114
11115 void bnxt_fw_reset(struct bnxt *bp)
11116 {
11117         bnxt_rtnl_lock_sp(bp);
11118         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
11119             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11120                 int n = 0, tmo;
11121
11122                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11123                 if (bp->pf.active_vfs &&
11124                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
11125                         n = bnxt_get_registered_vfs(bp);
11126                 if (n < 0) {
11127                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
11128                                    n);
11129                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11130                         dev_close(bp->dev);
11131                         goto fw_reset_exit;
11132                 } else if (n > 0) {
11133                         u16 vf_tmo_dsecs = n * 10;
11134
11135                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
11136                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
11137                         bp->fw_reset_state =
11138                                 BNXT_FW_RESET_STATE_POLL_VF;
11139                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11140                         goto fw_reset_exit;
11141                 }
11142                 bnxt_fw_reset_close(bp);
11143                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11144                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11145                         tmo = HZ / 10;
11146                 } else {
11147                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11148                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11149                 }
11150                 bnxt_queue_fw_reset_work(bp, tmo);
11151         }
11152 fw_reset_exit:
11153         bnxt_rtnl_unlock_sp(bp);
11154 }
11155
11156 static void bnxt_chk_missed_irq(struct bnxt *bp)
11157 {
11158         int i;
11159
11160         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11161                 return;
11162
11163         for (i = 0; i < bp->cp_nr_rings; i++) {
11164                 struct bnxt_napi *bnapi = bp->bnapi[i];
11165                 struct bnxt_cp_ring_info *cpr;
11166                 u32 fw_ring_id;
11167                 int j;
11168
11169                 if (!bnapi)
11170                         continue;
11171
11172                 cpr = &bnapi->cp_ring;
11173                 for (j = 0; j < 2; j++) {
11174                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
11175                         u32 val[2];
11176
11177                         if (!cpr2 || cpr2->has_more_work ||
11178                             !bnxt_has_work(bp, cpr2))
11179                                 continue;
11180
11181                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
11182                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
11183                                 continue;
11184                         }
11185                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
11186                         bnxt_dbg_hwrm_ring_info_get(bp,
11187                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
11188                                 fw_ring_id, &val[0], &val[1]);
11189                         cpr->sw_stats.cmn.missed_irqs++;
11190                 }
11191         }
11192 }
11193
11194 static void bnxt_cfg_ntp_filters(struct bnxt *);
11195
11196 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
11197 {
11198         struct bnxt_link_info *link_info = &bp->link_info;
11199
11200         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
11201                 link_info->autoneg = BNXT_AUTONEG_SPEED;
11202                 if (bp->hwrm_spec_code >= 0x10201) {
11203                         if (link_info->auto_pause_setting &
11204                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
11205                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11206                 } else {
11207                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11208                 }
11209                 link_info->advertising = link_info->auto_link_speeds;
11210                 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
11211         } else {
11212                 link_info->req_link_speed = link_info->force_link_speed;
11213                 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
11214                 if (link_info->force_pam4_link_speed) {
11215                         link_info->req_link_speed =
11216                                 link_info->force_pam4_link_speed;
11217                         link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
11218                 }
11219                 link_info->req_duplex = link_info->duplex_setting;
11220         }
11221         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
11222                 link_info->req_flow_ctrl =
11223                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
11224         else
11225                 link_info->req_flow_ctrl = link_info->force_pause_setting;
11226 }
11227
11228 static void bnxt_fw_echo_reply(struct bnxt *bp)
11229 {
11230         struct bnxt_fw_health *fw_health = bp->fw_health;
11231         struct hwrm_func_echo_response_input req = {0};
11232
11233         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_ECHO_RESPONSE, -1, -1);
11234         req.event_data1 = cpu_to_le32(fw_health->echo_req_data1);
11235         req.event_data2 = cpu_to_le32(fw_health->echo_req_data2);
11236         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11237 }
11238
11239 static void bnxt_sp_task(struct work_struct *work)
11240 {
11241         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
11242
11243         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11244         smp_mb__after_atomic();
11245         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11246                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11247                 return;
11248         }
11249
11250         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
11251                 bnxt_cfg_rx_mode(bp);
11252
11253         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
11254                 bnxt_cfg_ntp_filters(bp);
11255         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
11256                 bnxt_hwrm_exec_fwd_req(bp);
11257         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
11258                 bnxt_hwrm_port_qstats(bp, 0);
11259                 bnxt_hwrm_port_qstats_ext(bp, 0);
11260                 bnxt_accumulate_all_stats(bp);
11261         }
11262
11263         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
11264                 int rc;
11265
11266                 mutex_lock(&bp->link_lock);
11267                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
11268                                        &bp->sp_event))
11269                         bnxt_hwrm_phy_qcaps(bp);
11270
11271                 rc = bnxt_update_link(bp, true);
11272                 if (rc)
11273                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
11274                                    rc);
11275
11276                 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
11277                                        &bp->sp_event))
11278                         bnxt_init_ethtool_link_settings(bp);
11279                 mutex_unlock(&bp->link_lock);
11280         }
11281         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
11282                 int rc;
11283
11284                 mutex_lock(&bp->link_lock);
11285                 rc = bnxt_update_phy_setting(bp);
11286                 mutex_unlock(&bp->link_lock);
11287                 if (rc) {
11288                         netdev_warn(bp->dev, "update phy settings retry failed\n");
11289                 } else {
11290                         bp->link_info.phy_retry = false;
11291                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
11292                 }
11293         }
11294         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
11295                 mutex_lock(&bp->link_lock);
11296                 bnxt_get_port_module_status(bp);
11297                 mutex_unlock(&bp->link_lock);
11298         }
11299
11300         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
11301                 bnxt_tc_flow_stats_work(bp);
11302
11303         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
11304                 bnxt_chk_missed_irq(bp);
11305
11306         if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
11307                 bnxt_fw_echo_reply(bp);
11308
11309         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
11310          * must be the last functions to be called before exiting.
11311          */
11312         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
11313                 bnxt_reset(bp, false);
11314
11315         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
11316                 bnxt_reset(bp, true);
11317
11318         if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
11319                 bnxt_rx_ring_reset(bp);
11320
11321         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
11322                 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
11323
11324         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
11325                 if (!is_bnxt_fw_ok(bp))
11326                         bnxt_devlink_health_report(bp,
11327                                                    BNXT_FW_EXCEPTION_SP_EVENT);
11328         }
11329
11330         smp_mb__before_atomic();
11331         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11332 }
11333
11334 /* Under rtnl_lock */
11335 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
11336                      int tx_xdp)
11337 {
11338         int max_rx, max_tx, tx_sets = 1;
11339         int tx_rings_needed, stats;
11340         int rx_rings = rx;
11341         int cp, vnics, rc;
11342
11343         if (tcs)
11344                 tx_sets = tcs;
11345
11346         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
11347         if (rc)
11348                 return rc;
11349
11350         if (max_rx < rx)
11351                 return -ENOMEM;
11352
11353         tx_rings_needed = tx * tx_sets + tx_xdp;
11354         if (max_tx < tx_rings_needed)
11355                 return -ENOMEM;
11356
11357         vnics = 1;
11358         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
11359                 vnics += rx_rings;
11360
11361         if (bp->flags & BNXT_FLAG_AGG_RINGS)
11362                 rx_rings <<= 1;
11363         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
11364         stats = cp;
11365         if (BNXT_NEW_RM(bp)) {
11366                 cp += bnxt_get_ulp_msix_num(bp);
11367                 stats += bnxt_get_ulp_stat_ctxs(bp);
11368         }
11369         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
11370                                      stats, vnics);
11371 }
11372
11373 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
11374 {
11375         if (bp->bar2) {
11376                 pci_iounmap(pdev, bp->bar2);
11377                 bp->bar2 = NULL;
11378         }
11379
11380         if (bp->bar1) {
11381                 pci_iounmap(pdev, bp->bar1);
11382                 bp->bar1 = NULL;
11383         }
11384
11385         if (bp->bar0) {
11386                 pci_iounmap(pdev, bp->bar0);
11387                 bp->bar0 = NULL;
11388         }
11389 }
11390
11391 static void bnxt_cleanup_pci(struct bnxt *bp)
11392 {
11393         bnxt_unmap_bars(bp, bp->pdev);
11394         pci_release_regions(bp->pdev);
11395         if (pci_is_enabled(bp->pdev))
11396                 pci_disable_device(bp->pdev);
11397 }
11398
11399 static void bnxt_init_dflt_coal(struct bnxt *bp)
11400 {
11401         struct bnxt_coal *coal;
11402
11403         /* Tick values in micro seconds.
11404          * 1 coal_buf x bufs_per_record = 1 completion record.
11405          */
11406         coal = &bp->rx_coal;
11407         coal->coal_ticks = 10;
11408         coal->coal_bufs = 30;
11409         coal->coal_ticks_irq = 1;
11410         coal->coal_bufs_irq = 2;
11411         coal->idle_thresh = 50;
11412         coal->bufs_per_record = 2;
11413         coal->budget = 64;              /* NAPI budget */
11414
11415         coal = &bp->tx_coal;
11416         coal->coal_ticks = 28;
11417         coal->coal_bufs = 30;
11418         coal->coal_ticks_irq = 2;
11419         coal->coal_bufs_irq = 2;
11420         coal->bufs_per_record = 1;
11421
11422         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
11423 }
11424
11425 static int bnxt_fw_init_one_p1(struct bnxt *bp)
11426 {
11427         int rc;
11428
11429         bp->fw_cap = 0;
11430         rc = bnxt_hwrm_ver_get(bp);
11431         bnxt_try_map_fw_health_reg(bp);
11432         if (rc) {
11433                 rc = bnxt_try_recover_fw(bp);
11434                 if (rc)
11435                         return rc;
11436                 rc = bnxt_hwrm_ver_get(bp);
11437                 if (rc)
11438                         return rc;
11439         }
11440
11441         if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
11442                 rc = bnxt_alloc_kong_hwrm_resources(bp);
11443                 if (rc)
11444                         bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
11445         }
11446
11447         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
11448             bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
11449                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
11450                 if (rc)
11451                         return rc;
11452         }
11453         bnxt_nvm_cfg_ver_get(bp);
11454
11455         rc = bnxt_hwrm_func_reset(bp);
11456         if (rc)
11457                 return -ENODEV;
11458
11459         bnxt_hwrm_fw_set_time(bp);
11460         return 0;
11461 }
11462
11463 static int bnxt_fw_init_one_p2(struct bnxt *bp)
11464 {
11465         int rc;
11466
11467         /* Get the MAX capabilities for this function */
11468         rc = bnxt_hwrm_func_qcaps(bp);
11469         if (rc) {
11470                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
11471                            rc);
11472                 return -ENODEV;
11473         }
11474
11475         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
11476         if (rc)
11477                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
11478                             rc);
11479
11480         if (bnxt_alloc_fw_health(bp)) {
11481                 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
11482         } else {
11483                 rc = bnxt_hwrm_error_recovery_qcfg(bp);
11484                 if (rc)
11485                         netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
11486                                     rc);
11487         }
11488
11489         rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
11490         if (rc)
11491                 return -ENODEV;
11492
11493         bnxt_hwrm_func_qcfg(bp);
11494         bnxt_hwrm_vnic_qcaps(bp);
11495         bnxt_hwrm_port_led_qcaps(bp);
11496         bnxt_ethtool_init(bp);
11497         bnxt_dcb_init(bp);
11498         return 0;
11499 }
11500
11501 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
11502 {
11503         bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
11504         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
11505                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
11506                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
11507                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
11508         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
11509                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
11510                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
11511                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
11512         }
11513 }
11514
11515 static void bnxt_set_dflt_rfs(struct bnxt *bp)
11516 {
11517         struct net_device *dev = bp->dev;
11518
11519         dev->hw_features &= ~NETIF_F_NTUPLE;
11520         dev->features &= ~NETIF_F_NTUPLE;
11521         bp->flags &= ~BNXT_FLAG_RFS;
11522         if (bnxt_rfs_supported(bp)) {
11523                 dev->hw_features |= NETIF_F_NTUPLE;
11524                 if (bnxt_rfs_capable(bp)) {
11525                         bp->flags |= BNXT_FLAG_RFS;
11526                         dev->features |= NETIF_F_NTUPLE;
11527                 }
11528         }
11529 }
11530
11531 static void bnxt_fw_init_one_p3(struct bnxt *bp)
11532 {
11533         struct pci_dev *pdev = bp->pdev;
11534
11535         bnxt_set_dflt_rss_hash_type(bp);
11536         bnxt_set_dflt_rfs(bp);
11537
11538         bnxt_get_wol_settings(bp);
11539         if (bp->flags & BNXT_FLAG_WOL_CAP)
11540                 device_set_wakeup_enable(&pdev->dev, bp->wol);
11541         else
11542                 device_set_wakeup_capable(&pdev->dev, false);
11543
11544         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
11545         bnxt_hwrm_coal_params_qcaps(bp);
11546 }
11547
11548 static int bnxt_fw_init_one(struct bnxt *bp)
11549 {
11550         int rc;
11551
11552         rc = bnxt_fw_init_one_p1(bp);
11553         if (rc) {
11554                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
11555                 return rc;
11556         }
11557         rc = bnxt_fw_init_one_p2(bp);
11558         if (rc) {
11559                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
11560                 return rc;
11561         }
11562         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
11563         if (rc)
11564                 return rc;
11565
11566         /* In case fw capabilities have changed, destroy the unneeded
11567          * reporters and create newly capable ones.
11568          */
11569         bnxt_dl_fw_reporters_destroy(bp, false);
11570         bnxt_dl_fw_reporters_create(bp);
11571         bnxt_fw_init_one_p3(bp);
11572         return 0;
11573 }
11574
11575 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
11576 {
11577         struct bnxt_fw_health *fw_health = bp->fw_health;
11578         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
11579         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
11580         u32 reg_type, reg_off, delay_msecs;
11581
11582         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
11583         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
11584         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
11585         switch (reg_type) {
11586         case BNXT_FW_HEALTH_REG_TYPE_CFG:
11587                 pci_write_config_dword(bp->pdev, reg_off, val);
11588                 break;
11589         case BNXT_FW_HEALTH_REG_TYPE_GRC:
11590                 writel(reg_off & BNXT_GRC_BASE_MASK,
11591                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
11592                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
11593                 fallthrough;
11594         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
11595                 writel(val, bp->bar0 + reg_off);
11596                 break;
11597         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
11598                 writel(val, bp->bar1 + reg_off);
11599                 break;
11600         }
11601         if (delay_msecs) {
11602                 pci_read_config_dword(bp->pdev, 0, &val);
11603                 msleep(delay_msecs);
11604         }
11605 }
11606
11607 static void bnxt_reset_all(struct bnxt *bp)
11608 {
11609         struct bnxt_fw_health *fw_health = bp->fw_health;
11610         int i, rc;
11611
11612         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11613                 bnxt_fw_reset_via_optee(bp);
11614                 bp->fw_reset_timestamp = jiffies;
11615                 return;
11616         }
11617
11618         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
11619                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
11620                         bnxt_fw_reset_writel(bp, i);
11621         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
11622                 struct hwrm_fw_reset_input req = {0};
11623
11624                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
11625                 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
11626                 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
11627                 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
11628                 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
11629                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11630                 if (rc)
11631                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
11632         }
11633         bp->fw_reset_timestamp = jiffies;
11634 }
11635
11636 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
11637 {
11638         return time_after(jiffies, bp->fw_reset_timestamp +
11639                           (bp->fw_reset_max_dsecs * HZ / 10));
11640 }
11641
11642 static void bnxt_fw_reset_task(struct work_struct *work)
11643 {
11644         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
11645         int rc;
11646
11647         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11648                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
11649                 return;
11650         }
11651
11652         switch (bp->fw_reset_state) {
11653         case BNXT_FW_RESET_STATE_POLL_VF: {
11654                 int n = bnxt_get_registered_vfs(bp);
11655                 int tmo;
11656
11657                 if (n < 0) {
11658                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
11659                                    n, jiffies_to_msecs(jiffies -
11660                                    bp->fw_reset_timestamp));
11661                         goto fw_reset_abort;
11662                 } else if (n > 0) {
11663                         if (bnxt_fw_reset_timeout(bp)) {
11664                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11665                                 bp->fw_reset_state = 0;
11666                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
11667                                            n);
11668                                 return;
11669                         }
11670                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11671                         return;
11672                 }
11673                 bp->fw_reset_timestamp = jiffies;
11674                 rtnl_lock();
11675                 bnxt_fw_reset_close(bp);
11676                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11677                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11678                         tmo = HZ / 10;
11679                 } else {
11680                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11681                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11682                 }
11683                 rtnl_unlock();
11684                 bnxt_queue_fw_reset_work(bp, tmo);
11685                 return;
11686         }
11687         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
11688                 u32 val;
11689
11690                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11691                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
11692                     !bnxt_fw_reset_timeout(bp)) {
11693                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11694                         return;
11695                 }
11696
11697                 if (!bp->fw_health->master) {
11698                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
11699
11700                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11701                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11702                         return;
11703                 }
11704                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11705         }
11706                 fallthrough;
11707         case BNXT_FW_RESET_STATE_RESET_FW:
11708                 bnxt_reset_all(bp);
11709                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11710                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
11711                 return;
11712         case BNXT_FW_RESET_STATE_ENABLE_DEV:
11713                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11714                         u32 val;
11715
11716                         if (!bp->fw_reset_min_dsecs) {
11717                                 u16 val;
11718
11719                                 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID,
11720                                                      &val);
11721                                 if (val == 0xffff) {
11722                                         if (bnxt_fw_reset_timeout(bp)) {
11723                                                 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
11724                                                 goto fw_reset_abort;
11725                                         }
11726                                         bnxt_queue_fw_reset_work(bp, HZ / 1000);
11727                                         return;
11728                                 }
11729                         }
11730                         val = bnxt_fw_health_readl(bp,
11731                                                    BNXT_FW_RESET_INPROG_REG);
11732                         if (val)
11733                                 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n",
11734                                             val);
11735                 }
11736                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11737                 if (pci_enable_device(bp->pdev)) {
11738                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
11739                         goto fw_reset_abort;
11740                 }
11741                 pci_set_master(bp->pdev);
11742                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
11743                 fallthrough;
11744         case BNXT_FW_RESET_STATE_POLL_FW:
11745                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
11746                 rc = __bnxt_hwrm_ver_get(bp, true);
11747                 if (rc) {
11748                         if (bnxt_fw_reset_timeout(bp)) {
11749                                 netdev_err(bp->dev, "Firmware reset aborted\n");
11750                                 goto fw_reset_abort_status;
11751                         }
11752                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11753                         return;
11754                 }
11755                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
11756                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
11757                 fallthrough;
11758         case BNXT_FW_RESET_STATE_OPENING:
11759                 while (!rtnl_trylock()) {
11760                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11761                         return;
11762                 }
11763                 rc = bnxt_open(bp->dev);
11764                 if (rc) {
11765                         netdev_err(bp->dev, "bnxt_open_nic() failed\n");
11766                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11767                         dev_close(bp->dev);
11768                 }
11769
11770                 bp->fw_reset_state = 0;
11771                 /* Make sure fw_reset_state is 0 before clearing the flag */
11772                 smp_mb__before_atomic();
11773                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11774                 bnxt_ulp_start(bp, rc);
11775                 if (!rc)
11776                         bnxt_reenable_sriov(bp);
11777                 bnxt_dl_health_recovery_done(bp);
11778                 bnxt_dl_health_status_update(bp, true);
11779                 rtnl_unlock();
11780                 break;
11781         }
11782         return;
11783
11784 fw_reset_abort_status:
11785         if (bp->fw_health->status_reliable ||
11786             (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
11787                 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11788
11789                 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
11790         }
11791 fw_reset_abort:
11792         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11793         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
11794                 bnxt_dl_health_status_update(bp, false);
11795         bp->fw_reset_state = 0;
11796         rtnl_lock();
11797         dev_close(bp->dev);
11798         rtnl_unlock();
11799 }
11800
11801 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
11802 {
11803         int rc;
11804         struct bnxt *bp = netdev_priv(dev);
11805
11806         SET_NETDEV_DEV(dev, &pdev->dev);
11807
11808         /* enable device (incl. PCI PM wakeup), and bus-mastering */
11809         rc = pci_enable_device(pdev);
11810         if (rc) {
11811                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
11812                 goto init_err;
11813         }
11814
11815         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
11816                 dev_err(&pdev->dev,
11817                         "Cannot find PCI device base address, aborting\n");
11818                 rc = -ENODEV;
11819                 goto init_err_disable;
11820         }
11821
11822         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
11823         if (rc) {
11824                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
11825                 goto init_err_disable;
11826         }
11827
11828         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
11829             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
11830                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
11831                 rc = -EIO;
11832                 goto init_err_release;
11833         }
11834
11835         pci_set_master(pdev);
11836
11837         bp->dev = dev;
11838         bp->pdev = pdev;
11839
11840         /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
11841          * determines the BAR size.
11842          */
11843         bp->bar0 = pci_ioremap_bar(pdev, 0);
11844         if (!bp->bar0) {
11845                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
11846                 rc = -ENOMEM;
11847                 goto init_err_release;
11848         }
11849
11850         bp->bar2 = pci_ioremap_bar(pdev, 4);
11851         if (!bp->bar2) {
11852                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
11853                 rc = -ENOMEM;
11854                 goto init_err_release;
11855         }
11856
11857         pci_enable_pcie_error_reporting(pdev);
11858
11859         INIT_WORK(&bp->sp_task, bnxt_sp_task);
11860         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
11861
11862         spin_lock_init(&bp->ntp_fltr_lock);
11863 #if BITS_PER_LONG == 32
11864         spin_lock_init(&bp->db_lock);
11865 #endif
11866
11867         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
11868         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
11869
11870         bnxt_init_dflt_coal(bp);
11871
11872         timer_setup(&bp->timer, bnxt_timer, 0);
11873         bp->current_interval = BNXT_TIMER_INTERVAL;
11874
11875         bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
11876         bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
11877
11878         clear_bit(BNXT_STATE_OPEN, &bp->state);
11879         return 0;
11880
11881 init_err_release:
11882         bnxt_unmap_bars(bp, pdev);
11883         pci_release_regions(pdev);
11884
11885 init_err_disable:
11886         pci_disable_device(pdev);
11887
11888 init_err:
11889         return rc;
11890 }
11891
11892 /* rtnl_lock held */
11893 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
11894 {
11895         struct sockaddr *addr = p;
11896         struct bnxt *bp = netdev_priv(dev);
11897         int rc = 0;
11898
11899         if (!is_valid_ether_addr(addr->sa_data))
11900                 return -EADDRNOTAVAIL;
11901
11902         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
11903                 return 0;
11904
11905         rc = bnxt_approve_mac(bp, addr->sa_data, true);
11906         if (rc)
11907                 return rc;
11908
11909         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
11910         if (netif_running(dev)) {
11911                 bnxt_close_nic(bp, false, false);
11912                 rc = bnxt_open_nic(bp, false, false);
11913         }
11914
11915         return rc;
11916 }
11917
11918 /* rtnl_lock held */
11919 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
11920 {
11921         struct bnxt *bp = netdev_priv(dev);
11922
11923         if (netif_running(dev))
11924                 bnxt_close_nic(bp, true, false);
11925
11926         dev->mtu = new_mtu;
11927         bnxt_set_ring_params(bp);
11928
11929         if (netif_running(dev))
11930                 return bnxt_open_nic(bp, true, false);
11931
11932         return 0;
11933 }
11934
11935 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
11936 {
11937         struct bnxt *bp = netdev_priv(dev);
11938         bool sh = false;
11939         int rc;
11940
11941         if (tc > bp->max_tc) {
11942                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
11943                            tc, bp->max_tc);
11944                 return -EINVAL;
11945         }
11946
11947         if (netdev_get_num_tc(dev) == tc)
11948                 return 0;
11949
11950         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
11951                 sh = true;
11952
11953         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
11954                               sh, tc, bp->tx_nr_rings_xdp);
11955         if (rc)
11956                 return rc;
11957
11958         /* Needs to close the device and do hw resource re-allocations */
11959         if (netif_running(bp->dev))
11960                 bnxt_close_nic(bp, true, false);
11961
11962         if (tc) {
11963                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
11964                 netdev_set_num_tc(dev, tc);
11965         } else {
11966                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11967                 netdev_reset_tc(dev);
11968         }
11969         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
11970         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
11971                                bp->tx_nr_rings + bp->rx_nr_rings;
11972
11973         if (netif_running(bp->dev))
11974                 return bnxt_open_nic(bp, true, false);
11975
11976         return 0;
11977 }
11978
11979 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
11980                                   void *cb_priv)
11981 {
11982         struct bnxt *bp = cb_priv;
11983
11984         if (!bnxt_tc_flower_enabled(bp) ||
11985             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
11986                 return -EOPNOTSUPP;
11987
11988         switch (type) {
11989         case TC_SETUP_CLSFLOWER:
11990                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
11991         default:
11992                 return -EOPNOTSUPP;
11993         }
11994 }
11995
11996 LIST_HEAD(bnxt_block_cb_list);
11997
11998 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
11999                          void *type_data)
12000 {
12001         struct bnxt *bp = netdev_priv(dev);
12002
12003         switch (type) {
12004         case TC_SETUP_BLOCK:
12005                 return flow_block_cb_setup_simple(type_data,
12006                                                   &bnxt_block_cb_list,
12007                                                   bnxt_setup_tc_block_cb,
12008                                                   bp, bp, true);
12009         case TC_SETUP_QDISC_MQPRIO: {
12010                 struct tc_mqprio_qopt *mqprio = type_data;
12011
12012                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
12013
12014                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
12015         }
12016         default:
12017                 return -EOPNOTSUPP;
12018         }
12019 }
12020
12021 #ifdef CONFIG_RFS_ACCEL
12022 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
12023                             struct bnxt_ntuple_filter *f2)
12024 {
12025         struct flow_keys *keys1 = &f1->fkeys;
12026         struct flow_keys *keys2 = &f2->fkeys;
12027
12028         if (keys1->basic.n_proto != keys2->basic.n_proto ||
12029             keys1->basic.ip_proto != keys2->basic.ip_proto)
12030                 return false;
12031
12032         if (keys1->basic.n_proto == htons(ETH_P_IP)) {
12033                 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
12034                     keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
12035                         return false;
12036         } else {
12037                 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
12038                            sizeof(keys1->addrs.v6addrs.src)) ||
12039                     memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
12040                            sizeof(keys1->addrs.v6addrs.dst)))
12041                         return false;
12042         }
12043
12044         if (keys1->ports.ports == keys2->ports.ports &&
12045             keys1->control.flags == keys2->control.flags &&
12046             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
12047             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
12048                 return true;
12049
12050         return false;
12051 }
12052
12053 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
12054                               u16 rxq_index, u32 flow_id)
12055 {
12056         struct bnxt *bp = netdev_priv(dev);
12057         struct bnxt_ntuple_filter *fltr, *new_fltr;
12058         struct flow_keys *fkeys;
12059         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
12060         int rc = 0, idx, bit_id, l2_idx = 0;
12061         struct hlist_head *head;
12062         u32 flags;
12063
12064         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
12065                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12066                 int off = 0, j;
12067
12068                 netif_addr_lock_bh(dev);
12069                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
12070                         if (ether_addr_equal(eth->h_dest,
12071                                              vnic->uc_list + off)) {
12072                                 l2_idx = j + 1;
12073                                 break;
12074                         }
12075                 }
12076                 netif_addr_unlock_bh(dev);
12077                 if (!l2_idx)
12078                         return -EINVAL;
12079         }
12080         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
12081         if (!new_fltr)
12082                 return -ENOMEM;
12083
12084         fkeys = &new_fltr->fkeys;
12085         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
12086                 rc = -EPROTONOSUPPORT;
12087                 goto err_free;
12088         }
12089
12090         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
12091              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
12092             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
12093              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
12094                 rc = -EPROTONOSUPPORT;
12095                 goto err_free;
12096         }
12097         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
12098             bp->hwrm_spec_code < 0x10601) {
12099                 rc = -EPROTONOSUPPORT;
12100                 goto err_free;
12101         }
12102         flags = fkeys->control.flags;
12103         if (((flags & FLOW_DIS_ENCAPSULATION) &&
12104              bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
12105                 rc = -EPROTONOSUPPORT;
12106                 goto err_free;
12107         }
12108
12109         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
12110         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
12111
12112         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
12113         head = &bp->ntp_fltr_hash_tbl[idx];
12114         rcu_read_lock();
12115         hlist_for_each_entry_rcu(fltr, head, hash) {
12116                 if (bnxt_fltr_match(fltr, new_fltr)) {
12117                         rcu_read_unlock();
12118                         rc = 0;
12119                         goto err_free;
12120                 }
12121         }
12122         rcu_read_unlock();
12123
12124         spin_lock_bh(&bp->ntp_fltr_lock);
12125         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
12126                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
12127         if (bit_id < 0) {
12128                 spin_unlock_bh(&bp->ntp_fltr_lock);
12129                 rc = -ENOMEM;
12130                 goto err_free;
12131         }
12132
12133         new_fltr->sw_id = (u16)bit_id;
12134         new_fltr->flow_id = flow_id;
12135         new_fltr->l2_fltr_idx = l2_idx;
12136         new_fltr->rxq = rxq_index;
12137         hlist_add_head_rcu(&new_fltr->hash, head);
12138         bp->ntp_fltr_count++;
12139         spin_unlock_bh(&bp->ntp_fltr_lock);
12140
12141         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
12142         bnxt_queue_sp_work(bp);
12143
12144         return new_fltr->sw_id;
12145
12146 err_free:
12147         kfree(new_fltr);
12148         return rc;
12149 }
12150
12151 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
12152 {
12153         int i;
12154
12155         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
12156                 struct hlist_head *head;
12157                 struct hlist_node *tmp;
12158                 struct bnxt_ntuple_filter *fltr;
12159                 int rc;
12160
12161                 head = &bp->ntp_fltr_hash_tbl[i];
12162                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
12163                         bool del = false;
12164
12165                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
12166                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
12167                                                         fltr->flow_id,
12168                                                         fltr->sw_id)) {
12169                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
12170                                                                          fltr);
12171                                         del = true;
12172                                 }
12173                         } else {
12174                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
12175                                                                        fltr);
12176                                 if (rc)
12177                                         del = true;
12178                                 else
12179                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
12180                         }
12181
12182                         if (del) {
12183                                 spin_lock_bh(&bp->ntp_fltr_lock);
12184                                 hlist_del_rcu(&fltr->hash);
12185                                 bp->ntp_fltr_count--;
12186                                 spin_unlock_bh(&bp->ntp_fltr_lock);
12187                                 synchronize_rcu();
12188                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
12189                                 kfree(fltr);
12190                         }
12191                 }
12192         }
12193         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
12194                 netdev_info(bp->dev, "Receive PF driver unload event!\n");
12195 }
12196
12197 #else
12198
12199 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
12200 {
12201 }
12202
12203 #endif /* CONFIG_RFS_ACCEL */
12204
12205 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
12206 {
12207         struct bnxt *bp = netdev_priv(netdev);
12208         struct udp_tunnel_info ti;
12209         unsigned int cmd;
12210
12211         udp_tunnel_nic_get_port(netdev, table, 0, &ti);
12212         if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
12213                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
12214         else
12215                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
12216
12217         if (ti.port)
12218                 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
12219
12220         return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
12221 }
12222
12223 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
12224         .sync_table     = bnxt_udp_tunnel_sync,
12225         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
12226                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
12227         .tables         = {
12228                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
12229                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
12230         },
12231 };
12232
12233 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
12234                                struct net_device *dev, u32 filter_mask,
12235                                int nlflags)
12236 {
12237         struct bnxt *bp = netdev_priv(dev);
12238
12239         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
12240                                        nlflags, filter_mask, NULL);
12241 }
12242
12243 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
12244                                u16 flags, struct netlink_ext_ack *extack)
12245 {
12246         struct bnxt *bp = netdev_priv(dev);
12247         struct nlattr *attr, *br_spec;
12248         int rem, rc = 0;
12249
12250         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
12251                 return -EOPNOTSUPP;
12252
12253         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12254         if (!br_spec)
12255                 return -EINVAL;
12256
12257         nla_for_each_nested(attr, br_spec, rem) {
12258                 u16 mode;
12259
12260                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12261                         continue;
12262
12263                 if (nla_len(attr) < sizeof(mode))
12264                         return -EINVAL;
12265
12266                 mode = nla_get_u16(attr);
12267                 if (mode == bp->br_mode)
12268                         break;
12269
12270                 rc = bnxt_hwrm_set_br_mode(bp, mode);
12271                 if (!rc)
12272                         bp->br_mode = mode;
12273                 break;
12274         }
12275         return rc;
12276 }
12277
12278 int bnxt_get_port_parent_id(struct net_device *dev,
12279                             struct netdev_phys_item_id *ppid)
12280 {
12281         struct bnxt *bp = netdev_priv(dev);
12282
12283         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
12284                 return -EOPNOTSUPP;
12285
12286         /* The PF and it's VF-reps only support the switchdev framework */
12287         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
12288                 return -EOPNOTSUPP;
12289
12290         ppid->id_len = sizeof(bp->dsn);
12291         memcpy(ppid->id, bp->dsn, ppid->id_len);
12292
12293         return 0;
12294 }
12295
12296 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
12297 {
12298         struct bnxt *bp = netdev_priv(dev);
12299
12300         return &bp->dl_port;
12301 }
12302
12303 static const struct net_device_ops bnxt_netdev_ops = {
12304         .ndo_open               = bnxt_open,
12305         .ndo_start_xmit         = bnxt_start_xmit,
12306         .ndo_stop               = bnxt_close,
12307         .ndo_get_stats64        = bnxt_get_stats64,
12308         .ndo_set_rx_mode        = bnxt_set_rx_mode,
12309         .ndo_do_ioctl           = bnxt_ioctl,
12310         .ndo_validate_addr      = eth_validate_addr,
12311         .ndo_set_mac_address    = bnxt_change_mac_addr,
12312         .ndo_change_mtu         = bnxt_change_mtu,
12313         .ndo_fix_features       = bnxt_fix_features,
12314         .ndo_set_features       = bnxt_set_features,
12315         .ndo_tx_timeout         = bnxt_tx_timeout,
12316 #ifdef CONFIG_BNXT_SRIOV
12317         .ndo_get_vf_config      = bnxt_get_vf_config,
12318         .ndo_set_vf_mac         = bnxt_set_vf_mac,
12319         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
12320         .ndo_set_vf_rate        = bnxt_set_vf_bw,
12321         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
12322         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
12323         .ndo_set_vf_trust       = bnxt_set_vf_trust,
12324 #endif
12325         .ndo_setup_tc           = bnxt_setup_tc,
12326 #ifdef CONFIG_RFS_ACCEL
12327         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
12328 #endif
12329         .ndo_bpf                = bnxt_xdp,
12330         .ndo_xdp_xmit           = bnxt_xdp_xmit,
12331         .ndo_bridge_getlink     = bnxt_bridge_getlink,
12332         .ndo_bridge_setlink     = bnxt_bridge_setlink,
12333         .ndo_get_devlink_port   = bnxt_get_devlink_port,
12334 };
12335
12336 static void bnxt_remove_one(struct pci_dev *pdev)
12337 {
12338         struct net_device *dev = pci_get_drvdata(pdev);
12339         struct bnxt *bp = netdev_priv(dev);
12340
12341         if (BNXT_PF(bp))
12342                 bnxt_sriov_disable(bp);
12343
12344         if (BNXT_PF(bp))
12345                 devlink_port_type_clear(&bp->dl_port);
12346         pci_disable_pcie_error_reporting(pdev);
12347         unregister_netdev(dev);
12348         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12349         /* Flush any pending tasks */
12350         cancel_work_sync(&bp->sp_task);
12351         cancel_delayed_work_sync(&bp->fw_reset_task);
12352         bp->sp_event = 0;
12353
12354         bnxt_dl_fw_reporters_destroy(bp, true);
12355         bnxt_dl_unregister(bp);
12356         bnxt_shutdown_tc(bp);
12357
12358         bnxt_clear_int_mode(bp);
12359         bnxt_hwrm_func_drv_unrgtr(bp);
12360         bnxt_free_hwrm_resources(bp);
12361         bnxt_free_hwrm_short_cmd_req(bp);
12362         bnxt_ethtool_free(bp);
12363         bnxt_dcb_free(bp);
12364         kfree(bp->edev);
12365         bp->edev = NULL;
12366         kfree(bp->fw_health);
12367         bp->fw_health = NULL;
12368         bnxt_cleanup_pci(bp);
12369         bnxt_free_ctx_mem(bp);
12370         kfree(bp->ctx);
12371         bp->ctx = NULL;
12372         kfree(bp->rss_indir_tbl);
12373         bp->rss_indir_tbl = NULL;
12374         bnxt_free_port_stats(bp);
12375         free_netdev(dev);
12376 }
12377
12378 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
12379 {
12380         int rc = 0;
12381         struct bnxt_link_info *link_info = &bp->link_info;
12382
12383         rc = bnxt_hwrm_phy_qcaps(bp);
12384         if (rc) {
12385                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
12386                            rc);
12387                 return rc;
12388         }
12389         if (!fw_dflt)
12390                 return 0;
12391
12392         rc = bnxt_update_link(bp, false);
12393         if (rc) {
12394                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
12395                            rc);
12396                 return rc;
12397         }
12398
12399         /* Older firmware does not have supported_auto_speeds, so assume
12400          * that all supported speeds can be autonegotiated.
12401          */
12402         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
12403                 link_info->support_auto_speeds = link_info->support_speeds;
12404
12405         bnxt_init_ethtool_link_settings(bp);
12406         return 0;
12407 }
12408
12409 static int bnxt_get_max_irq(struct pci_dev *pdev)
12410 {
12411         u16 ctrl;
12412
12413         if (!pdev->msix_cap)
12414                 return 1;
12415
12416         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
12417         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
12418 }
12419
12420 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12421                                 int *max_cp)
12422 {
12423         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
12424         int max_ring_grps = 0, max_irq;
12425
12426         *max_tx = hw_resc->max_tx_rings;
12427         *max_rx = hw_resc->max_rx_rings;
12428         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
12429         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
12430                         bnxt_get_ulp_msix_num(bp),
12431                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
12432         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
12433                 *max_cp = min_t(int, *max_cp, max_irq);
12434         max_ring_grps = hw_resc->max_hw_ring_grps;
12435         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
12436                 *max_cp -= 1;
12437                 *max_rx -= 2;
12438         }
12439         if (bp->flags & BNXT_FLAG_AGG_RINGS)
12440                 *max_rx >>= 1;
12441         if (bp->flags & BNXT_FLAG_CHIP_P5) {
12442                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
12443                 /* On P5 chips, max_cp output param should be available NQs */
12444                 *max_cp = max_irq;
12445         }
12446         *max_rx = min_t(int, *max_rx, max_ring_grps);
12447 }
12448
12449 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
12450 {
12451         int rx, tx, cp;
12452
12453         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
12454         *max_rx = rx;
12455         *max_tx = tx;
12456         if (!rx || !tx || !cp)
12457                 return -ENOMEM;
12458
12459         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
12460 }
12461
12462 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12463                                bool shared)
12464 {
12465         int rc;
12466
12467         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12468         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
12469                 /* Not enough rings, try disabling agg rings. */
12470                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
12471                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12472                 if (rc) {
12473                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
12474                         bp->flags |= BNXT_FLAG_AGG_RINGS;
12475                         return rc;
12476                 }
12477                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
12478                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12479                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12480                 bnxt_set_ring_params(bp);
12481         }
12482
12483         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
12484                 int max_cp, max_stat, max_irq;
12485
12486                 /* Reserve minimum resources for RoCE */
12487                 max_cp = bnxt_get_max_func_cp_rings(bp);
12488                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
12489                 max_irq = bnxt_get_max_func_irqs(bp);
12490                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
12491                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
12492                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
12493                         return 0;
12494
12495                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
12496                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
12497                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
12498                 max_cp = min_t(int, max_cp, max_irq);
12499                 max_cp = min_t(int, max_cp, max_stat);
12500                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
12501                 if (rc)
12502                         rc = 0;
12503         }
12504         return rc;
12505 }
12506
12507 /* In initial default shared ring setting, each shared ring must have a
12508  * RX/TX ring pair.
12509  */
12510 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
12511 {
12512         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
12513         bp->rx_nr_rings = bp->cp_nr_rings;
12514         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
12515         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12516 }
12517
12518 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
12519 {
12520         int dflt_rings, max_rx_rings, max_tx_rings, rc;
12521
12522         if (!bnxt_can_reserve_rings(bp))
12523                 return 0;
12524
12525         if (sh)
12526                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
12527         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
12528         /* Reduce default rings on multi-port cards so that total default
12529          * rings do not exceed CPU count.
12530          */
12531         if (bp->port_count > 1) {
12532                 int max_rings =
12533                         max_t(int, num_online_cpus() / bp->port_count, 1);
12534
12535                 dflt_rings = min_t(int, dflt_rings, max_rings);
12536         }
12537         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
12538         if (rc)
12539                 return rc;
12540         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
12541         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
12542         if (sh)
12543                 bnxt_trim_dflt_sh_rings(bp);
12544         else
12545                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
12546         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12547
12548         rc = __bnxt_reserve_rings(bp);
12549         if (rc)
12550                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
12551         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12552         if (sh)
12553                 bnxt_trim_dflt_sh_rings(bp);
12554
12555         /* Rings may have been trimmed, re-reserve the trimmed rings. */
12556         if (bnxt_need_reserve_rings(bp)) {
12557                 rc = __bnxt_reserve_rings(bp);
12558                 if (rc)
12559                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
12560                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12561         }
12562         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
12563                 bp->rx_nr_rings++;
12564                 bp->cp_nr_rings++;
12565         }
12566         if (rc) {
12567                 bp->tx_nr_rings = 0;
12568                 bp->rx_nr_rings = 0;
12569         }
12570         return rc;
12571 }
12572
12573 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
12574 {
12575         int rc;
12576
12577         if (bp->tx_nr_rings)
12578                 return 0;
12579
12580         bnxt_ulp_irq_stop(bp);
12581         bnxt_clear_int_mode(bp);
12582         rc = bnxt_set_dflt_rings(bp, true);
12583         if (rc) {
12584                 netdev_err(bp->dev, "Not enough rings available.\n");
12585                 goto init_dflt_ring_err;
12586         }
12587         rc = bnxt_init_int_mode(bp);
12588         if (rc)
12589                 goto init_dflt_ring_err;
12590
12591         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12592         if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
12593                 bp->flags |= BNXT_FLAG_RFS;
12594                 bp->dev->features |= NETIF_F_NTUPLE;
12595         }
12596 init_dflt_ring_err:
12597         bnxt_ulp_irq_restart(bp, rc);
12598         return rc;
12599 }
12600
12601 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
12602 {
12603         int rc;
12604
12605         ASSERT_RTNL();
12606         bnxt_hwrm_func_qcaps(bp);
12607
12608         if (netif_running(bp->dev))
12609                 __bnxt_close_nic(bp, true, false);
12610
12611         bnxt_ulp_irq_stop(bp);
12612         bnxt_clear_int_mode(bp);
12613         rc = bnxt_init_int_mode(bp);
12614         bnxt_ulp_irq_restart(bp, rc);
12615
12616         if (netif_running(bp->dev)) {
12617                 if (rc)
12618                         dev_close(bp->dev);
12619                 else
12620                         rc = bnxt_open_nic(bp, true, false);
12621         }
12622
12623         return rc;
12624 }
12625
12626 static int bnxt_init_mac_addr(struct bnxt *bp)
12627 {
12628         int rc = 0;
12629
12630         if (BNXT_PF(bp)) {
12631                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
12632         } else {
12633 #ifdef CONFIG_BNXT_SRIOV
12634                 struct bnxt_vf_info *vf = &bp->vf;
12635                 bool strict_approval = true;
12636
12637                 if (is_valid_ether_addr(vf->mac_addr)) {
12638                         /* overwrite netdev dev_addr with admin VF MAC */
12639                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
12640                         /* Older PF driver or firmware may not approve this
12641                          * correctly.
12642                          */
12643                         strict_approval = false;
12644                 } else {
12645                         eth_hw_addr_random(bp->dev);
12646                 }
12647                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
12648 #endif
12649         }
12650         return rc;
12651 }
12652
12653 #define BNXT_VPD_LEN    512
12654 static void bnxt_vpd_read_info(struct bnxt *bp)
12655 {
12656         struct pci_dev *pdev = bp->pdev;
12657         int i, len, pos, ro_size, size;
12658         ssize_t vpd_size;
12659         u8 *vpd_data;
12660
12661         vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL);
12662         if (!vpd_data)
12663                 return;
12664
12665         vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data);
12666         if (vpd_size <= 0) {
12667                 netdev_err(bp->dev, "Unable to read VPD\n");
12668                 goto exit;
12669         }
12670
12671         i = pci_vpd_find_tag(vpd_data, vpd_size, PCI_VPD_LRDT_RO_DATA);
12672         if (i < 0) {
12673                 netdev_err(bp->dev, "VPD READ-Only not found\n");
12674                 goto exit;
12675         }
12676
12677         ro_size = pci_vpd_lrdt_size(&vpd_data[i]);
12678         i += PCI_VPD_LRDT_TAG_SIZE;
12679         if (i + ro_size > vpd_size)
12680                 goto exit;
12681
12682         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12683                                         PCI_VPD_RO_KEYWORD_PARTNO);
12684         if (pos < 0)
12685                 goto read_sn;
12686
12687         len = pci_vpd_info_field_size(&vpd_data[pos]);
12688         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12689         if (len + pos > vpd_size)
12690                 goto read_sn;
12691
12692         size = min(len, BNXT_VPD_FLD_LEN - 1);
12693         memcpy(bp->board_partno, &vpd_data[pos], size);
12694
12695 read_sn:
12696         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12697                                         PCI_VPD_RO_KEYWORD_SERIALNO);
12698         if (pos < 0)
12699                 goto exit;
12700
12701         len = pci_vpd_info_field_size(&vpd_data[pos]);
12702         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12703         if (len + pos > vpd_size)
12704                 goto exit;
12705
12706         size = min(len, BNXT_VPD_FLD_LEN - 1);
12707         memcpy(bp->board_serialno, &vpd_data[pos], size);
12708 exit:
12709         kfree(vpd_data);
12710 }
12711
12712 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
12713 {
12714         struct pci_dev *pdev = bp->pdev;
12715         u64 qword;
12716
12717         qword = pci_get_dsn(pdev);
12718         if (!qword) {
12719                 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
12720                 return -EOPNOTSUPP;
12721         }
12722
12723         put_unaligned_le64(qword, dsn);
12724
12725         bp->flags |= BNXT_FLAG_DSN_VALID;
12726         return 0;
12727 }
12728
12729 static int bnxt_map_db_bar(struct bnxt *bp)
12730 {
12731         if (!bp->db_size)
12732                 return -ENODEV;
12733         bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
12734         if (!bp->bar1)
12735                 return -ENOMEM;
12736         return 0;
12737 }
12738
12739 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
12740 {
12741         struct net_device *dev;
12742         struct bnxt *bp;
12743         int rc, max_irqs;
12744
12745         if (pci_is_bridge(pdev))
12746                 return -ENODEV;
12747
12748         /* Clear any pending DMA transactions from crash kernel
12749          * while loading driver in capture kernel.
12750          */
12751         if (is_kdump_kernel()) {
12752                 pci_clear_master(pdev);
12753                 pcie_flr(pdev);
12754         }
12755
12756         max_irqs = bnxt_get_max_irq(pdev);
12757         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
12758         if (!dev)
12759                 return -ENOMEM;
12760
12761         bp = netdev_priv(dev);
12762         bp->msg_enable = BNXT_DEF_MSG_ENABLE;
12763         bnxt_set_max_func_irqs(bp, max_irqs);
12764
12765         if (bnxt_vf_pciid(ent->driver_data))
12766                 bp->flags |= BNXT_FLAG_VF;
12767
12768         if (pdev->msix_cap)
12769                 bp->flags |= BNXT_FLAG_MSIX_CAP;
12770
12771         rc = bnxt_init_board(pdev, dev);
12772         if (rc < 0)
12773                 goto init_err_free;
12774
12775         dev->netdev_ops = &bnxt_netdev_ops;
12776         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
12777         dev->ethtool_ops = &bnxt_ethtool_ops;
12778         pci_set_drvdata(pdev, dev);
12779
12780         rc = bnxt_alloc_hwrm_resources(bp);
12781         if (rc)
12782                 goto init_err_pci_clean;
12783
12784         mutex_init(&bp->hwrm_cmd_lock);
12785         mutex_init(&bp->link_lock);
12786
12787         rc = bnxt_fw_init_one_p1(bp);
12788         if (rc)
12789                 goto init_err_pci_clean;
12790
12791         if (BNXT_PF(bp))
12792                 bnxt_vpd_read_info(bp);
12793
12794         if (BNXT_CHIP_P5(bp)) {
12795                 bp->flags |= BNXT_FLAG_CHIP_P5;
12796                 if (BNXT_CHIP_SR2(bp))
12797                         bp->flags |= BNXT_FLAG_CHIP_SR2;
12798         }
12799
12800         rc = bnxt_alloc_rss_indir_tbl(bp);
12801         if (rc)
12802                 goto init_err_pci_clean;
12803
12804         rc = bnxt_fw_init_one_p2(bp);
12805         if (rc)
12806                 goto init_err_pci_clean;
12807
12808         rc = bnxt_map_db_bar(bp);
12809         if (rc) {
12810                 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
12811                         rc);
12812                 goto init_err_pci_clean;
12813         }
12814
12815         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12816                            NETIF_F_TSO | NETIF_F_TSO6 |
12817                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12818                            NETIF_F_GSO_IPXIP4 |
12819                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12820                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
12821                            NETIF_F_RXCSUM | NETIF_F_GRO;
12822
12823         if (BNXT_SUPPORTS_TPA(bp))
12824                 dev->hw_features |= NETIF_F_LRO;
12825
12826         dev->hw_enc_features =
12827                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12828                         NETIF_F_TSO | NETIF_F_TSO6 |
12829                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12830                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12831                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
12832         dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
12833
12834         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
12835                                     NETIF_F_GSO_GRE_CSUM;
12836         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
12837         if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
12838                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12839         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
12840                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
12841         if (BNXT_SUPPORTS_TPA(bp))
12842                 dev->hw_features |= NETIF_F_GRO_HW;
12843         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
12844         if (dev->features & NETIF_F_GRO_HW)
12845                 dev->features &= ~NETIF_F_LRO;
12846         dev->priv_flags |= IFF_UNICAST_FLT;
12847
12848 #ifdef CONFIG_BNXT_SRIOV
12849         init_waitqueue_head(&bp->sriov_cfg_wait);
12850         mutex_init(&bp->sriov_lock);
12851 #endif
12852         if (BNXT_SUPPORTS_TPA(bp)) {
12853                 bp->gro_func = bnxt_gro_func_5730x;
12854                 if (BNXT_CHIP_P4(bp))
12855                         bp->gro_func = bnxt_gro_func_5731x;
12856                 else if (BNXT_CHIP_P5(bp))
12857                         bp->gro_func = bnxt_gro_func_5750x;
12858         }
12859         if (!BNXT_CHIP_P4_PLUS(bp))
12860                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
12861
12862         bp->ulp_probe = bnxt_ulp_probe;
12863
12864         rc = bnxt_init_mac_addr(bp);
12865         if (rc) {
12866                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
12867                 rc = -EADDRNOTAVAIL;
12868                 goto init_err_pci_clean;
12869         }
12870
12871         if (BNXT_PF(bp)) {
12872                 /* Read the adapter's DSN to use as the eswitch switch_id */
12873                 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
12874         }
12875
12876         /* MTU range: 60 - FW defined max */
12877         dev->min_mtu = ETH_ZLEN;
12878         dev->max_mtu = bp->max_mtu;
12879
12880         rc = bnxt_probe_phy(bp, true);
12881         if (rc)
12882                 goto init_err_pci_clean;
12883
12884         bnxt_set_rx_skb_mode(bp, false);
12885         bnxt_set_tpa_flags(bp);
12886         bnxt_set_ring_params(bp);
12887         rc = bnxt_set_dflt_rings(bp, true);
12888         if (rc) {
12889                 netdev_err(bp->dev, "Not enough rings available.\n");
12890                 rc = -ENOMEM;
12891                 goto init_err_pci_clean;
12892         }
12893
12894         bnxt_fw_init_one_p3(bp);
12895
12896         if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12897                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
12898
12899         rc = bnxt_init_int_mode(bp);
12900         if (rc)
12901                 goto init_err_pci_clean;
12902
12903         /* No TC has been set yet and rings may have been trimmed due to
12904          * limited MSIX, so we re-initialize the TX rings per TC.
12905          */
12906         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12907
12908         if (BNXT_PF(bp)) {
12909                 if (!bnxt_pf_wq) {
12910                         bnxt_pf_wq =
12911                                 create_singlethread_workqueue("bnxt_pf_wq");
12912                         if (!bnxt_pf_wq) {
12913                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
12914                                 rc = -ENOMEM;
12915                                 goto init_err_pci_clean;
12916                         }
12917                 }
12918                 rc = bnxt_init_tc(bp);
12919                 if (rc)
12920                         netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
12921                                    rc);
12922         }
12923
12924         bnxt_dl_register(bp);
12925
12926         rc = register_netdev(dev);
12927         if (rc)
12928                 goto init_err_cleanup;
12929
12930         if (BNXT_PF(bp))
12931                 devlink_port_type_eth_set(&bp->dl_port, bp->dev);
12932         bnxt_dl_fw_reporters_create(bp);
12933
12934         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
12935                     board_info[ent->driver_data].name,
12936                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
12937         pcie_print_link_status(pdev);
12938
12939         pci_save_state(pdev);
12940         return 0;
12941
12942 init_err_cleanup:
12943         bnxt_dl_unregister(bp);
12944         bnxt_shutdown_tc(bp);
12945         bnxt_clear_int_mode(bp);
12946
12947 init_err_pci_clean:
12948         bnxt_hwrm_func_drv_unrgtr(bp);
12949         bnxt_free_hwrm_short_cmd_req(bp);
12950         bnxt_free_hwrm_resources(bp);
12951         kfree(bp->fw_health);
12952         bp->fw_health = NULL;
12953         bnxt_cleanup_pci(bp);
12954         bnxt_free_ctx_mem(bp);
12955         kfree(bp->ctx);
12956         bp->ctx = NULL;
12957         kfree(bp->rss_indir_tbl);
12958         bp->rss_indir_tbl = NULL;
12959
12960 init_err_free:
12961         free_netdev(dev);
12962         return rc;
12963 }
12964
12965 static void bnxt_shutdown(struct pci_dev *pdev)
12966 {
12967         struct net_device *dev = pci_get_drvdata(pdev);
12968         struct bnxt *bp;
12969
12970         if (!dev)
12971                 return;
12972
12973         rtnl_lock();
12974         bp = netdev_priv(dev);
12975         if (!bp)
12976                 goto shutdown_exit;
12977
12978         if (netif_running(dev))
12979                 dev_close(dev);
12980
12981         bnxt_ulp_shutdown(bp);
12982         bnxt_clear_int_mode(bp);
12983         pci_disable_device(pdev);
12984
12985         if (system_state == SYSTEM_POWER_OFF) {
12986                 pci_wake_from_d3(pdev, bp->wol);
12987                 pci_set_power_state(pdev, PCI_D3hot);
12988         }
12989
12990 shutdown_exit:
12991         rtnl_unlock();
12992 }
12993
12994 #ifdef CONFIG_PM_SLEEP
12995 static int bnxt_suspend(struct device *device)
12996 {
12997         struct net_device *dev = dev_get_drvdata(device);
12998         struct bnxt *bp = netdev_priv(dev);
12999         int rc = 0;
13000
13001         rtnl_lock();
13002         bnxt_ulp_stop(bp);
13003         if (netif_running(dev)) {
13004                 netif_device_detach(dev);
13005                 rc = bnxt_close(dev);
13006         }
13007         bnxt_hwrm_func_drv_unrgtr(bp);
13008         pci_disable_device(bp->pdev);
13009         bnxt_free_ctx_mem(bp);
13010         kfree(bp->ctx);
13011         bp->ctx = NULL;
13012         rtnl_unlock();
13013         return rc;
13014 }
13015
13016 static int bnxt_resume(struct device *device)
13017 {
13018         struct net_device *dev = dev_get_drvdata(device);
13019         struct bnxt *bp = netdev_priv(dev);
13020         int rc = 0;
13021
13022         rtnl_lock();
13023         rc = pci_enable_device(bp->pdev);
13024         if (rc) {
13025                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
13026                            rc);
13027                 goto resume_exit;
13028         }
13029         pci_set_master(bp->pdev);
13030         if (bnxt_hwrm_ver_get(bp)) {
13031                 rc = -ENODEV;
13032                 goto resume_exit;
13033         }
13034         rc = bnxt_hwrm_func_reset(bp);
13035         if (rc) {
13036                 rc = -EBUSY;
13037                 goto resume_exit;
13038         }
13039
13040         rc = bnxt_hwrm_func_qcaps(bp);
13041         if (rc)
13042                 goto resume_exit;
13043
13044         if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
13045                 rc = -ENODEV;
13046                 goto resume_exit;
13047         }
13048
13049         bnxt_get_wol_settings(bp);
13050         if (netif_running(dev)) {
13051                 rc = bnxt_open(dev);
13052                 if (!rc)
13053                         netif_device_attach(dev);
13054         }
13055
13056 resume_exit:
13057         bnxt_ulp_start(bp, rc);
13058         if (!rc)
13059                 bnxt_reenable_sriov(bp);
13060         rtnl_unlock();
13061         return rc;
13062 }
13063
13064 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
13065 #define BNXT_PM_OPS (&bnxt_pm_ops)
13066
13067 #else
13068
13069 #define BNXT_PM_OPS NULL
13070
13071 #endif /* CONFIG_PM_SLEEP */
13072
13073 /**
13074  * bnxt_io_error_detected - called when PCI error is detected
13075  * @pdev: Pointer to PCI device
13076  * @state: The current pci connection state
13077  *
13078  * This function is called after a PCI bus error affecting
13079  * this device has been detected.
13080  */
13081 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
13082                                                pci_channel_state_t state)
13083 {
13084         struct net_device *netdev = pci_get_drvdata(pdev);
13085         struct bnxt *bp = netdev_priv(netdev);
13086
13087         netdev_info(netdev, "PCI I/O error detected\n");
13088
13089         rtnl_lock();
13090         netif_device_detach(netdev);
13091
13092         bnxt_ulp_stop(bp);
13093
13094         if (state == pci_channel_io_perm_failure) {
13095                 rtnl_unlock();
13096                 return PCI_ERS_RESULT_DISCONNECT;
13097         }
13098
13099         if (state == pci_channel_io_frozen)
13100                 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
13101
13102         if (netif_running(netdev))
13103                 bnxt_close(netdev);
13104
13105         pci_disable_device(pdev);
13106         bnxt_free_ctx_mem(bp);
13107         kfree(bp->ctx);
13108         bp->ctx = NULL;
13109         rtnl_unlock();
13110
13111         /* Request a slot slot reset. */
13112         return PCI_ERS_RESULT_NEED_RESET;
13113 }
13114
13115 /**
13116  * bnxt_io_slot_reset - called after the pci bus has been reset.
13117  * @pdev: Pointer to PCI device
13118  *
13119  * Restart the card from scratch, as if from a cold-boot.
13120  * At this point, the card has exprienced a hard reset,
13121  * followed by fixups by BIOS, and has its config space
13122  * set up identically to what it was at cold boot.
13123  */
13124 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
13125 {
13126         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
13127         struct net_device *netdev = pci_get_drvdata(pdev);
13128         struct bnxt *bp = netdev_priv(netdev);
13129         int err = 0, off;
13130
13131         netdev_info(bp->dev, "PCI Slot Reset\n");
13132
13133         rtnl_lock();
13134
13135         if (pci_enable_device(pdev)) {
13136                 dev_err(&pdev->dev,
13137                         "Cannot re-enable PCI device after reset.\n");
13138         } else {
13139                 pci_set_master(pdev);
13140                 /* Upon fatal error, our device internal logic that latches to
13141                  * BAR value is getting reset and will restore only upon
13142                  * rewritting the BARs.
13143                  *
13144                  * As pci_restore_state() does not re-write the BARs if the
13145                  * value is same as saved value earlier, driver needs to
13146                  * write the BARs to 0 to force restore, in case of fatal error.
13147                  */
13148                 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
13149                                        &bp->state)) {
13150                         for (off = PCI_BASE_ADDRESS_0;
13151                              off <= PCI_BASE_ADDRESS_5; off += 4)
13152                                 pci_write_config_dword(bp->pdev, off, 0);
13153                 }
13154                 pci_restore_state(pdev);
13155                 pci_save_state(pdev);
13156
13157                 err = bnxt_hwrm_func_reset(bp);
13158                 if (!err)
13159                         result = PCI_ERS_RESULT_RECOVERED;
13160         }
13161
13162         rtnl_unlock();
13163
13164         return result;
13165 }
13166
13167 /**
13168  * bnxt_io_resume - called when traffic can start flowing again.
13169  * @pdev: Pointer to PCI device
13170  *
13171  * This callback is called when the error recovery driver tells
13172  * us that its OK to resume normal operation.
13173  */
13174 static void bnxt_io_resume(struct pci_dev *pdev)
13175 {
13176         struct net_device *netdev = pci_get_drvdata(pdev);
13177         struct bnxt *bp = netdev_priv(netdev);
13178         int err;
13179
13180         netdev_info(bp->dev, "PCI Slot Resume\n");
13181         rtnl_lock();
13182
13183         err = bnxt_hwrm_func_qcaps(bp);
13184         if (!err && netif_running(netdev))
13185                 err = bnxt_open(netdev);
13186
13187         bnxt_ulp_start(bp, err);
13188         if (!err) {
13189                 bnxt_reenable_sriov(bp);
13190                 netif_device_attach(netdev);
13191         }
13192
13193         rtnl_unlock();
13194 }
13195
13196 static const struct pci_error_handlers bnxt_err_handler = {
13197         .error_detected = bnxt_io_error_detected,
13198         .slot_reset     = bnxt_io_slot_reset,
13199         .resume         = bnxt_io_resume
13200 };
13201
13202 static struct pci_driver bnxt_pci_driver = {
13203         .name           = DRV_MODULE_NAME,
13204         .id_table       = bnxt_pci_tbl,
13205         .probe          = bnxt_init_one,
13206         .remove         = bnxt_remove_one,
13207         .shutdown       = bnxt_shutdown,
13208         .driver.pm      = BNXT_PM_OPS,
13209         .err_handler    = &bnxt_err_handler,
13210 #if defined(CONFIG_BNXT_SRIOV)
13211         .sriov_configure = bnxt_sriov_configure,
13212 #endif
13213 };
13214
13215 static int __init bnxt_init(void)
13216 {
13217         bnxt_debug_init();
13218         return pci_register_driver(&bnxt_pci_driver);
13219 }
13220
13221 static void __exit bnxt_exit(void)
13222 {
13223         pci_unregister_driver(&bnxt_pci_driver);
13224         if (bnxt_pf_wq)
13225                 destroy_workqueue(bnxt_pf_wq);
13226         bnxt_debug_exit();
13227 }
13228
13229 module_init(bnxt_init);
13230 module_exit(bnxt_exit);