drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/ip.h>
  41 #include <net/tcp.h>
  42 #include <net/udp.h>
  43 #include <net/checksum.h>
  44 #include <net/ip6_checksum.h>
  45 #include <net/udp_tunnel.h>
  46 #include <linux/workqueue.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/cache.h>
  49 #include <linux/log2.h>
  50 #include <linux/aer.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53 #include <linux/cpumask.h>
  54 #include <net/pkt_cls.h>
  55 #include <linux/hwmon.h>
  56 #include <linux/hwmon-sysfs.h>
  57 #include <net/page_pool.h>
  58
  59 #include "bnxt_hsi.h"
  60 #include "bnxt.h"
  61 #include "bnxt_ulp.h"
  62 #include "bnxt_sriov.h"
  63 #include "bnxt_ethtool.h"
  64 #include "bnxt_dcb.h"
  65 #include "bnxt_xdp.h"
  66 #include "bnxt_vfr.h"
  67 #include "bnxt_tc.h"
  68 #include "bnxt_devlink.h"
  69 #include "bnxt_debugfs.h"
  70
  71 #define BNXT_TX_TIMEOUT         (5 * HZ)
  72
  73 MODULE_LICENSE("GPL");
  74 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  75
  76 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  77 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  78 #define BNXT_RX_COPY_THRESH 256
  79
  80 #define BNXT_TX_PUSH_THRESH 164
  81
  82 enum board_idx {
  83         BCM57301,
  84         BCM57302,
  85         BCM57304,
  86         BCM57417_NPAR,
  87         BCM58700,
  88         BCM57311,
  89         BCM57312,
  90         BCM57402,
  91         BCM57404,
  92         BCM57406,
  93         BCM57402_NPAR,
  94         BCM57407,
  95         BCM57412,
  96         BCM57414,
  97         BCM57416,
  98         BCM57417,
  99         BCM57412_NPAR,
 100         BCM57314,
 101         BCM57417_SFP,
 102         BCM57416_SFP,
 103         BCM57404_NPAR,
 104         BCM57406_NPAR,
 105         BCM57407_SFP,
 106         BCM57407_NPAR,
 107         BCM57414_NPAR,
 108         BCM57416_NPAR,
 109         BCM57452,
 110         BCM57454,
 111         BCM5745x_NPAR,
 112         BCM57508,
 113         BCM57504,
 114         BCM57502,
 115         BCM57508_NPAR,
 116         BCM57504_NPAR,
 117         BCM57502_NPAR,
 118         BCM58802,
 119         BCM58804,
 120         BCM58808,
 121         NETXTREME_E_VF,
 122         NETXTREME_C_VF,
 123         NETXTREME_S_VF,
 124         NETXTREME_E_P5_VF,
 125 };
 126
 127 /* indexed by enum above */
 128 static const struct {
 129         char *name;
 130 } board_info[] = {
 131         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 132         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 133         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 134         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 135         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 136         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 137         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 138         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 139         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 140         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 141         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 142         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 143         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 144         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 145         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 146         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 147         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 148         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 149         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 150         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 151         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 152         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 153         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 154         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 155         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 156         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 157         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 158         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 159         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 160         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 161         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 162         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 163         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 164         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 165         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 166         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 167         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 168         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 169         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 170         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 171         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 172         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 173 };
 174
 175 static const struct pci_device_id bnxt_pci_tbl[] = {
 176         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 177         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 178         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 179         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 180         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 181         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 182         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 183         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 184         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 185         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 186         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 187         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 188         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 189         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 190         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 191         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 192         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 193         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 194         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 195         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 196         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 197         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 198         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 199         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 200         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 201         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 202         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 203         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 204         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 205         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 206         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 207         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 208         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 209         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 210         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 211         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 212         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 213         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 214         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
 215         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 216         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
 217         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
 218         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 219         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
 220         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 221         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 222 #ifdef CONFIG_BNXT_SRIOV
 223         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 224         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 225         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 226         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 227         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 228         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 229         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 230         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 231         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 232         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 233         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 234 #endif
 235         { 0 }
 236 };
 237
 238 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 239
 240 static const u16 bnxt_vf_req_snif[] = {
 241         HWRM_FUNC_CFG,
 242         HWRM_FUNC_VF_CFG,
 243         HWRM_PORT_PHY_QCFG,
 244         HWRM_CFA_L2_FILTER_ALLOC,
 245 };
 246
 247 static const u16 bnxt_async_events_arr[] = {
 248         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 249         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
 250         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 251         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 252         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 253         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 254         ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
 255         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 256         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 257 };
 258
 259 static struct workqueue_struct *bnxt_pf_wq;
 260
 261 static bool bnxt_vf_pciid(enum board_idx idx)
 262 {
 263         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 264                 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
 265 }
 266
 267 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 268 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 269 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 270
 271 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 272                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 273
 274 #define BNXT_DB_CQ(db, idx)                                             \
 275         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 276
 277 #define BNXT_DB_NQ_P5(db, idx)                                          \
 278         writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
 279
 280 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 281         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 282
 283 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 284         writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
 285
 286 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 287 {
 288         if (bp->flags & BNXT_FLAG_CHIP_P5)
 289                 BNXT_DB_NQ_P5(db, idx);
 290         else
 291                 BNXT_DB_CQ(db, idx);
 292 }
 293
 294 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 295 {
 296         if (bp->flags & BNXT_FLAG_CHIP_P5)
 297                 BNXT_DB_NQ_ARM_P5(db, idx);
 298         else
 299                 BNXT_DB_CQ_ARM(db, idx);
 300 }
 301
 302 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 303 {
 304         if (bp->flags & BNXT_FLAG_CHIP_P5)
 305                 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
 306                        db->doorbell);
 307         else
 308                 BNXT_DB_CQ(db, idx);
 309 }
 310
 311 const u16 bnxt_lhint_arr[] = {
 312         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 313         TX_BD_FLAGS_LHINT_512_TO_1023,
 314         TX_BD_FLAGS_LHINT_1024_TO_2047,
 315         TX_BD_FLAGS_LHINT_1024_TO_2047,
 316         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 317         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 318         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 319         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 320         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 321         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 322         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 323         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 324         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 325         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 326         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 327         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 328         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 329         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 330         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 331 };
 332
 333 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 334 {
 335         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 336
 337         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 338                 return 0;
 339
 340         return md_dst->u.port_info.port_id;
 341 }
 342
 343 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 344 {
 345         struct bnxt *bp = netdev_priv(dev);
 346         struct tx_bd *txbd;
 347         struct tx_bd_ext *txbd1;
 348         struct netdev_queue *txq;
 349         int i;
 350         dma_addr_t mapping;
 351         unsigned int length, pad = 0;
 352         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 353         u16 prod, last_frag;
 354         struct pci_dev *pdev = bp->pdev;
 355         struct bnxt_tx_ring_info *txr;
 356         struct bnxt_sw_tx_bd *tx_buf;
 357
 358         i = skb_get_queue_mapping(skb);
 359         if (unlikely(i >= bp->tx_nr_rings)) {
 360                 dev_kfree_skb_any(skb);
 361                 return NETDEV_TX_OK;
 362         }
 363
 364         txq = netdev_get_tx_queue(dev, i);
 365         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 366         prod = txr->tx_prod;
 367
 368         free_size = bnxt_tx_avail(bp, txr);
 369         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 370                 netif_tx_stop_queue(txq);
 371                 return NETDEV_TX_BUSY;
 372         }
 373
 374         length = skb->len;
 375         len = skb_headlen(skb);
 376         last_frag = skb_shinfo(skb)->nr_frags;
 377
 378         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 379
 380         txbd->tx_bd_opaque = prod;
 381
 382         tx_buf = &txr->tx_buf_ring[prod];
 383         tx_buf->skb = skb;
 384         tx_buf->nr_frags = last_frag;
 385
 386         vlan_tag_flags = 0;
 387         cfa_action = bnxt_xmit_get_cfa_action(skb);
 388         if (skb_vlan_tag_present(skb)) {
 389                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 390                                  skb_vlan_tag_get(skb);
 391                 /* Currently supports 8021Q, 8021AD vlan offloads
 392                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 393                  */
 394                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 395                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 396         }
 397
 398         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 399                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 400                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 401                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 402                 void __iomem *db = txr->tx_db.doorbell;
 403                 void *pdata = tx_push_buf->data;
 404                 u64 *end;
 405                 int j, push_len;
 406
 407                 /* Set COAL_NOW to be ready quickly for the next push */
 408                 tx_push->tx_bd_len_flags_type =
 409                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 410                                         TX_BD_TYPE_LONG_TX_BD |
 411                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 412                                         TX_BD_FLAGS_COAL_NOW |
 413                                         TX_BD_FLAGS_PACKET_END |
 414                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 415
 416                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 417                         tx_push1->tx_bd_hsize_lflags =
 418                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 419                 else
 420                         tx_push1->tx_bd_hsize_lflags = 0;
 421
 422                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 423                 tx_push1->tx_bd_cfa_action =
 424                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 425
 426                 end = pdata + length;
 427                 end = PTR_ALIGN(end, 8) - 1;
 428                 *end = 0;
 429
 430                 skb_copy_from_linear_data(skb, pdata, len);
 431                 pdata += len;
 432                 for (j = 0; j < last_frag; j++) {
 433                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 434                         void *fptr;
 435
 436                         fptr = skb_frag_address_safe(frag);
 437                         if (!fptr)
 438                                 goto normal_tx;
 439
 440                         memcpy(pdata, fptr, skb_frag_size(frag));
 441                         pdata += skb_frag_size(frag);
 442                 }
 443
 444                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 445                 txbd->tx_bd_haddr = txr->data_mapping;
 446                 prod = NEXT_TX(prod);
 447                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 448                 memcpy(txbd, tx_push1, sizeof(*txbd));
 449                 prod = NEXT_TX(prod);
 450                 tx_push->doorbell =
 451                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 452                 txr->tx_prod = prod;
 453
 454                 tx_buf->is_push = 1;
 455                 netdev_tx_sent_queue(txq, skb->len);
 456                 wmb();  /* Sync is_push and byte queue before pushing data */
 457
 458                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 459                 if (push_len > 16) {
 460                         __iowrite64_copy(db, tx_push_buf, 16);
 461                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 462                                          (push_len - 16) << 1);
 463                 } else {
 464                         __iowrite64_copy(db, tx_push_buf, push_len);
 465                 }
 466
 467                 goto tx_done;
 468         }
 469
 470 normal_tx:
 471         if (length < BNXT_MIN_PKT_SIZE) {
 472                 pad = BNXT_MIN_PKT_SIZE - length;
 473                 if (skb_pad(skb, pad)) {
 474                         /* SKB already freed. */
 475                         tx_buf->skb = NULL;
 476                         return NETDEV_TX_OK;
 477                 }
 478                 length = BNXT_MIN_PKT_SIZE;
 479         }
 480
 481         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 482
 483         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 484                 dev_kfree_skb_any(skb);
 485                 tx_buf->skb = NULL;
 486                 return NETDEV_TX_OK;
 487         }
 488
 489         dma_unmap_addr_set(tx_buf, mapping, mapping);
 490         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 491                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 492
 493         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 494
 495         prod = NEXT_TX(prod);
 496         txbd1 = (struct tx_bd_ext *)
 497                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 498
 499         txbd1->tx_bd_hsize_lflags = 0;
 500         if (skb_is_gso(skb)) {
 501                 u32 hdr_len;
 502
 503                 if (skb->encapsulation)
 504                         hdr_len = skb_inner_network_offset(skb) +
 505                                 skb_inner_network_header_len(skb) +
 506                                 inner_tcp_hdrlen(skb);
 507                 else
 508                         hdr_len = skb_transport_offset(skb) +
 509                                 tcp_hdrlen(skb);
 510
 511                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 512                                         TX_BD_FLAGS_T_IPID |
 513                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 514                 length = skb_shinfo(skb)->gso_size;
 515                 txbd1->tx_bd_mss = cpu_to_le32(length);
 516                 length += hdr_len;
 517         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 518                 txbd1->tx_bd_hsize_lflags =
 519                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 520                 txbd1->tx_bd_mss = 0;
 521         }
 522
 523         length >>= 9;
 524         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 525                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 526                                      skb->len);
 527                 i = 0;
 528                 goto tx_dma_error;
 529         }
 530         flags |= bnxt_lhint_arr[length];
 531         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 532
 533         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 534         txbd1->tx_bd_cfa_action =
 535                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 536         for (i = 0; i < last_frag; i++) {
 537                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 538
 539                 prod = NEXT_TX(prod);
 540                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 541
 542                 len = skb_frag_size(frag);
 543                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 544                                            DMA_TO_DEVICE);
 545
 546                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 547                         goto tx_dma_error;
 548
 549                 tx_buf = &txr->tx_buf_ring[prod];
 550                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 551
 552                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 553
 554                 flags = len << TX_BD_LEN_SHIFT;
 555                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 556         }
 557
 558         flags &= ~TX_BD_LEN;
 559         txbd->tx_bd_len_flags_type =
 560                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 561                             TX_BD_FLAGS_PACKET_END);
 562
 563         netdev_tx_sent_queue(txq, skb->len);
 564
 565         /* Sync BD data before updating doorbell */
 566         wmb();
 567
 568         prod = NEXT_TX(prod);
 569         txr->tx_prod = prod;
 570
 571         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 572                 bnxt_db_write(bp, &txr->tx_db, prod);
 573
 574 tx_done:
 575
 576         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 577                 if (netdev_xmit_more() && !tx_buf->is_push)
 578                         bnxt_db_write(bp, &txr->tx_db, prod);
 579
 580                 netif_tx_stop_queue(txq);
 581
 582                 /* netif_tx_stop_queue() must be done before checking
 583                  * tx index in bnxt_tx_avail() below, because in
 584                  * bnxt_tx_int(), we update tx index before checking for
 585                  * netif_tx_queue_stopped().
 586                  */
 587                 smp_mb();
 588                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 589                         netif_tx_wake_queue(txq);
 590         }
 591         return NETDEV_TX_OK;
 592
 593 tx_dma_error:
 594         last_frag = i;
 595
 596         /* start back at beginning and unmap skb */
 597         prod = txr->tx_prod;
 598         tx_buf = &txr->tx_buf_ring[prod];
 599         tx_buf->skb = NULL;
 600         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 601                          skb_headlen(skb), PCI_DMA_TODEVICE);
 602         prod = NEXT_TX(prod);
 603
 604         /* unmap remaining mapped pages */
 605         for (i = 0; i < last_frag; i++) {
 606                 prod = NEXT_TX(prod);
 607                 tx_buf = &txr->tx_buf_ring[prod];
 608                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 609                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 610                                PCI_DMA_TODEVICE);
 611         }
 612
 613         dev_kfree_skb_any(skb);
 614         return NETDEV_TX_OK;
 615 }
 616
 617 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 618 {
 619         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 620         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 621         u16 cons = txr->tx_cons;
 622         struct pci_dev *pdev = bp->pdev;
 623         int i;
 624         unsigned int tx_bytes = 0;
 625
 626         for (i = 0; i < nr_pkts; i++) {
 627                 struct bnxt_sw_tx_bd *tx_buf;
 628                 struct sk_buff *skb;
 629                 int j, last;
 630
 631                 tx_buf = &txr->tx_buf_ring[cons];
 632                 cons = NEXT_TX(cons);
 633                 skb = tx_buf->skb;
 634                 tx_buf->skb = NULL;
 635
 636                 if (tx_buf->is_push) {
 637                         tx_buf->is_push = 0;
 638                         goto next_tx_int;
 639                 }
 640
 641                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 642                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 643                 last = tx_buf->nr_frags;
 644
 645                 for (j = 0; j < last; j++) {
 646                         cons = NEXT_TX(cons);
 647                         tx_buf = &txr->tx_buf_ring[cons];
 648                         dma_unmap_page(
 649                                 &pdev->dev,
 650                                 dma_unmap_addr(tx_buf, mapping),
 651                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 652                                 PCI_DMA_TODEVICE);
 653                 }
 654
 655 next_tx_int:
 656                 cons = NEXT_TX(cons);
 657
 658                 tx_bytes += skb->len;
 659                 dev_kfree_skb_any(skb);
 660         }
 661
 662         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 663         txr->tx_cons = cons;
 664
 665         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 666          * before checking for netif_tx_queue_stopped().  Without the
 667          * memory barrier, there is a small possibility that bnxt_start_xmit()
 668          * will miss it and cause the queue to be stopped forever.
 669          */
 670         smp_mb();
 671
 672         if (unlikely(netif_tx_queue_stopped(txq)) &&
 673             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 674                 __netif_tx_lock(txq, smp_processor_id());
 675                 if (netif_tx_queue_stopped(txq) &&
 676                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 677                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 678                         netif_tx_wake_queue(txq);
 679                 __netif_tx_unlock(txq);
 680         }
 681 }
 682
 683 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 684                                          struct bnxt_rx_ring_info *rxr,
 685                                          gfp_t gfp)
 686 {
 687         struct device *dev = &bp->pdev->dev;
 688         struct page *page;
 689
 690         page = page_pool_dev_alloc_pages(rxr->page_pool);
 691         if (!page)
 692                 return NULL;
 693
 694         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 695                                       DMA_ATTR_WEAK_ORDERING);
 696         if (dma_mapping_error(dev, *mapping)) {
 697                 page_pool_recycle_direct(rxr->page_pool, page);
 698                 return NULL;
 699         }
 700         *mapping += bp->rx_dma_offset;
 701         return page;
 702 }
 703
 704 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 705                                        gfp_t gfp)
 706 {
 707         u8 *data;
 708         struct pci_dev *pdev = bp->pdev;
 709
 710         data = kmalloc(bp->rx_buf_size, gfp);
 711         if (!data)
 712                 return NULL;
 713
 714         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 715                                         bp->rx_buf_use_size, bp->rx_dir,
 716                                         DMA_ATTR_WEAK_ORDERING);
 717
 718         if (dma_mapping_error(&pdev->dev, *mapping)) {
 719                 kfree(data);
 720                 data = NULL;
 721         }
 722         return data;
 723 }
 724
 725 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 726                        u16 prod, gfp_t gfp)
 727 {
 728         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 729         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 730         dma_addr_t mapping;
 731
 732         if (BNXT_RX_PAGE_MODE(bp)) {
 733                 struct page *page =
 734                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 735
 736                 if (!page)
 737                         return -ENOMEM;
 738
 739                 rx_buf->data = page;
 740                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 741         } else {
 742                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 743
 744                 if (!data)
 745                         return -ENOMEM;
 746
 747                 rx_buf->data = data;
 748                 rx_buf->data_ptr = data + bp->rx_offset;
 749         }
 750         rx_buf->mapping = mapping;
 751
 752         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 753         return 0;
 754 }
 755
 756 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 757 {
 758         u16 prod = rxr->rx_prod;
 759         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 760         struct rx_bd *cons_bd, *prod_bd;
 761
 762         prod_rx_buf = &rxr->rx_buf_ring[prod];
 763         cons_rx_buf = &rxr->rx_buf_ring[cons];
 764
 765         prod_rx_buf->data = data;
 766         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 767
 768         prod_rx_buf->mapping = cons_rx_buf->mapping;
 769
 770         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 771         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 772
 773         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 774 }
 775
 776 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 777 {
 778         u16 next, max = rxr->rx_agg_bmap_size;
 779
 780         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 781         if (next >= max)
 782                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 783         return next;
 784 }
 785
 786 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 787                                      struct bnxt_rx_ring_info *rxr,
 788                                      u16 prod, gfp_t gfp)
 789 {
 790         struct rx_bd *rxbd =
 791                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 792         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 793         struct pci_dev *pdev = bp->pdev;
 794         struct page *page;
 795         dma_addr_t mapping;
 796         u16 sw_prod = rxr->rx_sw_agg_prod;
 797         unsigned int offset = 0;
 798
 799         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 800                 page = rxr->rx_page;
 801                 if (!page) {
 802                         page = alloc_page(gfp);
 803                         if (!page)
 804                                 return -ENOMEM;
 805                         rxr->rx_page = page;
 806                         rxr->rx_page_offset = 0;
 807                 }
 808                 offset = rxr->rx_page_offset;
 809                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 810                 if (rxr->rx_page_offset == PAGE_SIZE)
 811                         rxr->rx_page = NULL;
 812                 else
 813                         get_page(page);
 814         } else {
 815                 page = alloc_page(gfp);
 816                 if (!page)
 817                         return -ENOMEM;
 818         }
 819
 820         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 821                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 822                                      DMA_ATTR_WEAK_ORDERING);
 823         if (dma_mapping_error(&pdev->dev, mapping)) {
 824                 __free_page(page);
 825                 return -EIO;
 826         }
 827
 828         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 829                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 830
 831         __set_bit(sw_prod, rxr->rx_agg_bmap);
 832         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 833         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 834
 835         rx_agg_buf->page = page;
 836         rx_agg_buf->offset = offset;
 837         rx_agg_buf->mapping = mapping;
 838         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 839         rxbd->rx_bd_opaque = sw_prod;
 840         return 0;
 841 }
 842
 843 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 844                                        struct bnxt_cp_ring_info *cpr,
 845                                        u16 cp_cons, u16 curr)
 846 {
 847         struct rx_agg_cmp *agg;
 848
 849         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 850         agg = (struct rx_agg_cmp *)
 851                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 852         return agg;
 853 }
 854
 855 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 856                                               struct bnxt_rx_ring_info *rxr,
 857                                               u16 agg_id, u16 curr)
 858 {
 859         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 860
 861         return &tpa_info->agg_arr[curr];
 862 }
 863
 864 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 865                                    u16 start, u32 agg_bufs, bool tpa)
 866 {
 867         struct bnxt_napi *bnapi = cpr->bnapi;
 868         struct bnxt *bp = bnapi->bp;
 869         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 870         u16 prod = rxr->rx_agg_prod;
 871         u16 sw_prod = rxr->rx_sw_agg_prod;
 872         bool p5_tpa = false;
 873         u32 i;
 874
 875         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 876                 p5_tpa = true;
 877
 878         for (i = 0; i < agg_bufs; i++) {
 879                 u16 cons;
 880                 struct rx_agg_cmp *agg;
 881                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 882                 struct rx_bd *prod_bd;
 883                 struct page *page;
 884
 885                 if (p5_tpa)
 886                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 887                 else
 888                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 889                 cons = agg->rx_agg_cmp_opaque;
 890                 __clear_bit(cons, rxr->rx_agg_bmap);
 891
 892                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 893                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 894
 895                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 896                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 897                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 898
 899                 /* It is possible for sw_prod to be equal to cons, so
 900                  * set cons_rx_buf->page to NULL first.
 901                  */
 902                 page = cons_rx_buf->page;
 903                 cons_rx_buf->page = NULL;
 904                 prod_rx_buf->page = page;
 905                 prod_rx_buf->offset = cons_rx_buf->offset;
 906
 907                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 908
 909                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 910
 911                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 912                 prod_bd->rx_bd_opaque = sw_prod;
 913
 914                 prod = NEXT_RX_AGG(prod);
 915                 sw_prod = NEXT_RX_AGG(sw_prod);
 916         }
 917         rxr->rx_agg_prod = prod;
 918         rxr->rx_sw_agg_prod = sw_prod;
 919 }
 920
 921 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 922                                         struct bnxt_rx_ring_info *rxr,
 923                                         u16 cons, void *data, u8 *data_ptr,
 924                                         dma_addr_t dma_addr,
 925                                         unsigned int offset_and_len)
 926 {
 927         unsigned int payload = offset_and_len >> 16;
 928         unsigned int len = offset_and_len & 0xffff;
 929         skb_frag_t *frag;
 930         struct page *page = data;
 931         u16 prod = rxr->rx_prod;
 932         struct sk_buff *skb;
 933         int off, err;
 934
 935         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 936         if (unlikely(err)) {
 937                 bnxt_reuse_rx_data(rxr, cons, data);
 938                 return NULL;
 939         }
 940         dma_addr -= bp->rx_dma_offset;
 941         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 942                              DMA_ATTR_WEAK_ORDERING);
 943         page_pool_release_page(rxr->page_pool, page);
 944
 945         if (unlikely(!payload))
 946                 payload = eth_get_headlen(bp->dev, data_ptr, len);
 947
 948         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 949         if (!skb) {
 950                 __free_page(page);
 951                 return NULL;
 952         }
 953
 954         off = (void *)data_ptr - page_address(page);
 955         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 956         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 957                payload + NET_IP_ALIGN);
 958
 959         frag = &skb_shinfo(skb)->frags[0];
 960         skb_frag_size_sub(frag, payload);
 961         skb_frag_off_add(frag, payload);
 962         skb->data_len -= payload;
 963         skb->tail += payload;
 964
 965         return skb;
 966 }
 967
 968 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 969                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 970                                    void *data, u8 *data_ptr,
 971                                    dma_addr_t dma_addr,
 972                                    unsigned int offset_and_len)
 973 {
 974         u16 prod = rxr->rx_prod;
 975         struct sk_buff *skb;
 976         int err;
 977
 978         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 979         if (unlikely(err)) {
 980                 bnxt_reuse_rx_data(rxr, cons, data);
 981                 return NULL;
 982         }
 983
 984         skb = build_skb(data, 0);
 985         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 986                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 987         if (!skb) {
 988                 kfree(data);
 989                 return NULL;
 990         }
 991
 992         skb_reserve(skb, bp->rx_offset);
 993         skb_put(skb, offset_and_len & 0xffff);
 994         return skb;
 995 }
 996
 997 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
 998                                      struct bnxt_cp_ring_info *cpr,
 999                                      struct sk_buff *skb, u16 idx,
1000                                      u32 agg_bufs, bool tpa)
1001 {
1002         struct bnxt_napi *bnapi = cpr->bnapi;
1003         struct pci_dev *pdev = bp->pdev;
1004         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1005         u16 prod = rxr->rx_agg_prod;
1006         bool p5_tpa = false;
1007         u32 i;
1008
1009         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1010                 p5_tpa = true;
1011
1012         for (i = 0; i < agg_bufs; i++) {
1013                 u16 cons, frag_len;
1014                 struct rx_agg_cmp *agg;
1015                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1016                 struct page *page;
1017                 dma_addr_t mapping;
1018
1019                 if (p5_tpa)
1020                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1021                 else
1022                         agg = bnxt_get_agg(bp, cpr, idx, i);
1023                 cons = agg->rx_agg_cmp_opaque;
1024                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1025                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1026
1027                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1028                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1029                                    cons_rx_buf->offset, frag_len);
1030                 __clear_bit(cons, rxr->rx_agg_bmap);
1031
1032                 /* It is possible for bnxt_alloc_rx_page() to allocate
1033                  * a sw_prod index that equals the cons index, so we
1034                  * need to clear the cons entry now.
1035                  */
1036                 mapping = cons_rx_buf->mapping;
1037                 page = cons_rx_buf->page;
1038                 cons_rx_buf->page = NULL;
1039
1040                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1041                         struct skb_shared_info *shinfo;
1042                         unsigned int nr_frags;
1043
1044                         shinfo = skb_shinfo(skb);
1045                         nr_frags = --shinfo->nr_frags;
1046                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1047
1048                         dev_kfree_skb(skb);
1049
1050                         cons_rx_buf->page = page;
1051
1052                         /* Update prod since possibly some pages have been
1053                          * allocated already.
1054                          */
1055                         rxr->rx_agg_prod = prod;
1056                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1057                         return NULL;
1058                 }
1059
1060                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1061                                      PCI_DMA_FROMDEVICE,
1062                                      DMA_ATTR_WEAK_ORDERING);
1063
1064                 skb->data_len += frag_len;
1065                 skb->len += frag_len;
1066                 skb->truesize += PAGE_SIZE;
1067
1068                 prod = NEXT_RX_AGG(prod);
1069         }
1070         rxr->rx_agg_prod = prod;
1071         return skb;
1072 }
1073
1074 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1075                                u8 agg_bufs, u32 *raw_cons)
1076 {
1077         u16 last;
1078         struct rx_agg_cmp *agg;
1079
1080         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1081         last = RING_CMP(*raw_cons);
1082         agg = (struct rx_agg_cmp *)
1083                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1084         return RX_AGG_CMP_VALID(agg, *raw_cons);
1085 }
1086
1087 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1088                                             unsigned int len,
1089                                             dma_addr_t mapping)
1090 {
1091         struct bnxt *bp = bnapi->bp;
1092         struct pci_dev *pdev = bp->pdev;
1093         struct sk_buff *skb;
1094
1095         skb = napi_alloc_skb(&bnapi->napi, len);
1096         if (!skb)
1097                 return NULL;
1098
1099         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1100                                 bp->rx_dir);
1101
1102         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1103                len + NET_IP_ALIGN);
1104
1105         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1106                                    bp->rx_dir);
1107
1108         skb_put(skb, len);
1109         return skb;
1110 }
1111
1112 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1113                            u32 *raw_cons, void *cmp)
1114 {
1115         struct rx_cmp *rxcmp = cmp;
1116         u32 tmp_raw_cons = *raw_cons;
1117         u8 cmp_type, agg_bufs = 0;
1118
1119         cmp_type = RX_CMP_TYPE(rxcmp);
1120
1121         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1122                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1123                             RX_CMP_AGG_BUFS) >>
1124                            RX_CMP_AGG_BUFS_SHIFT;
1125         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1126                 struct rx_tpa_end_cmp *tpa_end = cmp;
1127
1128                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1129                         return 0;
1130
1131                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1132         }
1133
1134         if (agg_bufs) {
1135                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1136                         return -EBUSY;
1137         }
1138         *raw_cons = tmp_raw_cons;
1139         return 0;
1140 }
1141
1142 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1143 {
1144         if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1145                 return;
1146
1147         if (BNXT_PF(bp))
1148                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1149         else
1150                 schedule_delayed_work(&bp->fw_reset_task, delay);
1151 }
1152
1153 static void bnxt_queue_sp_work(struct bnxt *bp)
1154 {
1155         if (BNXT_PF(bp))
1156                 queue_work(bnxt_pf_wq, &bp->sp_task);
1157         else
1158                 schedule_work(&bp->sp_task);
1159 }
1160
1161 static void bnxt_cancel_sp_work(struct bnxt *bp)
1162 {
1163         if (BNXT_PF(bp)) {
1164                 flush_workqueue(bnxt_pf_wq);
1165         } else {
1166                 cancel_work_sync(&bp->sp_task);
1167                 cancel_delayed_work_sync(&bp->fw_reset_task);
1168         }
1169 }
1170
1171 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1172 {
1173         if (!rxr->bnapi->in_reset) {
1174                 rxr->bnapi->in_reset = true;
1175                 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1176                 bnxt_queue_sp_work(bp);
1177         }
1178         rxr->rx_next_cons = 0xffff;
1179 }
1180
1181 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1182 {
1183         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1184         u16 idx = agg_id & MAX_TPA_P5_MASK;
1185
1186         if (test_bit(idx, map->agg_idx_bmap))
1187                 idx = find_first_zero_bit(map->agg_idx_bmap,
1188                                           BNXT_AGG_IDX_BMAP_SIZE);
1189         __set_bit(idx, map->agg_idx_bmap);
1190         map->agg_id_tbl[agg_id] = idx;
1191         return idx;
1192 }
1193
1194 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1195 {
1196         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1197
1198         __clear_bit(idx, map->agg_idx_bmap);
1199 }
1200
1201 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1202 {
1203         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1204
1205         return map->agg_id_tbl[agg_id];
1206 }
1207
1208 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1209                            struct rx_tpa_start_cmp *tpa_start,
1210                            struct rx_tpa_start_cmp_ext *tpa_start1)
1211 {
1212         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1213         struct bnxt_tpa_info *tpa_info;
1214         u16 cons, prod, agg_id;
1215         struct rx_bd *prod_bd;
1216         dma_addr_t mapping;
1217
1218         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1219                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1220                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1221         } else {
1222                 agg_id = TPA_START_AGG_ID(tpa_start);
1223         }
1224         cons = tpa_start->rx_tpa_start_cmp_opaque;
1225         prod = rxr->rx_prod;
1226         cons_rx_buf = &rxr->rx_buf_ring[cons];
1227         prod_rx_buf = &rxr->rx_buf_ring[prod];
1228         tpa_info = &rxr->rx_tpa[agg_id];
1229
1230         if (unlikely(cons != rxr->rx_next_cons ||
1231                      TPA_START_ERROR(tpa_start))) {
1232                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1233                             cons, rxr->rx_next_cons,
1234                             TPA_START_ERROR_CODE(tpa_start1));
1235                 bnxt_sched_reset(bp, rxr);
1236                 return;
1237         }
1238         /* Store cfa_code in tpa_info to use in tpa_end
1239          * completion processing.
1240          */
1241         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1242         prod_rx_buf->data = tpa_info->data;
1243         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1244
1245         mapping = tpa_info->mapping;
1246         prod_rx_buf->mapping = mapping;
1247
1248         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1249
1250         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1251
1252         tpa_info->data = cons_rx_buf->data;
1253         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1254         cons_rx_buf->data = NULL;
1255         tpa_info->mapping = cons_rx_buf->mapping;
1256
1257         tpa_info->len =
1258                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1259                                 RX_TPA_START_CMP_LEN_SHIFT;
1260         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1261                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1262
1263                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1264                 tpa_info->gso_type = SKB_GSO_TCPV4;
1265                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1266                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1267                         tpa_info->gso_type = SKB_GSO_TCPV6;
1268                 tpa_info->rss_hash =
1269                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1270         } else {
1271                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1272                 tpa_info->gso_type = 0;
1273                 if (netif_msg_rx_err(bp))
1274                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1275         }
1276         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1277         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1278         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1279         tpa_info->agg_count = 0;
1280
1281         rxr->rx_prod = NEXT_RX(prod);
1282         cons = NEXT_RX(cons);
1283         rxr->rx_next_cons = NEXT_RX(cons);
1284         cons_rx_buf = &rxr->rx_buf_ring[cons];
1285
1286         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1287         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1288         cons_rx_buf->data = NULL;
1289 }
1290
1291 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1292 {
1293         if (agg_bufs)
1294                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1295 }
1296
1297 #ifdef CONFIG_INET
1298 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1299 {
1300         struct udphdr *uh = NULL;
1301
1302         if (ip_proto == htons(ETH_P_IP)) {
1303                 struct iphdr *iph = (struct iphdr *)skb->data;
1304
1305                 if (iph->protocol == IPPROTO_UDP)
1306                         uh = (struct udphdr *)(iph + 1);
1307         } else {
1308                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1309
1310                 if (iph->nexthdr == IPPROTO_UDP)
1311                         uh = (struct udphdr *)(iph + 1);
1312         }
1313         if (uh) {
1314                 if (uh->check)
1315                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1316                 else
1317                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1318         }
1319 }
1320 #endif
1321
1322 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1323                                            int payload_off, int tcp_ts,
1324                                            struct sk_buff *skb)
1325 {
1326 #ifdef CONFIG_INET
1327         struct tcphdr *th;
1328         int len, nw_off;
1329         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1330         u32 hdr_info = tpa_info->hdr_info;
1331         bool loopback = false;
1332
1333         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1334         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1335         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1336
1337         /* If the packet is an internal loopback packet, the offsets will
1338          * have an extra 4 bytes.
1339          */
1340         if (inner_mac_off == 4) {
1341                 loopback = true;
1342         } else if (inner_mac_off > 4) {
1343                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1344                                             ETH_HLEN - 2));
1345
1346                 /* We only support inner iPv4/ipv6.  If we don't see the
1347                  * correct protocol ID, it must be a loopback packet where
1348                  * the offsets are off by 4.
1349                  */
1350                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1351                         loopback = true;
1352         }
1353         if (loopback) {
1354                 /* internal loopback packet, subtract all offsets by 4 */
1355                 inner_ip_off -= 4;
1356                 inner_mac_off -= 4;
1357                 outer_ip_off -= 4;
1358         }
1359
1360         nw_off = inner_ip_off - ETH_HLEN;
1361         skb_set_network_header(skb, nw_off);
1362         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1363                 struct ipv6hdr *iph = ipv6_hdr(skb);
1364
1365                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1366                 len = skb->len - skb_transport_offset(skb);
1367                 th = tcp_hdr(skb);
1368                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1369         } else {
1370                 struct iphdr *iph = ip_hdr(skb);
1371
1372                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1373                 len = skb->len - skb_transport_offset(skb);
1374                 th = tcp_hdr(skb);
1375                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1376         }
1377
1378         if (inner_mac_off) { /* tunnel */
1379                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1380                                             ETH_HLEN - 2));
1381
1382                 bnxt_gro_tunnel(skb, proto);
1383         }
1384 #endif
1385         return skb;
1386 }
1387
1388 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1389                                            int payload_off, int tcp_ts,
1390                                            struct sk_buff *skb)
1391 {
1392 #ifdef CONFIG_INET
1393         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1394         u32 hdr_info = tpa_info->hdr_info;
1395         int iphdr_len, nw_off;
1396
1397         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1398         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1399         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1400
1401         nw_off = inner_ip_off - ETH_HLEN;
1402         skb_set_network_header(skb, nw_off);
1403         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1404                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1405         skb_set_transport_header(skb, nw_off + iphdr_len);
1406
1407         if (inner_mac_off) { /* tunnel */
1408                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1409                                             ETH_HLEN - 2));
1410
1411                 bnxt_gro_tunnel(skb, proto);
1412         }
1413 #endif
1414         return skb;
1415 }
1416
1417 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1418 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1419
1420 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1421                                            int payload_off, int tcp_ts,
1422                                            struct sk_buff *skb)
1423 {
1424 #ifdef CONFIG_INET
1425         struct tcphdr *th;
1426         int len, nw_off, tcp_opt_len = 0;
1427
1428         if (tcp_ts)
1429                 tcp_opt_len = 12;
1430
1431         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1432                 struct iphdr *iph;
1433
1434                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1435                          ETH_HLEN;
1436                 skb_set_network_header(skb, nw_off);
1437                 iph = ip_hdr(skb);
1438                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1439                 len = skb->len - skb_transport_offset(skb);
1440                 th = tcp_hdr(skb);
1441                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1442         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1443                 struct ipv6hdr *iph;
1444
1445                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1446                          ETH_HLEN;
1447                 skb_set_network_header(skb, nw_off);
1448                 iph = ipv6_hdr(skb);
1449                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1450                 len = skb->len - skb_transport_offset(skb);
1451                 th = tcp_hdr(skb);
1452                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1453         } else {
1454                 dev_kfree_skb_any(skb);
1455                 return NULL;
1456         }
1457
1458         if (nw_off) /* tunnel */
1459                 bnxt_gro_tunnel(skb, skb->protocol);
1460 #endif
1461         return skb;
1462 }
1463
1464 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1465                                            struct bnxt_tpa_info *tpa_info,
1466                                            struct rx_tpa_end_cmp *tpa_end,
1467                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1468                                            struct sk_buff *skb)
1469 {
1470 #ifdef CONFIG_INET
1471         int payload_off;
1472         u16 segs;
1473
1474         segs = TPA_END_TPA_SEGS(tpa_end);
1475         if (segs == 1)
1476                 return skb;
1477
1478         NAPI_GRO_CB(skb)->count = segs;
1479         skb_shinfo(skb)->gso_size =
1480                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1481         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1482         if (bp->flags & BNXT_FLAG_CHIP_P5)
1483                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1484         else
1485                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1486         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1487         if (likely(skb))
1488                 tcp_gro_complete(skb);
1489 #endif
1490         return skb;
1491 }
1492
1493 /* Given the cfa_code of a received packet determine which
1494  * netdev (vf-rep or PF) the packet is destined to.
1495  */
1496 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1497 {
1498         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1499
1500         /* if vf-rep dev is NULL, the must belongs to the PF */
1501         return dev ? dev : bp->dev;
1502 }
1503
1504 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1505                                            struct bnxt_cp_ring_info *cpr,
1506                                            u32 *raw_cons,
1507                                            struct rx_tpa_end_cmp *tpa_end,
1508                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1509                                            u8 *event)
1510 {
1511         struct bnxt_napi *bnapi = cpr->bnapi;
1512         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1513         u8 *data_ptr, agg_bufs;
1514         unsigned int len;
1515         struct bnxt_tpa_info *tpa_info;
1516         dma_addr_t mapping;
1517         struct sk_buff *skb;
1518         u16 idx = 0, agg_id;
1519         void *data;
1520         bool gro;
1521
1522         if (unlikely(bnapi->in_reset)) {
1523                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1524
1525                 if (rc < 0)
1526                         return ERR_PTR(-EBUSY);
1527                 return NULL;
1528         }
1529
1530         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1531                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1532                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1533                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1534                 tpa_info = &rxr->rx_tpa[agg_id];
1535                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1536                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1537                                     agg_bufs, tpa_info->agg_count);
1538                         agg_bufs = tpa_info->agg_count;
1539                 }
1540                 tpa_info->agg_count = 0;
1541                 *event |= BNXT_AGG_EVENT;
1542                 bnxt_free_agg_idx(rxr, agg_id);
1543                 idx = agg_id;
1544                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1545         } else {
1546                 agg_id = TPA_END_AGG_ID(tpa_end);
1547                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1548                 tpa_info = &rxr->rx_tpa[agg_id];
1549                 idx = RING_CMP(*raw_cons);
1550                 if (agg_bufs) {
1551                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1552                                 return ERR_PTR(-EBUSY);
1553
1554                         *event |= BNXT_AGG_EVENT;
1555                         idx = NEXT_CMP(idx);
1556                 }
1557                 gro = !!TPA_END_GRO(tpa_end);
1558         }
1559         data = tpa_info->data;
1560         data_ptr = tpa_info->data_ptr;
1561         prefetch(data_ptr);
1562         len = tpa_info->len;
1563         mapping = tpa_info->mapping;
1564
1565         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1566                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1567                 if (agg_bufs > MAX_SKB_FRAGS)
1568                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1569                                     agg_bufs, (int)MAX_SKB_FRAGS);
1570                 return NULL;
1571         }
1572
1573         if (len <= bp->rx_copy_thresh) {
1574                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1575                 if (!skb) {
1576                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1577                         return NULL;
1578                 }
1579         } else {
1580                 u8 *new_data;
1581                 dma_addr_t new_mapping;
1582
1583                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1584                 if (!new_data) {
1585                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1586                         return NULL;
1587                 }
1588
1589                 tpa_info->data = new_data;
1590                 tpa_info->data_ptr = new_data + bp->rx_offset;
1591                 tpa_info->mapping = new_mapping;
1592
1593                 skb = build_skb(data, 0);
1594                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1595                                        bp->rx_buf_use_size, bp->rx_dir,
1596                                        DMA_ATTR_WEAK_ORDERING);
1597
1598                 if (!skb) {
1599                         kfree(data);
1600                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1601                         return NULL;
1602                 }
1603                 skb_reserve(skb, bp->rx_offset);
1604                 skb_put(skb, len);
1605         }
1606
1607         if (agg_bufs) {
1608                 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1609                 if (!skb) {
1610                         /* Page reuse already handled by bnxt_rx_pages(). */
1611                         return NULL;
1612                 }
1613         }
1614
1615         skb->protocol =
1616                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1617
1618         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1619                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1620
1621         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1622             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1623                 u16 vlan_proto = tpa_info->metadata >>
1624                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1625                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1626
1627                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1628         }
1629
1630         skb_checksum_none_assert(skb);
1631         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1632                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1633                 skb->csum_level =
1634                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1635         }
1636
1637         if (gro)
1638                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1639
1640         return skb;
1641 }
1642
1643 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1644                          struct rx_agg_cmp *rx_agg)
1645 {
1646         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1647         struct bnxt_tpa_info *tpa_info;
1648
1649         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1650         tpa_info = &rxr->rx_tpa[agg_id];
1651         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1652         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1653 }
1654
1655 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1656                              struct sk_buff *skb)
1657 {
1658         if (skb->dev != bp->dev) {
1659                 /* this packet belongs to a vf-rep */
1660                 bnxt_vf_rep_rx(bp, skb);
1661                 return;
1662         }
1663         skb_record_rx_queue(skb, bnapi->index);
1664         napi_gro_receive(&bnapi->napi, skb);
1665 }
1666
1667 /* returns the following:
1668  * 1       - 1 packet successfully received
1669  * 0       - successful TPA_START, packet not completed yet
1670  * -EBUSY  - completion ring does not have all the agg buffers yet
1671  * -ENOMEM - packet aborted due to out of memory
1672  * -EIO    - packet aborted due to hw error indicated in BD
1673  */
1674 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1675                        u32 *raw_cons, u8 *event)
1676 {
1677         struct bnxt_napi *bnapi = cpr->bnapi;
1678         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1679         struct net_device *dev = bp->dev;
1680         struct rx_cmp *rxcmp;
1681         struct rx_cmp_ext *rxcmp1;
1682         u32 tmp_raw_cons = *raw_cons;
1683         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1684         struct bnxt_sw_rx_bd *rx_buf;
1685         unsigned int len;
1686         u8 *data_ptr, agg_bufs, cmp_type;
1687         dma_addr_t dma_addr;
1688         struct sk_buff *skb;
1689         void *data;
1690         int rc = 0;
1691         u32 misc;
1692
1693         rxcmp = (struct rx_cmp *)
1694                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1695
1696         cmp_type = RX_CMP_TYPE(rxcmp);
1697
1698         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1699                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1700                 goto next_rx_no_prod_no_len;
1701         }
1702
1703         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1704         cp_cons = RING_CMP(tmp_raw_cons);
1705         rxcmp1 = (struct rx_cmp_ext *)
1706                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1707
1708         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1709                 return -EBUSY;
1710
1711         prod = rxr->rx_prod;
1712
1713         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1714                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1715                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1716
1717                 *event |= BNXT_RX_EVENT;
1718                 goto next_rx_no_prod_no_len;
1719
1720         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1721                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1722                                    (struct rx_tpa_end_cmp *)rxcmp,
1723                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1724
1725                 if (IS_ERR(skb))
1726                         return -EBUSY;
1727
1728                 rc = -ENOMEM;
1729                 if (likely(skb)) {
1730                         bnxt_deliver_skb(bp, bnapi, skb);
1731                         rc = 1;
1732                 }
1733                 *event |= BNXT_RX_EVENT;
1734                 goto next_rx_no_prod_no_len;
1735         }
1736
1737         cons = rxcmp->rx_cmp_opaque;
1738         if (unlikely(cons != rxr->rx_next_cons)) {
1739                 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1740
1741                 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1742                             cons, rxr->rx_next_cons);
1743                 bnxt_sched_reset(bp, rxr);
1744                 return rc1;
1745         }
1746         rx_buf = &rxr->rx_buf_ring[cons];
1747         data = rx_buf->data;
1748         data_ptr = rx_buf->data_ptr;
1749         prefetch(data_ptr);
1750
1751         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1752         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1753
1754         if (agg_bufs) {
1755                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1756                         return -EBUSY;
1757
1758                 cp_cons = NEXT_CMP(cp_cons);
1759                 *event |= BNXT_AGG_EVENT;
1760         }
1761         *event |= BNXT_RX_EVENT;
1762
1763         rx_buf->data = NULL;
1764         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1765                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1766
1767                 bnxt_reuse_rx_data(rxr, cons, data);
1768                 if (agg_bufs)
1769                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1770                                                false);
1771
1772                 rc = -EIO;
1773                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1774                         bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1775                         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
1776                                 netdev_warn(bp->dev, "RX buffer error %x\n",
1777                                             rx_err);
1778                                 bnxt_sched_reset(bp, rxr);
1779                         }
1780                 }
1781                 goto next_rx_no_len;
1782         }
1783
1784         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1785         dma_addr = rx_buf->mapping;
1786
1787         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1788                 rc = 1;
1789                 goto next_rx;
1790         }
1791
1792         if (len <= bp->rx_copy_thresh) {
1793                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1794                 bnxt_reuse_rx_data(rxr, cons, data);
1795                 if (!skb) {
1796                         if (agg_bufs)
1797                                 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1798                                                        agg_bufs, false);
1799                         rc = -ENOMEM;
1800                         goto next_rx;
1801                 }
1802         } else {
1803                 u32 payload;
1804
1805                 if (rx_buf->data_ptr == data_ptr)
1806                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1807                 else
1808                         payload = 0;
1809                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1810                                       payload | len);
1811                 if (!skb) {
1812                         rc = -ENOMEM;
1813                         goto next_rx;
1814                 }
1815         }
1816
1817         if (agg_bufs) {
1818                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1819                 if (!skb) {
1820                         rc = -ENOMEM;
1821                         goto next_rx;
1822                 }
1823         }
1824
1825         if (RX_CMP_HASH_VALID(rxcmp)) {
1826                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1827                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1828
1829                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1830                 if (hash_type != 1 && hash_type != 3)
1831                         type = PKT_HASH_TYPE_L3;
1832                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1833         }
1834
1835         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1836         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1837
1838         if ((rxcmp1->rx_cmp_flags2 &
1839              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1840             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1841                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1842                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1843                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1844
1845                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1846         }
1847
1848         skb_checksum_none_assert(skb);
1849         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1850                 if (dev->features & NETIF_F_RXCSUM) {
1851                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1852                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1853                 }
1854         } else {
1855                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1856                         if (dev->features & NETIF_F_RXCSUM)
1857                                 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
1858                 }
1859         }
1860
1861         bnxt_deliver_skb(bp, bnapi, skb);
1862         rc = 1;
1863
1864 next_rx:
1865         cpr->rx_packets += 1;
1866         cpr->rx_bytes += len;
1867
1868 next_rx_no_len:
1869         rxr->rx_prod = NEXT_RX(prod);
1870         rxr->rx_next_cons = NEXT_RX(cons);
1871
1872 next_rx_no_prod_no_len:
1873         *raw_cons = tmp_raw_cons;
1874
1875         return rc;
1876 }
1877
1878 /* In netpoll mode, if we are using a combined completion ring, we need to
1879  * discard the rx packets and recycle the buffers.
1880  */
1881 static int bnxt_force_rx_discard(struct bnxt *bp,
1882                                  struct bnxt_cp_ring_info *cpr,
1883                                  u32 *raw_cons, u8 *event)
1884 {
1885         u32 tmp_raw_cons = *raw_cons;
1886         struct rx_cmp_ext *rxcmp1;
1887         struct rx_cmp *rxcmp;
1888         u16 cp_cons;
1889         u8 cmp_type;
1890
1891         cp_cons = RING_CMP(tmp_raw_cons);
1892         rxcmp = (struct rx_cmp *)
1893                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1894
1895         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1896         cp_cons = RING_CMP(tmp_raw_cons);
1897         rxcmp1 = (struct rx_cmp_ext *)
1898                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1899
1900         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1901                 return -EBUSY;
1902
1903         cmp_type = RX_CMP_TYPE(rxcmp);
1904         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1905                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1906                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1907         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1908                 struct rx_tpa_end_cmp_ext *tpa_end1;
1909
1910                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1911                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1912                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1913         }
1914         return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1915 }
1916
1917 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1918 {
1919         struct bnxt_fw_health *fw_health = bp->fw_health;
1920         u32 reg = fw_health->regs[reg_idx];
1921         u32 reg_type, reg_off, val = 0;
1922
1923         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1924         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1925         switch (reg_type) {
1926         case BNXT_FW_HEALTH_REG_TYPE_CFG:
1927                 pci_read_config_dword(bp->pdev, reg_off, &val);
1928                 break;
1929         case BNXT_FW_HEALTH_REG_TYPE_GRC:
1930                 reg_off = fw_health->mapped_regs[reg_idx];
1931                 fallthrough;
1932         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1933                 val = readl(bp->bar0 + reg_off);
1934                 break;
1935         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1936                 val = readl(bp->bar1 + reg_off);
1937                 break;
1938         }
1939         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1940                 val &= fw_health->fw_reset_inprog_reg_mask;
1941         return val;
1942 }
1943
1944 #define BNXT_GET_EVENT_PORT(data)       \
1945         ((data) &                       \
1946          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1947
1948 static int bnxt_async_event_process(struct bnxt *bp,
1949                                     struct hwrm_async_event_cmpl *cmpl)
1950 {
1951         u16 event_id = le16_to_cpu(cmpl->event_id);
1952
1953         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1954         switch (event_id) {
1955         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1956                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1957                 struct bnxt_link_info *link_info = &bp->link_info;
1958
1959                 if (BNXT_VF(bp))
1960                         goto async_event_process_exit;
1961
1962                 /* print unsupported speed warning in forced speed mode only */
1963                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1964                     (data1 & 0x20000)) {
1965                         u16 fw_speed = link_info->force_link_speed;
1966                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1967
1968                         if (speed != SPEED_UNKNOWN)
1969                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1970                                             speed);
1971                 }
1972                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1973         }
1974                 fallthrough;
1975         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
1976         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
1977                 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
1978                 fallthrough;
1979         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1980                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1981                 break;
1982         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1983                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1984                 break;
1985         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1986                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1987                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1988
1989                 if (BNXT_VF(bp))
1990                         break;
1991
1992                 if (bp->pf.port_id != port_id)
1993                         break;
1994
1995                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1996                 break;
1997         }
1998         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1999                 if (BNXT_PF(bp))
2000                         goto async_event_process_exit;
2001                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2002                 break;
2003         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2004                 u32 data1 = le32_to_cpu(cmpl->event_data1);
2005
2006                 if (!bp->fw_health)
2007                         goto async_event_process_exit;
2008
2009                 bp->fw_reset_timestamp = jiffies;
2010                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2011                 if (!bp->fw_reset_min_dsecs)
2012                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2013                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2014                 if (!bp->fw_reset_max_dsecs)
2015                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2016                 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2017                         netdev_warn(bp->dev, "Firmware fatal reset event received\n");
2018                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2019                 } else {
2020                         netdev_warn(bp->dev, "Firmware non-fatal reset event received, max wait time %d msec\n",
2021                                     bp->fw_reset_max_dsecs * 100);
2022                 }
2023                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2024                 break;
2025         }
2026         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2027                 struct bnxt_fw_health *fw_health = bp->fw_health;
2028                 u32 data1 = le32_to_cpu(cmpl->event_data1);
2029
2030                 if (!fw_health)
2031                         goto async_event_process_exit;
2032
2033                 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2034                 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2035                 if (!fw_health->enabled)
2036                         break;
2037
2038                 if (netif_msg_drv(bp))
2039                         netdev_info(bp->dev, "Error recovery info: error recovery[%d], master[%d], reset count[0x%x], health status: 0x%x\n",
2040                                     fw_health->enabled, fw_health->master,
2041                                     bnxt_fw_health_readl(bp,
2042                                                          BNXT_FW_RESET_CNT_REG),
2043                                     bnxt_fw_health_readl(bp,
2044                                                          BNXT_FW_HEALTH_REG));
2045                 fw_health->tmr_multiplier =
2046                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2047                                      bp->current_interval * 10);
2048                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2049                 fw_health->last_fw_heartbeat =
2050                         bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2051                 fw_health->last_fw_reset_cnt =
2052                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2053                 goto async_event_process_exit;
2054         }
2055         default:
2056                 goto async_event_process_exit;
2057         }
2058         bnxt_queue_sp_work(bp);
2059 async_event_process_exit:
2060         bnxt_ulp_async_events(bp, cmpl);
2061         return 0;
2062 }
2063
2064 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2065 {
2066         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2067         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2068         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2069                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2070
2071         switch (cmpl_type) {
2072         case CMPL_BASE_TYPE_HWRM_DONE:
2073                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2074                 if (seq_id == bp->hwrm_intr_seq_id)
2075                         bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2076                 else
2077                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2078                 break;
2079
2080         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2081                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2082
2083                 if ((vf_id < bp->pf.first_vf_id) ||
2084                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2085                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2086                                    vf_id);
2087                         return -EINVAL;
2088                 }
2089
2090                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2091                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2092                 bnxt_queue_sp_work(bp);
2093                 break;
2094
2095         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2096                 bnxt_async_event_process(bp,
2097                                          (struct hwrm_async_event_cmpl *)txcmp);
2098
2099         default:
2100                 break;
2101         }
2102
2103         return 0;
2104 }
2105
2106 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2107 {
2108         struct bnxt_napi *bnapi = dev_instance;
2109         struct bnxt *bp = bnapi->bp;
2110         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2111         u32 cons = RING_CMP(cpr->cp_raw_cons);
2112
2113         cpr->event_ctr++;
2114         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2115         napi_schedule(&bnapi->napi);
2116         return IRQ_HANDLED;
2117 }
2118
2119 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2120 {
2121         u32 raw_cons = cpr->cp_raw_cons;
2122         u16 cons = RING_CMP(raw_cons);
2123         struct tx_cmp *txcmp;
2124
2125         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2126
2127         return TX_CMP_VALID(txcmp, raw_cons);
2128 }
2129
2130 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2131 {
2132         struct bnxt_napi *bnapi = dev_instance;
2133         struct bnxt *bp = bnapi->bp;
2134         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2135         u32 cons = RING_CMP(cpr->cp_raw_cons);
2136         u32 int_status;
2137
2138         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2139
2140         if (!bnxt_has_work(bp, cpr)) {
2141                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2142                 /* return if erroneous interrupt */
2143                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2144                         return IRQ_NONE;
2145         }
2146
2147         /* disable ring IRQ */
2148         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2149
2150         /* Return here if interrupt is shared and is disabled. */
2151         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2152                 return IRQ_HANDLED;
2153
2154         napi_schedule(&bnapi->napi);
2155         return IRQ_HANDLED;
2156 }
2157
2158 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2159                             int budget)
2160 {
2161         struct bnxt_napi *bnapi = cpr->bnapi;
2162         u32 raw_cons = cpr->cp_raw_cons;
2163         u32 cons;
2164         int tx_pkts = 0;
2165         int rx_pkts = 0;
2166         u8 event = 0;
2167         struct tx_cmp *txcmp;
2168
2169         cpr->has_more_work = 0;
2170         cpr->had_work_done = 1;
2171         while (1) {
2172                 int rc;
2173
2174                 cons = RING_CMP(raw_cons);
2175                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2176
2177                 if (!TX_CMP_VALID(txcmp, raw_cons))
2178                         break;
2179
2180                 /* The valid test of the entry must be done first before
2181                  * reading any further.
2182                  */
2183                 dma_rmb();
2184                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2185                         tx_pkts++;
2186                         /* return full budget so NAPI will complete. */
2187                         if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2188                                 rx_pkts = budget;
2189                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2190                                 if (budget)
2191                                         cpr->has_more_work = 1;
2192                                 break;
2193                         }
2194                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2195                         if (likely(budget))
2196                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2197                         else
2198                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2199                                                            &event);
2200                         if (likely(rc >= 0))
2201                                 rx_pkts += rc;
2202                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2203                          * the NAPI budget.  Otherwise, we may potentially loop
2204                          * here forever if we consistently cannot allocate
2205                          * buffers.
2206                          */
2207                         else if (rc == -ENOMEM && budget)
2208                                 rx_pkts++;
2209                         else if (rc == -EBUSY)  /* partial completion */
2210                                 break;
2211                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2212                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2213                                     (TX_CMP_TYPE(txcmp) ==
2214                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2215                                     (TX_CMP_TYPE(txcmp) ==
2216                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2217                         bnxt_hwrm_handler(bp, txcmp);
2218                 }
2219                 raw_cons = NEXT_RAW_CMP(raw_cons);
2220
2221                 if (rx_pkts && rx_pkts == budget) {
2222                         cpr->has_more_work = 1;
2223                         break;
2224                 }
2225         }
2226
2227         if (event & BNXT_REDIRECT_EVENT)
2228                 xdp_do_flush_map();
2229
2230         if (event & BNXT_TX_EVENT) {
2231                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2232                 u16 prod = txr->tx_prod;
2233
2234                 /* Sync BD data before updating doorbell */
2235                 wmb();
2236
2237                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2238         }
2239
2240         cpr->cp_raw_cons = raw_cons;
2241         bnapi->tx_pkts += tx_pkts;
2242         bnapi->events |= event;
2243         return rx_pkts;
2244 }
2245
2246 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2247 {
2248         if (bnapi->tx_pkts) {
2249                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2250                 bnapi->tx_pkts = 0;
2251         }
2252
2253         if (bnapi->events & BNXT_RX_EVENT) {
2254                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2255
2256                 if (bnapi->events & BNXT_AGG_EVENT)
2257                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2258                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2259         }
2260         bnapi->events = 0;
2261 }
2262
2263 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2264                           int budget)
2265 {
2266         struct bnxt_napi *bnapi = cpr->bnapi;
2267         int rx_pkts;
2268
2269         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2270
2271         /* ACK completion ring before freeing tx ring and producing new
2272          * buffers in rx/agg rings to prevent overflowing the completion
2273          * ring.
2274          */
2275         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2276
2277         __bnxt_poll_work_done(bp, bnapi);
2278         return rx_pkts;
2279 }
2280
2281 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2282 {
2283         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2284         struct bnxt *bp = bnapi->bp;
2285         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2286         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2287         struct tx_cmp *txcmp;
2288         struct rx_cmp_ext *rxcmp1;
2289         u32 cp_cons, tmp_raw_cons;
2290         u32 raw_cons = cpr->cp_raw_cons;
2291         u32 rx_pkts = 0;
2292         u8 event = 0;
2293
2294         while (1) {
2295                 int rc;
2296
2297                 cp_cons = RING_CMP(raw_cons);
2298                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2299
2300                 if (!TX_CMP_VALID(txcmp, raw_cons))
2301                         break;
2302
2303                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2304                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2305                         cp_cons = RING_CMP(tmp_raw_cons);
2306                         rxcmp1 = (struct rx_cmp_ext *)
2307                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2308
2309                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2310                                 break;
2311
2312                         /* force an error to recycle the buffer */
2313                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2314                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2315
2316                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2317                         if (likely(rc == -EIO) && budget)
2318                                 rx_pkts++;
2319                         else if (rc == -EBUSY)  /* partial completion */
2320                                 break;
2321                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2322                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2323                         bnxt_hwrm_handler(bp, txcmp);
2324                 } else {
2325                         netdev_err(bp->dev,
2326                                    "Invalid completion received on special ring\n");
2327                 }
2328                 raw_cons = NEXT_RAW_CMP(raw_cons);
2329
2330                 if (rx_pkts == budget)
2331                         break;
2332         }
2333
2334         cpr->cp_raw_cons = raw_cons;
2335         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2336         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2337
2338         if (event & BNXT_AGG_EVENT)
2339                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2340
2341         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2342                 napi_complete_done(napi, rx_pkts);
2343                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2344         }
2345         return rx_pkts;
2346 }
2347
2348 static int bnxt_poll(struct napi_struct *napi, int budget)
2349 {
2350         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2351         struct bnxt *bp = bnapi->bp;
2352         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2353         int work_done = 0;
2354
2355         while (1) {
2356                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2357
2358                 if (work_done >= budget) {
2359                         if (!budget)
2360                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2361                         break;
2362                 }
2363
2364                 if (!bnxt_has_work(bp, cpr)) {
2365                         if (napi_complete_done(napi, work_done))
2366                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2367                         break;
2368                 }
2369         }
2370         if (bp->flags & BNXT_FLAG_DIM) {
2371                 struct dim_sample dim_sample = {};
2372
2373                 dim_update_sample(cpr->event_ctr,
2374                                   cpr->rx_packets,
2375                                   cpr->rx_bytes,
2376                                   &dim_sample);
2377                 net_dim(&cpr->dim, dim_sample);
2378         }
2379         return work_done;
2380 }
2381
2382 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2383 {
2384         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2385         int i, work_done = 0;
2386
2387         for (i = 0; i < 2; i++) {
2388                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2389
2390                 if (cpr2) {
2391                         work_done += __bnxt_poll_work(bp, cpr2,
2392                                                       budget - work_done);
2393                         cpr->has_more_work |= cpr2->has_more_work;
2394                 }
2395         }
2396         return work_done;
2397 }
2398
2399 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2400                                  u64 dbr_type)
2401 {
2402         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2403         int i;
2404
2405         for (i = 0; i < 2; i++) {
2406                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2407                 struct bnxt_db_info *db;
2408
2409                 if (cpr2 && cpr2->had_work_done) {
2410                         db = &cpr2->cp_db;
2411                         writeq(db->db_key64 | dbr_type |
2412                                RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2413                         cpr2->had_work_done = 0;
2414                 }
2415         }
2416         __bnxt_poll_work_done(bp, bnapi);
2417 }
2418
2419 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2420 {
2421         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2422         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2423         u32 raw_cons = cpr->cp_raw_cons;
2424         struct bnxt *bp = bnapi->bp;
2425         struct nqe_cn *nqcmp;
2426         int work_done = 0;
2427         u32 cons;
2428
2429         if (cpr->has_more_work) {
2430                 cpr->has_more_work = 0;
2431                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2432         }
2433         while (1) {
2434                 cons = RING_CMP(raw_cons);
2435                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2436
2437                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2438                         if (cpr->has_more_work)
2439                                 break;
2440
2441                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2442                         cpr->cp_raw_cons = raw_cons;
2443                         if (napi_complete_done(napi, work_done))
2444                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2445                                                   cpr->cp_raw_cons);
2446                         return work_done;
2447                 }
2448
2449                 /* The valid test of the entry must be done first before
2450                  * reading any further.
2451                  */
2452                 dma_rmb();
2453
2454                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2455                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2456                         struct bnxt_cp_ring_info *cpr2;
2457
2458                         cpr2 = cpr->cp_ring_arr[idx];
2459                         work_done += __bnxt_poll_work(bp, cpr2,
2460                                                       budget - work_done);
2461                         cpr->has_more_work |= cpr2->has_more_work;
2462                 } else {
2463                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2464                 }
2465                 raw_cons = NEXT_RAW_CMP(raw_cons);
2466         }
2467         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2468         if (raw_cons != cpr->cp_raw_cons) {
2469                 cpr->cp_raw_cons = raw_cons;
2470                 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2471         }
2472         return work_done;
2473 }
2474
2475 static void bnxt_free_tx_skbs(struct bnxt *bp)
2476 {
2477         int i, max_idx;
2478         struct pci_dev *pdev = bp->pdev;
2479
2480         if (!bp->tx_ring)
2481                 return;
2482
2483         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2484         for (i = 0; i < bp->tx_nr_rings; i++) {
2485                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2486                 int j;
2487
2488                 for (j = 0; j < max_idx;) {
2489                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2490                         struct sk_buff *skb;
2491                         int k, last;
2492
2493                         if (i < bp->tx_nr_rings_xdp &&
2494                             tx_buf->action == XDP_REDIRECT) {
2495                                 dma_unmap_single(&pdev->dev,
2496                                         dma_unmap_addr(tx_buf, mapping),
2497                                         dma_unmap_len(tx_buf, len),
2498                                         PCI_DMA_TODEVICE);
2499                                 xdp_return_frame(tx_buf->xdpf);
2500                                 tx_buf->action = 0;
2501                                 tx_buf->xdpf = NULL;
2502                                 j++;
2503                                 continue;
2504                         }
2505
2506                         skb = tx_buf->skb;
2507                         if (!skb) {
2508                                 j++;
2509                                 continue;
2510                         }
2511
2512                         tx_buf->skb = NULL;
2513
2514                         if (tx_buf->is_push) {
2515                                 dev_kfree_skb(skb);
2516                                 j += 2;
2517                                 continue;
2518                         }
2519
2520                         dma_unmap_single(&pdev->dev,
2521                                          dma_unmap_addr(tx_buf, mapping),
2522                                          skb_headlen(skb),
2523                                          PCI_DMA_TODEVICE);
2524
2525                         last = tx_buf->nr_frags;
2526                         j += 2;
2527                         for (k = 0; k < last; k++, j++) {
2528                                 int ring_idx = j & bp->tx_ring_mask;
2529                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2530
2531                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2532                                 dma_unmap_page(
2533                                         &pdev->dev,
2534                                         dma_unmap_addr(tx_buf, mapping),
2535                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2536                         }
2537                         dev_kfree_skb(skb);
2538                 }
2539                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2540         }
2541 }
2542
2543 static void bnxt_free_rx_skbs(struct bnxt *bp)
2544 {
2545         int i, max_idx, max_agg_idx;
2546         struct pci_dev *pdev = bp->pdev;
2547
2548         if (!bp->rx_ring)
2549                 return;
2550
2551         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2552         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2553         for (i = 0; i < bp->rx_nr_rings; i++) {
2554                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2555                 struct bnxt_tpa_idx_map *map;
2556                 int j;
2557
2558                 if (rxr->rx_tpa) {
2559                         for (j = 0; j < bp->max_tpa; j++) {
2560                                 struct bnxt_tpa_info *tpa_info =
2561                                                         &rxr->rx_tpa[j];
2562                                 u8 *data = tpa_info->data;
2563
2564                                 if (!data)
2565                                         continue;
2566
2567                                 dma_unmap_single_attrs(&pdev->dev,
2568                                                        tpa_info->mapping,
2569                                                        bp->rx_buf_use_size,
2570                                                        bp->rx_dir,
2571                                                        DMA_ATTR_WEAK_ORDERING);
2572
2573                                 tpa_info->data = NULL;
2574
2575                                 kfree(data);
2576                         }
2577                 }
2578
2579                 for (j = 0; j < max_idx; j++) {
2580                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2581                         dma_addr_t mapping = rx_buf->mapping;
2582                         void *data = rx_buf->data;
2583
2584                         if (!data)
2585                                 continue;
2586
2587                         rx_buf->data = NULL;
2588
2589                         if (BNXT_RX_PAGE_MODE(bp)) {
2590                                 mapping -= bp->rx_dma_offset;
2591                                 dma_unmap_page_attrs(&pdev->dev, mapping,
2592                                                      PAGE_SIZE, bp->rx_dir,
2593                                                      DMA_ATTR_WEAK_ORDERING);
2594                                 page_pool_recycle_direct(rxr->page_pool, data);
2595                         } else {
2596                                 dma_unmap_single_attrs(&pdev->dev, mapping,
2597                                                        bp->rx_buf_use_size,
2598                                                        bp->rx_dir,
2599                                                        DMA_ATTR_WEAK_ORDERING);
2600                                 kfree(data);
2601                         }
2602                 }
2603
2604                 for (j = 0; j < max_agg_idx; j++) {
2605                         struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2606                                 &rxr->rx_agg_ring[j];
2607                         struct page *page = rx_agg_buf->page;
2608
2609                         if (!page)
2610                                 continue;
2611
2612                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2613                                              BNXT_RX_PAGE_SIZE,
2614                                              PCI_DMA_FROMDEVICE,
2615                                              DMA_ATTR_WEAK_ORDERING);
2616
2617                         rx_agg_buf->page = NULL;
2618                         __clear_bit(j, rxr->rx_agg_bmap);
2619
2620                         __free_page(page);
2621                 }
2622                 if (rxr->rx_page) {
2623                         __free_page(rxr->rx_page);
2624                         rxr->rx_page = NULL;
2625                 }
2626                 map = rxr->rx_tpa_idx_map;
2627                 if (map)
2628                         memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2629         }
2630 }
2631
2632 static void bnxt_free_skbs(struct bnxt *bp)
2633 {
2634         bnxt_free_tx_skbs(bp);
2635         bnxt_free_rx_skbs(bp);
2636 }
2637
2638 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2639 {
2640         struct pci_dev *pdev = bp->pdev;
2641         int i;
2642
2643         for (i = 0; i < rmem->nr_pages; i++) {
2644                 if (!rmem->pg_arr[i])
2645                         continue;
2646
2647                 dma_free_coherent(&pdev->dev, rmem->page_size,
2648                                   rmem->pg_arr[i], rmem->dma_arr[i]);
2649
2650                 rmem->pg_arr[i] = NULL;
2651         }
2652         if (rmem->pg_tbl) {
2653                 size_t pg_tbl_size = rmem->nr_pages * 8;
2654
2655                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2656                         pg_tbl_size = rmem->page_size;
2657                 dma_free_coherent(&pdev->dev, pg_tbl_size,
2658                                   rmem->pg_tbl, rmem->pg_tbl_map);
2659                 rmem->pg_tbl = NULL;
2660         }
2661         if (rmem->vmem_size && *rmem->vmem) {
2662                 vfree(*rmem->vmem);
2663                 *rmem->vmem = NULL;
2664         }
2665 }
2666
2667 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2668 {
2669         struct pci_dev *pdev = bp->pdev;
2670         u64 valid_bit = 0;
2671         int i;
2672
2673         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2674                 valid_bit = PTU_PTE_VALID;
2675         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2676                 size_t pg_tbl_size = rmem->nr_pages * 8;
2677
2678                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2679                         pg_tbl_size = rmem->page_size;
2680                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2681                                                   &rmem->pg_tbl_map,
2682                                                   GFP_KERNEL);
2683                 if (!rmem->pg_tbl)
2684                         return -ENOMEM;
2685         }
2686
2687         for (i = 0; i < rmem->nr_pages; i++) {
2688                 u64 extra_bits = valid_bit;
2689
2690                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2691                                                      rmem->page_size,
2692                                                      &rmem->dma_arr[i],
2693                                                      GFP_KERNEL);
2694                 if (!rmem->pg_arr[i])
2695                         return -ENOMEM;
2696
2697                 if (rmem->init_val)
2698                         memset(rmem->pg_arr[i], rmem->init_val,
2699                                rmem->page_size);
2700                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2701                         if (i == rmem->nr_pages - 2 &&
2702                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2703                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2704                         else if (i == rmem->nr_pages - 1 &&
2705                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2706                                 extra_bits |= PTU_PTE_LAST;
2707                         rmem->pg_tbl[i] =
2708                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2709                 }
2710         }
2711
2712         if (rmem->vmem_size) {
2713                 *rmem->vmem = vzalloc(rmem->vmem_size);
2714                 if (!(*rmem->vmem))
2715                         return -ENOMEM;
2716         }
2717         return 0;
2718 }
2719
2720 static void bnxt_free_tpa_info(struct bnxt *bp)
2721 {
2722         int i;
2723
2724         for (i = 0; i < bp->rx_nr_rings; i++) {
2725                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2726
2727                 kfree(rxr->rx_tpa_idx_map);
2728                 rxr->rx_tpa_idx_map = NULL;
2729                 if (rxr->rx_tpa) {
2730                         kfree(rxr->rx_tpa[0].agg_arr);
2731                         rxr->rx_tpa[0].agg_arr = NULL;
2732                 }
2733                 kfree(rxr->rx_tpa);
2734                 rxr->rx_tpa = NULL;
2735         }
2736 }
2737
2738 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2739 {
2740         int i, j, total_aggs = 0;
2741
2742         bp->max_tpa = MAX_TPA;
2743         if (bp->flags & BNXT_FLAG_CHIP_P5) {
2744                 if (!bp->max_tpa_v2)
2745                         return 0;
2746                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2747                 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2748         }
2749
2750         for (i = 0; i < bp->rx_nr_rings; i++) {
2751                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2752                 struct rx_agg_cmp *agg;
2753
2754                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2755                                       GFP_KERNEL);
2756                 if (!rxr->rx_tpa)
2757                         return -ENOMEM;
2758
2759                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2760                         continue;
2761                 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2762                 rxr->rx_tpa[0].agg_arr = agg;
2763                 if (!agg)
2764                         return -ENOMEM;
2765                 for (j = 1; j < bp->max_tpa; j++)
2766                         rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2767                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2768                                               GFP_KERNEL);
2769                 if (!rxr->rx_tpa_idx_map)
2770                         return -ENOMEM;
2771         }
2772         return 0;
2773 }
2774
2775 static void bnxt_free_rx_rings(struct bnxt *bp)
2776 {
2777         int i;
2778
2779         if (!bp->rx_ring)
2780                 return;
2781
2782         bnxt_free_tpa_info(bp);
2783         for (i = 0; i < bp->rx_nr_rings; i++) {
2784                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2785                 struct bnxt_ring_struct *ring;
2786
2787                 if (rxr->xdp_prog)
2788                         bpf_prog_put(rxr->xdp_prog);
2789
2790                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2791                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2792
2793                 page_pool_destroy(rxr->page_pool);
2794                 rxr->page_pool = NULL;
2795
2796                 kfree(rxr->rx_agg_bmap);
2797                 rxr->rx_agg_bmap = NULL;
2798
2799                 ring = &rxr->rx_ring_struct;
2800                 bnxt_free_ring(bp, &ring->ring_mem);
2801
2802                 ring = &rxr->rx_agg_ring_struct;
2803                 bnxt_free_ring(bp, &ring->ring_mem);
2804         }
2805 }
2806
2807 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2808                                    struct bnxt_rx_ring_info *rxr)
2809 {
2810         struct page_pool_params pp = { 0 };
2811
2812         pp.pool_size = bp->rx_ring_size;
2813         pp.nid = dev_to_node(&bp->pdev->dev);
2814         pp.dev = &bp->pdev->dev;
2815         pp.dma_dir = DMA_BIDIRECTIONAL;
2816
2817         rxr->page_pool = page_pool_create(&pp);
2818         if (IS_ERR(rxr->page_pool)) {
2819                 int err = PTR_ERR(rxr->page_pool);
2820
2821                 rxr->page_pool = NULL;
2822                 return err;
2823         }
2824         return 0;
2825 }
2826
2827 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2828 {
2829         int i, rc = 0, agg_rings = 0;
2830
2831         if (!bp->rx_ring)
2832                 return -ENOMEM;
2833
2834         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2835                 agg_rings = 1;
2836
2837         for (i = 0; i < bp->rx_nr_rings; i++) {
2838                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2839                 struct bnxt_ring_struct *ring;
2840
2841                 ring = &rxr->rx_ring_struct;
2842
2843                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2844                 if (rc)
2845                         return rc;
2846
2847                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2848                 if (rc < 0)
2849                         return rc;
2850
2851                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2852                                                 MEM_TYPE_PAGE_POOL,
2853                                                 rxr->page_pool);
2854                 if (rc) {
2855                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2856                         return rc;
2857                 }
2858
2859                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2860                 if (rc)
2861                         return rc;
2862
2863                 ring->grp_idx = i;
2864                 if (agg_rings) {
2865                         u16 mem_size;
2866
2867                         ring = &rxr->rx_agg_ring_struct;
2868                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2869                         if (rc)
2870                                 return rc;
2871
2872                         ring->grp_idx = i;
2873                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2874                         mem_size = rxr->rx_agg_bmap_size / 8;
2875                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2876                         if (!rxr->rx_agg_bmap)
2877                                 return -ENOMEM;
2878                 }
2879         }
2880         if (bp->flags & BNXT_FLAG_TPA)
2881                 rc = bnxt_alloc_tpa_info(bp);
2882         return rc;
2883 }
2884
2885 static void bnxt_free_tx_rings(struct bnxt *bp)
2886 {
2887         int i;
2888         struct pci_dev *pdev = bp->pdev;
2889
2890         if (!bp->tx_ring)
2891                 return;
2892
2893         for (i = 0; i < bp->tx_nr_rings; i++) {
2894                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2895                 struct bnxt_ring_struct *ring;
2896
2897                 if (txr->tx_push) {
2898                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2899                                           txr->tx_push, txr->tx_push_mapping);
2900                         txr->tx_push = NULL;
2901                 }
2902
2903                 ring = &txr->tx_ring_struct;
2904
2905                 bnxt_free_ring(bp, &ring->ring_mem);
2906         }
2907 }
2908
2909 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2910 {
2911         int i, j, rc;
2912         struct pci_dev *pdev = bp->pdev;
2913
2914         bp->tx_push_size = 0;
2915         if (bp->tx_push_thresh) {
2916                 int push_size;
2917
2918                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2919                                         bp->tx_push_thresh);
2920
2921                 if (push_size > 256) {
2922                         push_size = 0;
2923                         bp->tx_push_thresh = 0;
2924                 }
2925
2926                 bp->tx_push_size = push_size;
2927         }
2928
2929         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2930                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2931                 struct bnxt_ring_struct *ring;
2932                 u8 qidx;
2933
2934                 ring = &txr->tx_ring_struct;
2935
2936                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2937                 if (rc)
2938                         return rc;
2939
2940                 ring->grp_idx = txr->bnapi->index;
2941                 if (bp->tx_push_size) {
2942                         dma_addr_t mapping;
2943
2944                         /* One pre-allocated DMA buffer to backup
2945                          * TX push operation
2946                          */
2947                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2948                                                 bp->tx_push_size,
2949                                                 &txr->tx_push_mapping,
2950                                                 GFP_KERNEL);
2951
2952                         if (!txr->tx_push)
2953                                 return -ENOMEM;
2954
2955                         mapping = txr->tx_push_mapping +
2956                                 sizeof(struct tx_push_bd);
2957                         txr->data_mapping = cpu_to_le64(mapping);
2958                 }
2959                 qidx = bp->tc_to_qidx[j];
2960                 ring->queue_id = bp->q_info[qidx].queue_id;
2961                 if (i < bp->tx_nr_rings_xdp)
2962                         continue;
2963                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2964                         j++;
2965         }
2966         return 0;
2967 }
2968
2969 static void bnxt_free_cp_rings(struct bnxt *bp)
2970 {
2971         int i;
2972
2973         if (!bp->bnapi)
2974                 return;
2975
2976         for (i = 0; i < bp->cp_nr_rings; i++) {
2977                 struct bnxt_napi *bnapi = bp->bnapi[i];
2978                 struct bnxt_cp_ring_info *cpr;
2979                 struct bnxt_ring_struct *ring;
2980                 int j;
2981
2982                 if (!bnapi)
2983                         continue;
2984
2985                 cpr = &bnapi->cp_ring;
2986                 ring = &cpr->cp_ring_struct;
2987
2988                 bnxt_free_ring(bp, &ring->ring_mem);
2989
2990                 for (j = 0; j < 2; j++) {
2991                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2992
2993                         if (cpr2) {
2994                                 ring = &cpr2->cp_ring_struct;
2995                                 bnxt_free_ring(bp, &ring->ring_mem);
2996                                 kfree(cpr2);
2997                                 cpr->cp_ring_arr[j] = NULL;
2998                         }
2999                 }
3000         }
3001 }
3002
3003 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3004 {
3005         struct bnxt_ring_mem_info *rmem;
3006         struct bnxt_ring_struct *ring;
3007         struct bnxt_cp_ring_info *cpr;
3008         int rc;
3009
3010         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3011         if (!cpr)
3012                 return NULL;
3013
3014         ring = &cpr->cp_ring_struct;
3015         rmem = &ring->ring_mem;
3016         rmem->nr_pages = bp->cp_nr_pages;
3017         rmem->page_size = HW_CMPD_RING_SIZE;
3018         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3019         rmem->dma_arr = cpr->cp_desc_mapping;
3020         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3021         rc = bnxt_alloc_ring(bp, rmem);
3022         if (rc) {
3023                 bnxt_free_ring(bp, rmem);
3024                 kfree(cpr);
3025                 cpr = NULL;
3026         }
3027         return cpr;
3028 }
3029
3030 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3031 {
3032         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3033         int i, rc, ulp_base_vec, ulp_msix;
3034
3035         ulp_msix = bnxt_get_ulp_msix_num(bp);
3036         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3037         for (i = 0; i < bp->cp_nr_rings; i++) {
3038                 struct bnxt_napi *bnapi = bp->bnapi[i];
3039                 struct bnxt_cp_ring_info *cpr;
3040                 struct bnxt_ring_struct *ring;
3041
3042                 if (!bnapi)
3043                         continue;
3044
3045                 cpr = &bnapi->cp_ring;
3046                 cpr->bnapi = bnapi;
3047                 ring = &cpr->cp_ring_struct;
3048
3049                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3050                 if (rc)
3051                         return rc;
3052
3053                 if (ulp_msix && i >= ulp_base_vec)
3054                         ring->map_idx = i + ulp_msix;
3055                 else
3056                         ring->map_idx = i;
3057
3058                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3059                         continue;
3060
3061                 if (i < bp->rx_nr_rings) {
3062                         struct bnxt_cp_ring_info *cpr2 =
3063                                 bnxt_alloc_cp_sub_ring(bp);
3064
3065                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3066                         if (!cpr2)
3067                                 return -ENOMEM;
3068                         cpr2->bnapi = bnapi;
3069                 }
3070                 if ((sh && i < bp->tx_nr_rings) ||
3071                     (!sh && i >= bp->rx_nr_rings)) {
3072                         struct bnxt_cp_ring_info *cpr2 =
3073                                 bnxt_alloc_cp_sub_ring(bp);
3074
3075                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3076                         if (!cpr2)
3077                                 return -ENOMEM;
3078                         cpr2->bnapi = bnapi;
3079                 }
3080         }
3081         return 0;
3082 }
3083
3084 static void bnxt_init_ring_struct(struct bnxt *bp)
3085 {
3086         int i;
3087
3088         for (i = 0; i < bp->cp_nr_rings; i++) {
3089                 struct bnxt_napi *bnapi = bp->bnapi[i];
3090                 struct bnxt_ring_mem_info *rmem;
3091                 struct bnxt_cp_ring_info *cpr;
3092                 struct bnxt_rx_ring_info *rxr;
3093                 struct bnxt_tx_ring_info *txr;
3094                 struct bnxt_ring_struct *ring;
3095
3096                 if (!bnapi)
3097                         continue;
3098
3099                 cpr = &bnapi->cp_ring;
3100                 ring = &cpr->cp_ring_struct;
3101                 rmem = &ring->ring_mem;
3102                 rmem->nr_pages = bp->cp_nr_pages;
3103                 rmem->page_size = HW_CMPD_RING_SIZE;
3104                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3105                 rmem->dma_arr = cpr->cp_desc_mapping;
3106                 rmem->vmem_size = 0;
3107
3108                 rxr = bnapi->rx_ring;
3109                 if (!rxr)
3110                         goto skip_rx;
3111
3112                 ring = &rxr->rx_ring_struct;
3113                 rmem = &ring->ring_mem;
3114                 rmem->nr_pages = bp->rx_nr_pages;
3115                 rmem->page_size = HW_RXBD_RING_SIZE;
3116                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3117                 rmem->dma_arr = rxr->rx_desc_mapping;
3118                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3119                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3120
3121                 ring = &rxr->rx_agg_ring_struct;
3122                 rmem = &ring->ring_mem;
3123                 rmem->nr_pages = bp->rx_agg_nr_pages;
3124                 rmem->page_size = HW_RXBD_RING_SIZE;
3125                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3126                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3127                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3128                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3129
3130 skip_rx:
3131                 txr = bnapi->tx_ring;
3132                 if (!txr)
3133                         continue;
3134
3135                 ring = &txr->tx_ring_struct;
3136                 rmem = &ring->ring_mem;
3137                 rmem->nr_pages = bp->tx_nr_pages;
3138                 rmem->page_size = HW_RXBD_RING_SIZE;
3139                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3140                 rmem->dma_arr = txr->tx_desc_mapping;
3141                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3142                 rmem->vmem = (void **)&txr->tx_buf_ring;
3143         }
3144 }
3145
3146 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3147 {
3148         int i;
3149         u32 prod;
3150         struct rx_bd **rx_buf_ring;
3151
3152         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3153         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3154                 int j;
3155                 struct rx_bd *rxbd;
3156
3157                 rxbd = rx_buf_ring[i];
3158                 if (!rxbd)
3159                         continue;
3160
3161                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3162                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3163                         rxbd->rx_bd_opaque = prod;
3164                 }
3165         }
3166 }
3167
3168 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3169 {
3170         struct net_device *dev = bp->dev;
3171         struct bnxt_rx_ring_info *rxr;
3172         struct bnxt_ring_struct *ring;
3173         u32 prod, type;
3174         int i;
3175
3176         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3177                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3178
3179         if (NET_IP_ALIGN == 2)
3180                 type |= RX_BD_FLAGS_SOP;
3181
3182         rxr = &bp->rx_ring[ring_nr];
3183         ring = &rxr->rx_ring_struct;
3184         bnxt_init_rxbd_pages(ring, type);
3185
3186         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3187                 bpf_prog_add(bp->xdp_prog, 1);
3188                 rxr->xdp_prog = bp->xdp_prog;
3189         }
3190         prod = rxr->rx_prod;
3191         for (i = 0; i < bp->rx_ring_size; i++) {
3192                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
3193                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3194                                     ring_nr, i, bp->rx_ring_size);
3195                         break;
3196                 }
3197                 prod = NEXT_RX(prod);
3198         }
3199         rxr->rx_prod = prod;
3200         ring->fw_ring_id = INVALID_HW_RING_ID;
3201
3202         ring = &rxr->rx_agg_ring_struct;
3203         ring->fw_ring_id = INVALID_HW_RING_ID;
3204
3205         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3206                 return 0;
3207
3208         type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3209                 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3210
3211         bnxt_init_rxbd_pages(ring, type);
3212
3213         prod = rxr->rx_agg_prod;
3214         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3215                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
3216                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3217                                     ring_nr, i, bp->rx_ring_size);
3218                         break;
3219                 }
3220                 prod = NEXT_RX_AGG(prod);
3221         }
3222         rxr->rx_agg_prod = prod;
3223
3224         if (bp->flags & BNXT_FLAG_TPA) {
3225                 if (rxr->rx_tpa) {
3226                         u8 *data;
3227                         dma_addr_t mapping;
3228
3229                         for (i = 0; i < bp->max_tpa; i++) {
3230                                 data = __bnxt_alloc_rx_data(bp, &mapping,
3231                                                             GFP_KERNEL);
3232                                 if (!data)
3233                                         return -ENOMEM;
3234
3235                                 rxr->rx_tpa[i].data = data;
3236                                 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3237                                 rxr->rx_tpa[i].mapping = mapping;
3238                         }
3239                 } else {
3240                         netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
3241                         return -ENOMEM;
3242                 }
3243         }
3244
3245         return 0;
3246 }
3247
3248 static void bnxt_init_cp_rings(struct bnxt *bp)
3249 {
3250         int i, j;
3251
3252         for (i = 0; i < bp->cp_nr_rings; i++) {
3253                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3254                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3255
3256                 ring->fw_ring_id = INVALID_HW_RING_ID;
3257                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3258                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3259                 for (j = 0; j < 2; j++) {
3260                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3261
3262                         if (!cpr2)
3263                                 continue;
3264
3265                         ring = &cpr2->cp_ring_struct;
3266                         ring->fw_ring_id = INVALID_HW_RING_ID;
3267                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3268                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3269                 }
3270         }
3271 }
3272
3273 static int bnxt_init_rx_rings(struct bnxt *bp)
3274 {
3275         int i, rc = 0;
3276
3277         if (BNXT_RX_PAGE_MODE(bp)) {
3278                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3279                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3280         } else {
3281                 bp->rx_offset = BNXT_RX_OFFSET;
3282                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3283         }
3284
3285         for (i = 0; i < bp->rx_nr_rings; i++) {
3286                 rc = bnxt_init_one_rx_ring(bp, i);
3287                 if (rc)
3288                         break;
3289         }
3290
3291         return rc;
3292 }
3293
3294 static int bnxt_init_tx_rings(struct bnxt *bp)
3295 {
3296         u16 i;
3297
3298         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3299                                    MAX_SKB_FRAGS + 1);
3300
3301         for (i = 0; i < bp->tx_nr_rings; i++) {
3302                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3303                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3304
3305                 ring->fw_ring_id = INVALID_HW_RING_ID;
3306         }
3307
3308         return 0;
3309 }
3310
3311 static void bnxt_free_ring_grps(struct bnxt *bp)
3312 {
3313         kfree(bp->grp_info);
3314         bp->grp_info = NULL;
3315 }
3316
3317 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3318 {
3319         int i;
3320
3321         if (irq_re_init) {
3322                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3323                                        sizeof(struct bnxt_ring_grp_info),
3324                                        GFP_KERNEL);
3325                 if (!bp->grp_info)
3326                         return -ENOMEM;
3327         }
3328         for (i = 0; i < bp->cp_nr_rings; i++) {
3329                 if (irq_re_init)
3330                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3331                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3332                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3333                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3334                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3335         }
3336         return 0;
3337 }
3338
3339 static void bnxt_free_vnics(struct bnxt *bp)
3340 {
3341         kfree(bp->vnic_info);
3342         bp->vnic_info = NULL;
3343         bp->nr_vnics = 0;
3344 }
3345
3346 static int bnxt_alloc_vnics(struct bnxt *bp)
3347 {
3348         int num_vnics = 1;
3349
3350 #ifdef CONFIG_RFS_ACCEL
3351         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3352                 num_vnics += bp->rx_nr_rings;
3353 #endif
3354
3355         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3356                 num_vnics++;
3357
3358         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3359                                 GFP_KERNEL);
3360         if (!bp->vnic_info)
3361                 return -ENOMEM;
3362
3363         bp->nr_vnics = num_vnics;
3364         return 0;
3365 }
3366
3367 static void bnxt_init_vnics(struct bnxt *bp)
3368 {
3369         int i;
3370
3371         for (i = 0; i < bp->nr_vnics; i++) {
3372                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3373                 int j;
3374
3375                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3376                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3377                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3378
3379                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3380
3381                 if (bp->vnic_info[i].rss_hash_key) {
3382                         if (i == 0)
3383                                 prandom_bytes(vnic->rss_hash_key,
3384                                               HW_HASH_KEY_SIZE);
3385                         else
3386                                 memcpy(vnic->rss_hash_key,
3387                                        bp->vnic_info[0].rss_hash_key,
3388                                        HW_HASH_KEY_SIZE);
3389                 }
3390         }
3391 }
3392
3393 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3394 {
3395         int pages;
3396
3397         pages = ring_size / desc_per_pg;
3398
3399         if (!pages)
3400                 return 1;
3401
3402         pages++;
3403
3404         while (pages & (pages - 1))
3405                 pages++;
3406
3407         return pages;
3408 }
3409
3410 void bnxt_set_tpa_flags(struct bnxt *bp)
3411 {
3412         bp->flags &= ~BNXT_FLAG_TPA;
3413         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3414                 return;
3415         if (bp->dev->features & NETIF_F_LRO)
3416                 bp->flags |= BNXT_FLAG_LRO;
3417         else if (bp->dev->features & NETIF_F_GRO_HW)
3418                 bp->flags |= BNXT_FLAG_GRO;
3419 }
3420
3421 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3422  * be set on entry.
3423  */
3424 void bnxt_set_ring_params(struct bnxt *bp)
3425 {
3426         u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3427         u32 agg_factor = 0, agg_ring_size = 0;
3428
3429         /* 8 for CRC and VLAN */
3430         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3431
3432         rx_space = rx_size + NET_SKB_PAD +
3433                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3434
3435         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3436         ring_size = bp->rx_ring_size;
3437         bp->rx_agg_ring_size = 0;
3438         bp->rx_agg_nr_pages = 0;
3439
3440         if (bp->flags & BNXT_FLAG_TPA)
3441                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3442
3443         bp->flags &= ~BNXT_FLAG_JUMBO;
3444         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3445                 u32 jumbo_factor;
3446
3447                 bp->flags |= BNXT_FLAG_JUMBO;
3448                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3449                 if (jumbo_factor > agg_factor)
3450                         agg_factor = jumbo_factor;
3451         }
3452         agg_ring_size = ring_size * agg_factor;
3453
3454         if (agg_ring_size) {
3455                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3456                                                         RX_DESC_CNT);
3457                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3458                         u32 tmp = agg_ring_size;
3459
3460                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3461                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3462                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3463                                     tmp, agg_ring_size);
3464                 }
3465                 bp->rx_agg_ring_size = agg_ring_size;
3466                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3467                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3468                 rx_space = rx_size + NET_SKB_PAD +
3469                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3470         }
3471
3472         bp->rx_buf_use_size = rx_size;
3473         bp->rx_buf_size = rx_space;
3474
3475         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3476         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3477
3478         ring_size = bp->tx_ring_size;
3479         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3480         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3481
3482         max_rx_cmpl = bp->rx_ring_size;
3483         /* MAX TPA needs to be added because TPA_START completions are
3484          * immediately recycled, so the TPA completions are not bound by
3485          * the RX ring size.
3486          */
3487         if (bp->flags & BNXT_FLAG_TPA)
3488                 max_rx_cmpl += bp->max_tpa;
3489         /* RX and TPA completions are 32-byte, all others are 16-byte */
3490         ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3491         bp->cp_ring_size = ring_size;
3492
3493         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3494         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3495                 bp->cp_nr_pages = MAX_CP_PAGES;
3496                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3497                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3498                             ring_size, bp->cp_ring_size);
3499         }
3500         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3501         bp->cp_ring_mask = bp->cp_bit - 1;
3502 }
3503
3504 /* Changing allocation mode of RX rings.
3505  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3506  */
3507 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3508 {
3509         if (page_mode) {
3510                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3511                         return -EOPNOTSUPP;
3512                 bp->dev->max_mtu =
3513                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3514                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3515                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3516                 bp->rx_dir = DMA_BIDIRECTIONAL;
3517                 bp->rx_skb_func = bnxt_rx_page_skb;
3518                 /* Disable LRO or GRO_HW */
3519                 netdev_update_features(bp->dev);
3520         } else {
3521                 bp->dev->max_mtu = bp->max_mtu;
3522                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3523                 bp->rx_dir = DMA_FROM_DEVICE;
3524                 bp->rx_skb_func = bnxt_rx_skb;
3525         }
3526         return 0;
3527 }
3528
3529 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3530 {
3531         int i;
3532         struct bnxt_vnic_info *vnic;
3533         struct pci_dev *pdev = bp->pdev;
3534
3535         if (!bp->vnic_info)
3536                 return;
3537
3538         for (i = 0; i < bp->nr_vnics; i++) {
3539                 vnic = &bp->vnic_info[i];
3540
3541                 kfree(vnic->fw_grp_ids);
3542                 vnic->fw_grp_ids = NULL;
3543
3544                 kfree(vnic->uc_list);
3545                 vnic->uc_list = NULL;
3546
3547                 if (vnic->mc_list) {
3548                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3549                                           vnic->mc_list, vnic->mc_list_mapping);
3550                         vnic->mc_list = NULL;
3551                 }
3552
3553                 if (vnic->rss_table) {
3554                         dma_free_coherent(&pdev->dev, vnic->rss_table_size,
3555                                           vnic->rss_table,
3556                                           vnic->rss_table_dma_addr);
3557                         vnic->rss_table = NULL;
3558                 }
3559
3560                 vnic->rss_hash_key = NULL;
3561                 vnic->flags = 0;
3562         }
3563 }
3564
3565 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3566 {
3567         int i, rc = 0, size;
3568         struct bnxt_vnic_info *vnic;
3569         struct pci_dev *pdev = bp->pdev;
3570         int max_rings;
3571
3572         for (i = 0; i < bp->nr_vnics; i++) {
3573                 vnic = &bp->vnic_info[i];
3574
3575                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3576                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3577
3578                         if (mem_size > 0) {
3579                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3580                                 if (!vnic->uc_list) {
3581                                         rc = -ENOMEM;
3582                                         goto out;
3583                                 }
3584                         }
3585                 }
3586
3587                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3588                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3589                         vnic->mc_list =
3590                                 dma_alloc_coherent(&pdev->dev,
3591                                                    vnic->mc_list_size,
3592                                                    &vnic->mc_list_mapping,
3593                                                    GFP_KERNEL);
3594                         if (!vnic->mc_list) {
3595                                 rc = -ENOMEM;
3596                                 goto out;
3597                         }
3598                 }
3599
3600                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3601                         goto vnic_skip_grps;
3602
3603                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3604                         max_rings = bp->rx_nr_rings;
3605                 else
3606                         max_rings = 1;
3607
3608                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3609                 if (!vnic->fw_grp_ids) {
3610                         rc = -ENOMEM;
3611                         goto out;
3612                 }
3613 vnic_skip_grps:
3614                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3615                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3616                         continue;
3617
3618                 /* Allocate rss table and hash key */
3619                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3620                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3621                         size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
3622
3623                 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
3624                 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
3625                                                      vnic->rss_table_size,
3626                                                      &vnic->rss_table_dma_addr,
3627                                                      GFP_KERNEL);
3628                 if (!vnic->rss_table) {
3629                         rc = -ENOMEM;
3630                         goto out;
3631                 }
3632
3633                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3634                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3635         }
3636         return 0;
3637
3638 out:
3639         return rc;
3640 }
3641
3642 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3643 {
3644         struct pci_dev *pdev = bp->pdev;
3645
3646         if (bp->hwrm_cmd_resp_addr) {
3647                 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3648                                   bp->hwrm_cmd_resp_dma_addr);
3649                 bp->hwrm_cmd_resp_addr = NULL;
3650         }
3651
3652         if (bp->hwrm_cmd_kong_resp_addr) {
3653                 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3654                                   bp->hwrm_cmd_kong_resp_addr,
3655                                   bp->hwrm_cmd_kong_resp_dma_addr);
3656                 bp->hwrm_cmd_kong_resp_addr = NULL;
3657         }
3658 }
3659
3660 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3661 {
3662         struct pci_dev *pdev = bp->pdev;
3663
3664         if (bp->hwrm_cmd_kong_resp_addr)
3665                 return 0;
3666
3667         bp->hwrm_cmd_kong_resp_addr =
3668                 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3669                                    &bp->hwrm_cmd_kong_resp_dma_addr,
3670                                    GFP_KERNEL);
3671         if (!bp->hwrm_cmd_kong_resp_addr)
3672                 return -ENOMEM;
3673
3674         return 0;
3675 }
3676
3677 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3678 {
3679         struct pci_dev *pdev = bp->pdev;
3680
3681         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3682                                                    &bp->hwrm_cmd_resp_dma_addr,
3683                                                    GFP_KERNEL);
3684         if (!bp->hwrm_cmd_resp_addr)
3685                 return -ENOMEM;
3686
3687         return 0;
3688 }
3689
3690 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3691 {
3692         if (bp->hwrm_short_cmd_req_addr) {
3693                 struct pci_dev *pdev = bp->pdev;
3694
3695                 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3696                                   bp->hwrm_short_cmd_req_addr,
3697                                   bp->hwrm_short_cmd_req_dma_addr);
3698                 bp->hwrm_short_cmd_req_addr = NULL;
3699         }
3700 }
3701
3702 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3703 {
3704         struct pci_dev *pdev = bp->pdev;
3705
3706         if (bp->hwrm_short_cmd_req_addr)
3707                 return 0;
3708
3709         bp->hwrm_short_cmd_req_addr =
3710                 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3711                                    &bp->hwrm_short_cmd_req_dma_addr,
3712                                    GFP_KERNEL);
3713         if (!bp->hwrm_short_cmd_req_addr)
3714                 return -ENOMEM;
3715
3716         return 0;
3717 }
3718
3719 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
3720 {
3721         kfree(stats->hw_masks);
3722         stats->hw_masks = NULL;
3723         kfree(stats->sw_stats);
3724         stats->sw_stats = NULL;
3725         if (stats->hw_stats) {
3726                 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
3727                                   stats->hw_stats_map);
3728                 stats->hw_stats = NULL;
3729         }
3730 }
3731
3732 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
3733                                 bool alloc_masks)
3734 {
3735         stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
3736                                              &stats->hw_stats_map, GFP_KERNEL);
3737         if (!stats->hw_stats)
3738                 return -ENOMEM;
3739
3740         stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
3741         if (!stats->sw_stats)
3742                 goto stats_mem_err;
3743
3744         if (alloc_masks) {
3745                 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
3746                 if (!stats->hw_masks)
3747                         goto stats_mem_err;
3748         }
3749         return 0;
3750
3751 stats_mem_err:
3752         bnxt_free_stats_mem(bp, stats);
3753         return -ENOMEM;
3754 }
3755
3756 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
3757 {
3758         int i;
3759
3760         for (i = 0; i < count; i++)
3761                 mask_arr[i] = mask;
3762 }
3763
3764 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
3765 {
3766         int i;
3767
3768         for (i = 0; i < count; i++)
3769                 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
3770 }
3771
3772 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
3773                                     struct bnxt_stats_mem *stats)
3774 {
3775         struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
3776         struct hwrm_func_qstats_ext_input req = {0};
3777         __le64 *hw_masks;
3778         int rc;
3779
3780         if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
3781             !(bp->flags & BNXT_FLAG_CHIP_P5))
3782                 return -EOPNOTSUPP;
3783
3784         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1);
3785         req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3786         mutex_lock(&bp->hwrm_cmd_lock);
3787         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3788         if (rc)
3789                 goto qstat_exit;
3790
3791         hw_masks = &resp->rx_ucast_pkts;
3792         bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
3793
3794 qstat_exit:
3795         mutex_unlock(&bp->hwrm_cmd_lock);
3796         return rc;
3797 }
3798
3799 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
3800 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
3801
3802 static void bnxt_init_stats(struct bnxt *bp)
3803 {
3804         struct bnxt_napi *bnapi = bp->bnapi[0];
3805         struct bnxt_cp_ring_info *cpr;
3806         struct bnxt_stats_mem *stats;
3807         __le64 *rx_stats, *tx_stats;
3808         int rc, rx_count, tx_count;
3809         u64 *rx_masks, *tx_masks;
3810         u64 mask;
3811         u8 flags;
3812
3813         cpr = &bnapi->cp_ring;
3814         stats = &cpr->stats;
3815         rc = bnxt_hwrm_func_qstat_ext(bp, stats);
3816         if (rc) {
3817                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3818                         mask = (1ULL << 48) - 1;
3819                 else
3820                         mask = -1ULL;
3821                 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
3822         }
3823         if (bp->flags & BNXT_FLAG_PORT_STATS) {
3824                 stats = &bp->port_stats;
3825                 rx_stats = stats->hw_stats;
3826                 rx_masks = stats->hw_masks;
3827                 rx_count = sizeof(struct rx_port_stats) / 8;
3828                 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3829                 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3830                 tx_count = sizeof(struct tx_port_stats) / 8;
3831
3832                 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
3833                 rc = bnxt_hwrm_port_qstats(bp, flags);
3834                 if (rc) {
3835                         mask = (1ULL << 40) - 1;
3836
3837                         bnxt_fill_masks(rx_masks, mask, rx_count);
3838                         bnxt_fill_masks(tx_masks, mask, tx_count);
3839                 } else {
3840                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3841                         bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
3842                         bnxt_hwrm_port_qstats(bp, 0);
3843                 }
3844         }
3845         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
3846                 stats = &bp->rx_port_stats_ext;
3847                 rx_stats = stats->hw_stats;
3848                 rx_masks = stats->hw_masks;
3849                 rx_count = sizeof(struct rx_port_stats_ext) / 8;
3850                 stats = &bp->tx_port_stats_ext;
3851                 tx_stats = stats->hw_stats;
3852                 tx_masks = stats->hw_masks;
3853                 tx_count = sizeof(struct tx_port_stats_ext) / 8;
3854
3855                 flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3856                 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
3857                 if (rc) {
3858                         mask = (1ULL << 40) - 1;
3859
3860                         bnxt_fill_masks(rx_masks, mask, rx_count);
3861                         if (tx_stats)
3862                                 bnxt_fill_masks(tx_masks, mask, tx_count);
3863                 } else {
3864                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3865                         if (tx_stats)
3866                                 bnxt_copy_hw_masks(tx_masks, tx_stats,
3867                                                    tx_count);
3868                         bnxt_hwrm_port_qstats_ext(bp, 0);
3869                 }
3870         }
3871 }
3872
3873 static void bnxt_free_port_stats(struct bnxt *bp)
3874 {
3875         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3876         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3877
3878         bnxt_free_stats_mem(bp, &bp->port_stats);
3879         bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
3880         bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
3881 }
3882
3883 static void bnxt_free_ring_stats(struct bnxt *bp)
3884 {
3885         int i;
3886
3887         if (!bp->bnapi)
3888                 return;
3889
3890         for (i = 0; i < bp->cp_nr_rings; i++) {
3891                 struct bnxt_napi *bnapi = bp->bnapi[i];
3892                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3893
3894                 bnxt_free_stats_mem(bp, &cpr->stats);
3895         }
3896 }
3897
3898 static int bnxt_alloc_stats(struct bnxt *bp)
3899 {
3900         u32 size, i;
3901         int rc;
3902
3903         size = bp->hw_ring_stats_size;
3904
3905         for (i = 0; i < bp->cp_nr_rings; i++) {
3906                 struct bnxt_napi *bnapi = bp->bnapi[i];
3907                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3908
3909                 cpr->stats.len = size;
3910                 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
3911                 if (rc)
3912                         return rc;
3913
3914                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3915         }
3916
3917         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3918                 return 0;
3919
3920         if (bp->port_stats.hw_stats)
3921                 goto alloc_ext_stats;
3922
3923         bp->port_stats.len = BNXT_PORT_STATS_SIZE;
3924         rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
3925         if (rc)
3926                 return rc;
3927
3928         bp->flags |= BNXT_FLAG_PORT_STATS;
3929
3930 alloc_ext_stats:
3931         /* Display extended statistics only if FW supports it */
3932         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
3933                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
3934                         return 0;
3935
3936         if (bp->rx_port_stats_ext.hw_stats)
3937                 goto alloc_tx_ext_stats;
3938
3939         bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
3940         rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
3941         /* Extended stats are optional */
3942         if (rc)
3943                 return 0;
3944
3945 alloc_tx_ext_stats:
3946         if (bp->tx_port_stats_ext.hw_stats)
3947                 return 0;
3948
3949         if (bp->hwrm_spec_code >= 0x10902 ||
3950             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
3951                 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
3952                 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
3953                 /* Extended stats are optional */
3954                 if (rc)
3955                         return 0;
3956         }
3957         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
3958         return 0;
3959 }
3960
3961 static void bnxt_clear_ring_indices(struct bnxt *bp)
3962 {
3963         int i;
3964
3965         if (!bp->bnapi)
3966                 return;
3967
3968         for (i = 0; i < bp->cp_nr_rings; i++) {
3969                 struct bnxt_napi *bnapi = bp->bnapi[i];
3970                 struct bnxt_cp_ring_info *cpr;
3971                 struct bnxt_rx_ring_info *rxr;
3972                 struct bnxt_tx_ring_info *txr;
3973
3974                 if (!bnapi)
3975                         continue;
3976
3977                 cpr = &bnapi->cp_ring;
3978                 cpr->cp_raw_cons = 0;
3979
3980                 txr = bnapi->tx_ring;
3981                 if (txr) {
3982                         txr->tx_prod = 0;
3983                         txr->tx_cons = 0;
3984                 }
3985
3986                 rxr = bnapi->rx_ring;
3987                 if (rxr) {
3988                         rxr->rx_prod = 0;
3989                         rxr->rx_agg_prod = 0;
3990                         rxr->rx_sw_agg_prod = 0;
3991                         rxr->rx_next_cons = 0;
3992                 }
3993         }
3994 }
3995
3996 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3997 {
3998 #ifdef CONFIG_RFS_ACCEL
3999         int i;
4000
4001         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
4002          * safe to delete the hash table.
4003          */
4004         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4005                 struct hlist_head *head;
4006                 struct hlist_node *tmp;
4007                 struct bnxt_ntuple_filter *fltr;
4008
4009                 head = &bp->ntp_fltr_hash_tbl[i];
4010                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4011                         hlist_del(&fltr->hash);
4012                         kfree(fltr);
4013                 }
4014         }
4015         if (irq_reinit) {
4016                 kfree(bp->ntp_fltr_bmap);
4017                 bp->ntp_fltr_bmap = NULL;
4018         }
4019         bp->ntp_fltr_count = 0;
4020 #endif
4021 }
4022
4023 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4024 {
4025 #ifdef CONFIG_RFS_ACCEL
4026         int i, rc = 0;
4027
4028         if (!(bp->flags & BNXT_FLAG_RFS))
4029                 return 0;
4030
4031         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4032                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4033
4034         bp->ntp_fltr_count = 0;
4035         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
4036                                     sizeof(long),
4037                                     GFP_KERNEL);
4038
4039         if (!bp->ntp_fltr_bmap)
4040                 rc = -ENOMEM;
4041
4042         return rc;
4043 #else
4044         return 0;
4045 #endif
4046 }
4047
4048 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4049 {
4050         bnxt_free_vnic_attributes(bp);
4051         bnxt_free_tx_rings(bp);
4052         bnxt_free_rx_rings(bp);
4053         bnxt_free_cp_rings(bp);
4054         bnxt_free_ntp_fltrs(bp, irq_re_init);
4055         if (irq_re_init) {
4056                 bnxt_free_ring_stats(bp);
4057                 if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET))
4058                         bnxt_free_port_stats(bp);
4059                 bnxt_free_ring_grps(bp);
4060                 bnxt_free_vnics(bp);
4061                 kfree(bp->tx_ring_map);
4062                 bp->tx_ring_map = NULL;
4063                 kfree(bp->tx_ring);
4064                 bp->tx_ring = NULL;
4065                 kfree(bp->rx_ring);
4066                 bp->rx_ring = NULL;
4067                 kfree(bp->bnapi);
4068                 bp->bnapi = NULL;
4069         } else {
4070                 bnxt_clear_ring_indices(bp);
4071         }
4072 }
4073
4074 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4075 {
4076         int i, j, rc, size, arr_size;
4077         void *bnapi;
4078
4079         if (irq_re_init) {
4080                 /* Allocate bnapi mem pointer array and mem block for
4081                  * all queues
4082                  */
4083                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4084                                 bp->cp_nr_rings);
4085                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4086                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4087                 if (!bnapi)
4088                         return -ENOMEM;
4089
4090                 bp->bnapi = bnapi;
4091                 bnapi += arr_size;
4092                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4093                         bp->bnapi[i] = bnapi;
4094                         bp->bnapi[i]->index = i;
4095                         bp->bnapi[i]->bp = bp;
4096                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4097                                 struct bnxt_cp_ring_info *cpr =
4098                                         &bp->bnapi[i]->cp_ring;
4099
4100                                 cpr->cp_ring_struct.ring_mem.flags =
4101                                         BNXT_RMEM_RING_PTE_FLAG;
4102                         }
4103                 }
4104
4105                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4106                                       sizeof(struct bnxt_rx_ring_info),
4107                                       GFP_KERNEL);
4108                 if (!bp->rx_ring)
4109                         return -ENOMEM;
4110
4111                 for (i = 0; i < bp->rx_nr_rings; i++) {
4112                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4113
4114                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4115                                 rxr->rx_ring_struct.ring_mem.flags =
4116                                         BNXT_RMEM_RING_PTE_FLAG;
4117                                 rxr->rx_agg_ring_struct.ring_mem.flags =
4118                                         BNXT_RMEM_RING_PTE_FLAG;
4119                         }
4120                         rxr->bnapi = bp->bnapi[i];
4121                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4122                 }
4123
4124                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4125                                       sizeof(struct bnxt_tx_ring_info),
4126                                       GFP_KERNEL);
4127                 if (!bp->tx_ring)
4128                         return -ENOMEM;
4129
4130                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4131                                           GFP_KERNEL);
4132
4133                 if (!bp->tx_ring_map)
4134                         return -ENOMEM;
4135
4136                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4137                         j = 0;
4138                 else
4139                         j = bp->rx_nr_rings;
4140
4141                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4142                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4143
4144                         if (bp->flags & BNXT_FLAG_CHIP_P5)
4145                                 txr->tx_ring_struct.ring_mem.flags =
4146                                         BNXT_RMEM_RING_PTE_FLAG;
4147                         txr->bnapi = bp->bnapi[j];
4148                         bp->bnapi[j]->tx_ring = txr;
4149                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4150                         if (i >= bp->tx_nr_rings_xdp) {
4151                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4152                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
4153                         } else {
4154                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4155                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4156                         }
4157                 }
4158
4159                 rc = bnxt_alloc_stats(bp);
4160                 if (rc)
4161                         goto alloc_mem_err;
4162                 bnxt_init_stats(bp);
4163
4164                 rc = bnxt_alloc_ntp_fltrs(bp);
4165                 if (rc)
4166                         goto alloc_mem_err;
4167
4168                 rc = bnxt_alloc_vnics(bp);
4169                 if (rc)
4170                         goto alloc_mem_err;
4171         }
4172
4173         bnxt_init_ring_struct(bp);
4174
4175         rc = bnxt_alloc_rx_rings(bp);
4176         if (rc)
4177                 goto alloc_mem_err;
4178
4179         rc = bnxt_alloc_tx_rings(bp);
4180         if (rc)
4181                 goto alloc_mem_err;
4182
4183         rc = bnxt_alloc_cp_rings(bp);
4184         if (rc)
4185                 goto alloc_mem_err;
4186
4187         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4188                                   BNXT_VNIC_UCAST_FLAG;
4189         rc = bnxt_alloc_vnic_attributes(bp);
4190         if (rc)
4191                 goto alloc_mem_err;
4192         return 0;
4193
4194 alloc_mem_err:
4195         bnxt_free_mem(bp, true);
4196         return rc;
4197 }
4198
4199 static void bnxt_disable_int(struct bnxt *bp)
4200 {
4201         int i;
4202
4203         if (!bp->bnapi)
4204                 return;
4205
4206         for (i = 0; i < bp->cp_nr_rings; i++) {
4207                 struct bnxt_napi *bnapi = bp->bnapi[i];
4208                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4209                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4210
4211                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4212                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4213         }
4214 }
4215
4216 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4217 {
4218         struct bnxt_napi *bnapi = bp->bnapi[n];
4219         struct bnxt_cp_ring_info *cpr;
4220
4221         cpr = &bnapi->cp_ring;
4222         return cpr->cp_ring_struct.map_idx;
4223 }
4224
4225 static void bnxt_disable_int_sync(struct bnxt *bp)
4226 {
4227         int i;
4228
4229         atomic_inc(&bp->intr_sem);
4230
4231         bnxt_disable_int(bp);
4232         for (i = 0; i < bp->cp_nr_rings; i++) {
4233                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4234
4235                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4236         }
4237 }
4238
4239 static void bnxt_enable_int(struct bnxt *bp)
4240 {
4241         int i;
4242
4243         atomic_set(&bp->intr_sem, 0);
4244         for (i = 0; i < bp->cp_nr_rings; i++) {
4245                 struct bnxt_napi *bnapi = bp->bnapi[i];
4246                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4247
4248                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4249         }
4250 }
4251
4252 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4253                             u16 cmpl_ring, u16 target_id)
4254 {
4255         struct input *req = request;
4256
4257         req->req_type = cpu_to_le16(req_type);
4258         req->cmpl_ring = cpu_to_le16(cmpl_ring);
4259         req->target_id = cpu_to_le16(target_id);
4260         if (bnxt_kong_hwrm_message(bp, req))
4261                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4262         else
4263                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4264 }
4265
4266 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4267 {
4268         switch (hwrm_err) {
4269         case HWRM_ERR_CODE_SUCCESS:
4270                 return 0;
4271         case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4272                 return -EACCES;
4273         case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4274                 return -ENOSPC;
4275         case HWRM_ERR_CODE_INVALID_PARAMS:
4276         case HWRM_ERR_CODE_INVALID_FLAGS:
4277         case HWRM_ERR_CODE_INVALID_ENABLES:
4278         case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4279         case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4280                 return -EINVAL;
4281         case HWRM_ERR_CODE_NO_BUFFER:
4282                 return -ENOMEM;
4283         case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4284         case HWRM_ERR_CODE_BUSY:
4285                 return -EAGAIN;
4286         case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4287                 return -EOPNOTSUPP;
4288         default:
4289                 return -EIO;
4290         }
4291 }
4292
4293 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4294                                  int timeout, bool silent)
4295 {
4296         int i, intr_process, rc, tmo_count;
4297         struct input *req = msg;
4298         u32 *data = msg;
4299         u8 *valid;
4300         u16 cp_ring_id, len = 0;
4301         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4302         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4303         struct hwrm_short_input short_input = {0};
4304         u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4305         u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4306         u16 dst = BNXT_HWRM_CHNL_CHIMP;
4307
4308         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4309                 return -EBUSY;
4310
4311         if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4312                 if (msg_len > bp->hwrm_max_ext_req_len ||
4313                     !bp->hwrm_short_cmd_req_addr)
4314                         return -EINVAL;
4315         }
4316
4317         if (bnxt_hwrm_kong_chnl(bp, req)) {
4318                 dst = BNXT_HWRM_CHNL_KONG;
4319                 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4320                 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4321                 resp = bp->hwrm_cmd_kong_resp_addr;
4322         }
4323
4324         memset(resp, 0, PAGE_SIZE);
4325         cp_ring_id = le16_to_cpu(req->cmpl_ring);
4326         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4327
4328         req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4329         /* currently supports only one outstanding message */
4330         if (intr_process)
4331                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4332
4333         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4334             msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4335                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4336                 u16 max_msg_len;
4337
4338                 /* Set boundary for maximum extended request length for short
4339                  * cmd format. If passed up from device use the max supported
4340                  * internal req length.
4341                  */
4342                 max_msg_len = bp->hwrm_max_ext_req_len;
4343
4344                 memcpy(short_cmd_req, req, msg_len);
4345                 if (msg_len < max_msg_len)
4346                         memset(short_cmd_req + msg_len, 0,
4347                                max_msg_len - msg_len);
4348
4349                 short_input.req_type = req->req_type;
4350                 short_input.signature =
4351                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4352                 short_input.size = cpu_to_le16(msg_len);
4353                 short_input.req_addr =
4354                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4355
4356                 data = (u32 *)&short_input;
4357                 msg_len = sizeof(short_input);
4358
4359                 /* Sync memory write before updating doorbell */
4360                 wmb();
4361
4362                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4363         }
4364
4365         /* Write request msg to hwrm channel */
4366         __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4367
4368         for (i = msg_len; i < max_req_len; i += 4)
4369                 writel(0, bp->bar0 + bar_offset + i);
4370
4371         /* Ring channel doorbell */
4372         writel(1, bp->bar0 + doorbell_offset);
4373
4374         if (!pci_is_enabled(bp->pdev))
4375                 return 0;
4376
4377         if (!timeout)
4378                 timeout = DFLT_HWRM_CMD_TIMEOUT;
4379         /* convert timeout to usec */
4380         timeout *= 1000;
4381
4382         i = 0;
4383         /* Short timeout for the first few iterations:
4384          * number of loops = number of loops for short timeout +
4385          * number of loops for standard timeout.
4386          */
4387         tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4388         timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4389         tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4390
4391         if (intr_process) {
4392                 u16 seq_id = bp->hwrm_intr_seq_id;
4393
4394                 /* Wait until hwrm response cmpl interrupt is processed */
4395                 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4396                        i++ < tmo_count) {
4397                         /* Abort the wait for completion if the FW health
4398                          * check has failed.
4399                          */
4400                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4401                                 return -EBUSY;
4402                         /* on first few passes, just barely sleep */
4403                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4404                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4405                                              HWRM_SHORT_MAX_TIMEOUT);
4406                         else
4407                                 usleep_range(HWRM_MIN_TIMEOUT,
4408                                              HWRM_MAX_TIMEOUT);
4409                 }
4410
4411                 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4412                         if (!silent)
4413                                 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4414                                            le16_to_cpu(req->req_type));
4415                         return -EBUSY;
4416                 }
4417                 len = le16_to_cpu(resp->resp_len);
4418                 valid = ((u8 *)resp) + len - 1;
4419         } else {
4420                 int j;
4421
4422                 /* Check if response len is updated */
4423                 for (i = 0; i < tmo_count; i++) {
4424                         /* Abort the wait for completion if the FW health
4425                          * check has failed.
4426                          */
4427                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4428                                 return -EBUSY;
4429                         len = le16_to_cpu(resp->resp_len);
4430                         if (len)
4431                                 break;
4432                         /* on first few passes, just barely sleep */
4433                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4434                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4435                                              HWRM_SHORT_MAX_TIMEOUT);
4436                         else
4437                                 usleep_range(HWRM_MIN_TIMEOUT,
4438                                              HWRM_MAX_TIMEOUT);
4439                 }
4440
4441                 if (i >= tmo_count) {
4442                         if (!silent)
4443                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4444                                            HWRM_TOTAL_TIMEOUT(i),
4445                                            le16_to_cpu(req->req_type),
4446                                            le16_to_cpu(req->seq_id), len);
4447                         return -EBUSY;
4448                 }
4449
4450                 /* Last byte of resp contains valid bit */
4451                 valid = ((u8 *)resp) + len - 1;
4452                 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4453                         /* make sure we read from updated DMA memory */
4454                         dma_rmb();
4455                         if (*valid)
4456                                 break;
4457                         usleep_range(1, 5);
4458                 }
4459
4460                 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4461                         if (!silent)
4462                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4463                                            HWRM_TOTAL_TIMEOUT(i),
4464                                            le16_to_cpu(req->req_type),
4465                                            le16_to_cpu(req->seq_id), len,
4466                                            *valid);
4467                         return -EBUSY;
4468                 }
4469         }
4470
4471         /* Zero valid bit for compatibility.  Valid bit in an older spec
4472          * may become a new field in a newer spec.  We must make sure that
4473          * a new field not implemented by old spec will read zero.
4474          */
4475         *valid = 0;
4476         rc = le16_to_cpu(resp->error_code);
4477         if (rc && !silent)
4478                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4479                            le16_to_cpu(resp->req_type),
4480                            le16_to_cpu(resp->seq_id), rc);
4481         return bnxt_hwrm_to_stderr(rc);
4482 }
4483
4484 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4485 {
4486         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4487 }
4488
4489 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4490                               int timeout)
4491 {
4492         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4493 }
4494
4495 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4496 {
4497         int rc;
4498
4499         mutex_lock(&bp->hwrm_cmd_lock);
4500         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4501         mutex_unlock(&bp->hwrm_cmd_lock);
4502         return rc;
4503 }
4504
4505 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4506                              int timeout)
4507 {
4508         int rc;
4509
4510         mutex_lock(&bp->hwrm_cmd_lock);
4511         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4512         mutex_unlock(&bp->hwrm_cmd_lock);
4513         return rc;
4514 }
4515
4516 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4517                             bool async_only)
4518 {
4519         struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4520         struct hwrm_func_drv_rgtr_input req = {0};
4521         DECLARE_BITMAP(async_events_bmap, 256);
4522         u32 *events = (u32 *)async_events_bmap;
4523         u32 flags;
4524         int rc, i;
4525
4526         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4527
4528         req.enables =
4529                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4530                             FUNC_DRV_RGTR_REQ_ENABLES_VER |
4531                             FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4532
4533         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4534         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4535         if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4536                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4537         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4538                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4539                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4540         req.flags = cpu_to_le32(flags);
4541         req.ver_maj_8b = DRV_VER_MAJ;
4542         req.ver_min_8b = DRV_VER_MIN;
4543         req.ver_upd_8b = DRV_VER_UPD;
4544         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4545         req.ver_min = cpu_to_le16(DRV_VER_MIN);
4546         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4547
4548         if (BNXT_PF(bp)) {
4549                 u32 data[8];
4550                 int i;
4551
4552                 memset(data, 0, sizeof(data));
4553                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4554                         u16 cmd = bnxt_vf_req_snif[i];
4555                         unsigned int bit, idx;
4556
4557                         idx = cmd / 32;
4558                         bit = cmd % 32;
4559                         data[idx] |= 1 << bit;
4560                 }
4561
4562                 for (i = 0; i < 8; i++)
4563                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4564
4565                 req.enables |=
4566                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4567         }
4568
4569         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4570                 req.flags |= cpu_to_le32(
4571                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4572
4573         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4574         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4575                 u16 event_id = bnxt_async_events_arr[i];
4576
4577                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4578                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4579                         continue;
4580                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4581         }
4582         if (bmap && bmap_size) {
4583                 for (i = 0; i < bmap_size; i++) {
4584                         if (test_bit(i, bmap))
4585                                 __set_bit(i, async_events_bmap);
4586                 }
4587         }
4588         for (i = 0; i < 8; i++)
4589                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4590
4591         if (async_only)
4592                 req.enables =
4593                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4594
4595         mutex_lock(&bp->hwrm_cmd_lock);
4596         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4597         if (!rc) {
4598                 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4599                 if (resp->flags &
4600                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4601                         bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4602         }
4603         mutex_unlock(&bp->hwrm_cmd_lock);
4604         return rc;
4605 }
4606
4607 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4608 {
4609         struct hwrm_func_drv_unrgtr_input req = {0};
4610
4611         if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4612                 return 0;
4613
4614         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4615         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4616 }
4617
4618 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4619 {
4620         u32 rc = 0;
4621         struct hwrm_tunnel_dst_port_free_input req = {0};
4622
4623         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4624         req.tunnel_type = tunnel_type;
4625
4626         switch (tunnel_type) {
4627         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4628                 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4629                 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4630                 break;
4631         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4632                 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4633                 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4634                 break;
4635         default:
4636                 break;
4637         }
4638
4639         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4640         if (rc)
4641                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4642                            rc);
4643         return rc;
4644 }
4645
4646 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4647                                            u8 tunnel_type)
4648 {
4649         u32 rc = 0;
4650         struct hwrm_tunnel_dst_port_alloc_input req = {0};
4651         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4652
4653         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4654
4655         req.tunnel_type = tunnel_type;
4656         req.tunnel_dst_port_val = port;
4657
4658         mutex_lock(&bp->hwrm_cmd_lock);
4659         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4660         if (rc) {
4661                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4662                            rc);
4663                 goto err_out;
4664         }
4665
4666         switch (tunnel_type) {
4667         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4668                 bp->vxlan_fw_dst_port_id =
4669                         le16_to_cpu(resp->tunnel_dst_port_id);
4670                 break;
4671         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4672                 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4673                 break;
4674         default:
4675                 break;
4676         }
4677
4678 err_out:
4679         mutex_unlock(&bp->hwrm_cmd_lock);
4680         return rc;
4681 }
4682
4683 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4684 {
4685         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4686         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4687
4688         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4689         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4690
4691         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4692         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4693         req.mask = cpu_to_le32(vnic->rx_mask);
4694         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4695 }
4696
4697 #ifdef CONFIG_RFS_ACCEL
4698 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4699                                             struct bnxt_ntuple_filter *fltr)
4700 {
4701         struct hwrm_cfa_ntuple_filter_free_input req = {0};
4702
4703         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4704         req.ntuple_filter_id = fltr->filter_id;
4705         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4706 }
4707
4708 #define BNXT_NTP_FLTR_FLAGS                                     \
4709         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4710          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4711          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4712          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4713          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4714          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4715          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4716          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4717          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4718          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4719          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4720          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4721          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4722          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4723
4724 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4725                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4726
4727 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4728                                              struct bnxt_ntuple_filter *fltr)
4729 {
4730         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4731         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4732         struct flow_keys *keys = &fltr->fkeys;
4733         struct bnxt_vnic_info *vnic;
4734         u32 flags = 0;
4735         int rc = 0;
4736
4737         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4738         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4739
4740         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4741                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4742                 req.dst_id = cpu_to_le16(fltr->rxq);
4743         } else {
4744                 vnic = &bp->vnic_info[fltr->rxq + 1];
4745                 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4746         }
4747         req.flags = cpu_to_le32(flags);
4748         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4749
4750         req.ethertype = htons(ETH_P_IP);
4751         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4752         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4753         req.ip_protocol = keys->basic.ip_proto;
4754
4755         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4756                 int i;
4757
4758                 req.ethertype = htons(ETH_P_IPV6);
4759                 req.ip_addr_type =
4760                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4761                 *(struct in6_addr *)&req.src_ipaddr[0] =
4762                         keys->addrs.v6addrs.src;
4763                 *(struct in6_addr *)&req.dst_ipaddr[0] =
4764                         keys->addrs.v6addrs.dst;
4765                 for (i = 0; i < 4; i++) {
4766                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4767                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4768                 }
4769         } else {
4770                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4771                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4772                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4773                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4774         }
4775         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4776                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4777                 req.tunnel_type =
4778                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4779         }
4780
4781         req.src_port = keys->ports.src;
4782         req.src_port_mask = cpu_to_be16(0xffff);
4783         req.dst_port = keys->ports.dst;
4784         req.dst_port_mask = cpu_to_be16(0xffff);
4785
4786         mutex_lock(&bp->hwrm_cmd_lock);
4787         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4788         if (!rc) {
4789                 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4790                 fltr->filter_id = resp->ntuple_filter_id;
4791         }
4792         mutex_unlock(&bp->hwrm_cmd_lock);
4793         return rc;
4794 }
4795 #endif
4796
4797 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4798                                      u8 *mac_addr)
4799 {
4800         u32 rc = 0;
4801         struct hwrm_cfa_l2_filter_alloc_input req = {0};
4802         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4803
4804         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4805         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4806         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4807                 req.flags |=
4808                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4809         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4810         req.enables =
4811                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4812                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4813                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4814         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4815         req.l2_addr_mask[0] = 0xff;
4816         req.l2_addr_mask[1] = 0xff;
4817         req.l2_addr_mask[2] = 0xff;
4818         req.l2_addr_mask[3] = 0xff;
4819         req.l2_addr_mask[4] = 0xff;
4820         req.l2_addr_mask[5] = 0xff;
4821
4822         mutex_lock(&bp->hwrm_cmd_lock);
4823         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4824         if (!rc)
4825                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4826                                                         resp->l2_filter_id;
4827         mutex_unlock(&bp->hwrm_cmd_lock);
4828         return rc;
4829 }
4830
4831 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4832 {
4833         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4834         int rc = 0;
4835
4836         /* Any associated ntuple filters will also be cleared by firmware. */
4837         mutex_lock(&bp->hwrm_cmd_lock);
4838         for (i = 0; i < num_of_vnics; i++) {
4839                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4840
4841                 for (j = 0; j < vnic->uc_filter_count; j++) {
4842                         struct hwrm_cfa_l2_filter_free_input req = {0};
4843
4844                         bnxt_hwrm_cmd_hdr_init(bp, &req,
4845                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
4846
4847                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
4848
4849                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4850                                                 HWRM_CMD_TIMEOUT);
4851                 }
4852                 vnic->uc_filter_count = 0;
4853         }
4854         mutex_unlock(&bp->hwrm_cmd_lock);
4855
4856         return rc;
4857 }
4858
4859 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4860 {
4861         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4862         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4863         struct hwrm_vnic_tpa_cfg_input req = {0};
4864
4865         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4866                 return 0;
4867
4868         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4869
4870         if (tpa_flags) {
4871                 u16 mss = bp->dev->mtu - 40;
4872                 u32 nsegs, n, segs = 0, flags;
4873
4874                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4875                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4876                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4877                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4878                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4879                 if (tpa_flags & BNXT_FLAG_GRO)
4880                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4881
4882                 req.flags = cpu_to_le32(flags);
4883
4884                 req.enables =
4885                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4886                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4887                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4888
4889                 /* Number of segs are log2 units, and first packet is not
4890                  * included as part of this units.
4891                  */
4892                 if (mss <= BNXT_RX_PAGE_SIZE) {
4893                         n = BNXT_RX_PAGE_SIZE / mss;
4894                         nsegs = (MAX_SKB_FRAGS - 1) * n;
4895                 } else {
4896                         n = mss / BNXT_RX_PAGE_SIZE;
4897                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
4898                                 n++;
4899                         nsegs = (MAX_SKB_FRAGS - n) / n;
4900                 }
4901
4902                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4903                         segs = MAX_TPA_SEGS_P5;
4904                         max_aggs = bp->max_tpa;
4905                 } else {
4906                         segs = ilog2(nsegs);
4907                 }
4908                 req.max_agg_segs = cpu_to_le16(segs);
4909                 req.max_aggs = cpu_to_le16(max_aggs);
4910
4911                 req.min_agg_len = cpu_to_le32(512);
4912         }
4913         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4914
4915         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4916 }
4917
4918 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4919 {
4920         struct bnxt_ring_grp_info *grp_info;
4921
4922         grp_info = &bp->grp_info[ring->grp_idx];
4923         return grp_info->cp_fw_ring_id;
4924 }
4925
4926 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4927 {
4928         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4929                 struct bnxt_napi *bnapi = rxr->bnapi;
4930                 struct bnxt_cp_ring_info *cpr;
4931
4932                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4933                 return cpr->cp_ring_struct.fw_ring_id;
4934         } else {
4935                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4936         }
4937 }
4938
4939 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4940 {
4941         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4942                 struct bnxt_napi *bnapi = txr->bnapi;
4943                 struct bnxt_cp_ring_info *cpr;
4944
4945                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4946                 return cpr->cp_ring_struct.fw_ring_id;
4947         } else {
4948                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4949         }
4950 }
4951
4952 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
4953 {
4954         int entries;
4955
4956         if (bp->flags & BNXT_FLAG_CHIP_P5)
4957                 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
4958         else
4959                 entries = HW_HASH_INDEX_SIZE;
4960
4961         bp->rss_indir_tbl_entries = entries;
4962         bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
4963                                           GFP_KERNEL);
4964         if (!bp->rss_indir_tbl)
4965                 return -ENOMEM;
4966         return 0;
4967 }
4968
4969 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
4970 {
4971         u16 max_rings, max_entries, pad, i;
4972
4973         if (!bp->rx_nr_rings)
4974                 return;
4975
4976         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4977                 max_rings = bp->rx_nr_rings - 1;
4978         else
4979                 max_rings = bp->rx_nr_rings;
4980
4981         max_entries = bnxt_get_rxfh_indir_size(bp->dev);
4982
4983         for (i = 0; i < max_entries; i++)
4984                 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
4985
4986         pad = bp->rss_indir_tbl_entries - max_entries;
4987         if (pad)
4988                 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
4989 }
4990
4991 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
4992 {
4993         u16 i, tbl_size, max_ring = 0;
4994
4995         if (!bp->rss_indir_tbl)
4996                 return 0;
4997
4998         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
4999         for (i = 0; i < tbl_size; i++)
5000                 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5001         return max_ring;
5002 }
5003
5004 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5005 {
5006         if (bp->flags & BNXT_FLAG_CHIP_P5)
5007                 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5008         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5009                 return 2;
5010         return 1;
5011 }
5012
5013 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5014 {
5015         bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5016         u16 i, j;
5017
5018         /* Fill the RSS indirection table with ring group ids */
5019         for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5020                 if (!no_rss)
5021                         j = bp->rss_indir_tbl[i];
5022                 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5023         }
5024 }
5025
5026 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5027                                       struct bnxt_vnic_info *vnic)
5028 {
5029         __le16 *ring_tbl = vnic->rss_table;
5030         struct bnxt_rx_ring_info *rxr;
5031         u16 tbl_size, i;
5032
5033         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5034
5035         for (i = 0; i < tbl_size; i++) {
5036                 u16 ring_id, j;
5037
5038                 j = bp->rss_indir_tbl[i];
5039                 rxr = &bp->rx_ring[j];
5040
5041                 ring_id = rxr->rx_ring_struct.fw_ring_id;
5042                 *ring_tbl++ = cpu_to_le16(ring_id);
5043                 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5044                 *ring_tbl++ = cpu_to_le16(ring_id);
5045         }
5046 }
5047
5048 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5049 {
5050         if (bp->flags & BNXT_FLAG_CHIP_P5)
5051                 __bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5052         else
5053                 __bnxt_fill_hw_rss_tbl(bp, vnic);
5054 }
5055
5056 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5057 {
5058         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5059         struct hwrm_vnic_rss_cfg_input req = {0};
5060
5061         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5062             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5063                 return 0;
5064
5065         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5066         if (set_rss) {
5067                 bnxt_fill_hw_rss_tbl(bp, vnic);
5068                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5069                 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5070                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5071                 req.hash_key_tbl_addr =
5072                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
5073         }
5074         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5075         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5076 }
5077
5078 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5079 {
5080         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5081         struct hwrm_vnic_rss_cfg_input req = {0};
5082         dma_addr_t ring_tbl_map;
5083         u32 i, nr_ctxs;
5084
5085         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5086         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5087         if (!set_rss) {
5088                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5089                 return 0;
5090         }
5091         bnxt_fill_hw_rss_tbl(bp, vnic);
5092         req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5093         req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5094         req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5095         ring_tbl_map = vnic->rss_table_dma_addr;
5096         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5097         for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5098                 int rc;
5099
5100                 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5101                 req.ring_table_pair_index = i;
5102                 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5103                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5104                 if (rc)
5105                         return rc;
5106         }
5107         return 0;
5108 }
5109
5110 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5111 {
5112         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5113         struct hwrm_vnic_plcmodes_cfg_input req = {0};
5114
5115         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
5116         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
5117                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5118                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5119         req.enables =
5120                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
5121                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5122         /* thresholds not implemented in firmware yet */
5123         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5124         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5125         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5126         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5127 }
5128
5129 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5130                                         u16 ctx_idx)
5131 {
5132         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
5133
5134         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
5135         req.rss_cos_lb_ctx_id =
5136                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5137
5138         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5139         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5140 }
5141
5142 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5143 {
5144         int i, j;
5145
5146         for (i = 0; i < bp->nr_vnics; i++) {
5147                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5148
5149                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5150                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5151                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5152                 }
5153         }
5154         bp->rsscos_nr_ctxs = 0;
5155 }
5156
5157 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5158 {
5159         int rc;
5160         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
5161         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
5162                                                 bp->hwrm_cmd_resp_addr;
5163
5164         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
5165                                -1);
5166
5167         mutex_lock(&bp->hwrm_cmd_lock);
5168         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5169         if (!rc)
5170                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5171                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
5172         mutex_unlock(&bp->hwrm_cmd_lock);
5173
5174         return rc;
5175 }
5176
5177 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5178 {
5179         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5180                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5181         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5182 }
5183
5184 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5185 {
5186         unsigned int ring = 0, grp_idx;
5187         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5188         struct hwrm_vnic_cfg_input req = {0};
5189         u16 def_vlan = 0;
5190
5191         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
5192
5193         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5194                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5195
5196                 req.default_rx_ring_id =
5197                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5198                 req.default_cmpl_ring_id =
5199                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5200                 req.enables =
5201                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5202                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5203                 goto vnic_mru;
5204         }
5205         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5206         /* Only RSS support for now TBD: COS & LB */
5207         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5208                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5209                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5210                                            VNIC_CFG_REQ_ENABLES_MRU);
5211         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5212                 req.rss_rule =
5213                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5214                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5215                                            VNIC_CFG_REQ_ENABLES_MRU);
5216                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5217         } else {
5218                 req.rss_rule = cpu_to_le16(0xffff);
5219         }
5220
5221         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5222             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5223                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5224                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5225         } else {
5226                 req.cos_rule = cpu_to_le16(0xffff);
5227         }
5228
5229         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5230                 ring = 0;
5231         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5232                 ring = vnic_id - 1;
5233         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5234                 ring = bp->rx_nr_rings - 1;
5235
5236         grp_idx = bp->rx_ring[ring].bnapi->index;
5237         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5238         req.lb_rule = cpu_to_le16(0xffff);
5239 vnic_mru:
5240         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5241
5242         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5243 #ifdef CONFIG_BNXT_SRIOV
5244         if (BNXT_VF(bp))
5245                 def_vlan = bp->vf.vlan;
5246 #endif
5247         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5248                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5249         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5250                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5251
5252         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5253 }
5254
5255 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5256 {
5257         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5258                 struct hwrm_vnic_free_input req = {0};
5259
5260                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5261                 req.vnic_id =
5262                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5263
5264                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5265                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5266         }
5267 }
5268
5269 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5270 {
5271         u16 i;
5272
5273         for (i = 0; i < bp->nr_vnics; i++)
5274                 bnxt_hwrm_vnic_free_one(bp, i);
5275 }
5276
5277 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5278                                 unsigned int start_rx_ring_idx,
5279                                 unsigned int nr_rings)
5280 {
5281         int rc = 0;
5282         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5283         struct hwrm_vnic_alloc_input req = {0};
5284         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5285         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5286
5287         if (bp->flags & BNXT_FLAG_CHIP_P5)
5288                 goto vnic_no_ring_grps;
5289
5290         /* map ring groups to this vnic */
5291         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5292                 grp_idx = bp->rx_ring[i].bnapi->index;
5293                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5294                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5295                                    j, nr_rings);
5296                         break;
5297                 }
5298                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5299         }
5300
5301 vnic_no_ring_grps:
5302         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5303                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5304         if (vnic_id == 0)
5305                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5306
5307         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5308
5309         mutex_lock(&bp->hwrm_cmd_lock);
5310         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5311         if (!rc)
5312                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5313         mutex_unlock(&bp->hwrm_cmd_lock);
5314         return rc;
5315 }
5316
5317 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5318 {
5319         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5320         struct hwrm_vnic_qcaps_input req = {0};
5321         int rc;
5322
5323         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5324         bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5325         if (bp->hwrm_spec_code < 0x10600)
5326                 return 0;
5327
5328         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5329         mutex_lock(&bp->hwrm_cmd_lock);
5330         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5331         if (!rc) {
5332                 u32 flags = le32_to_cpu(resp->flags);
5333
5334                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5335                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5336                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5337                 if (flags &
5338                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5339                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5340
5341                 /* Older P5 fw before EXT_HW_STATS support did not set
5342                  * VLAN_STRIP_CAP properly.
5343                  */
5344                 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5345                     ((bp->flags & BNXT_FLAG_CHIP_P5) &&
5346                      !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5347                         bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5348                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5349                 if (bp->max_tpa_v2)
5350                         bp->hw_ring_stats_size =
5351                                 sizeof(struct ctx_hw_stats_ext);
5352         }
5353         mutex_unlock(&bp->hwrm_cmd_lock);
5354         return rc;
5355 }
5356
5357 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5358 {
5359         u16 i;
5360         u32 rc = 0;
5361
5362         if (bp->flags & BNXT_FLAG_CHIP_P5)
5363                 return 0;
5364
5365         mutex_lock(&bp->hwrm_cmd_lock);
5366         for (i = 0; i < bp->rx_nr_rings; i++) {
5367                 struct hwrm_ring_grp_alloc_input req = {0};
5368                 struct hwrm_ring_grp_alloc_output *resp =
5369                                         bp->hwrm_cmd_resp_addr;
5370                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5371
5372                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5373
5374                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5375                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5376                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5377                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5378
5379                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5380                                         HWRM_CMD_TIMEOUT);
5381                 if (rc)
5382                         break;
5383
5384                 bp->grp_info[grp_idx].fw_grp_id =
5385                         le32_to_cpu(resp->ring_group_id);
5386         }
5387         mutex_unlock(&bp->hwrm_cmd_lock);
5388         return rc;
5389 }
5390
5391 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5392 {
5393         u16 i;
5394         struct hwrm_ring_grp_free_input req = {0};
5395
5396         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5397                 return;
5398
5399         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5400
5401         mutex_lock(&bp->hwrm_cmd_lock);
5402         for (i = 0; i < bp->cp_nr_rings; i++) {
5403                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5404                         continue;
5405                 req.ring_group_id =
5406                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5407
5408                 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5409                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5410         }
5411         mutex_unlock(&bp->hwrm_cmd_lock);
5412 }
5413
5414 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5415                                     struct bnxt_ring_struct *ring,
5416                                     u32 ring_type, u32 map_index)
5417 {
5418         int rc = 0, err = 0;
5419         struct hwrm_ring_alloc_input req = {0};
5420         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5421         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5422         struct bnxt_ring_grp_info *grp_info;
5423         u16 ring_id;
5424
5425         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5426
5427         req.enables = 0;
5428         if (rmem->nr_pages > 1) {
5429                 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5430                 /* Page size is in log2 units */
5431                 req.page_size = BNXT_PAGE_SHIFT;
5432                 req.page_tbl_depth = 1;
5433         } else {
5434                 req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5435         }
5436         req.fbo = 0;
5437         /* Association of ring index with doorbell index and MSIX number */
5438         req.logical_id = cpu_to_le16(map_index);
5439
5440         switch (ring_type) {
5441         case HWRM_RING_ALLOC_TX: {
5442                 struct bnxt_tx_ring_info *txr;
5443
5444                 txr = container_of(ring, struct bnxt_tx_ring_info,
5445                                    tx_ring_struct);
5446                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5447                 /* Association of transmit ring with completion ring */
5448                 grp_info = &bp->grp_info[ring->grp_idx];
5449                 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5450                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5451                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5452                 req.queue_id = cpu_to_le16(ring->queue_id);
5453                 break;
5454         }
5455         case HWRM_RING_ALLOC_RX:
5456                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5457                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5458                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5459                         u16 flags = 0;
5460
5461                         /* Association of rx ring with stats context */
5462                         grp_info = &bp->grp_info[ring->grp_idx];
5463                         req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5464                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5465                         req.enables |= cpu_to_le32(
5466                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5467                         if (NET_IP_ALIGN == 2)
5468                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5469                         req.flags = cpu_to_le16(flags);
5470                 }
5471                 break;
5472         case HWRM_RING_ALLOC_AGG:
5473                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5474                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5475                         /* Association of agg ring with rx ring */
5476                         grp_info = &bp->grp_info[ring->grp_idx];
5477                         req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5478                         req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5479                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5480                         req.enables |= cpu_to_le32(
5481                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5482                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5483                 } else {
5484                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5485                 }
5486                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5487                 break;
5488         case HWRM_RING_ALLOC_CMPL:
5489                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5490                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5491                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5492                         /* Association of cp ring with nq */
5493                         grp_info = &bp->grp_info[map_index];
5494                         req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5495                         req.cq_handle = cpu_to_le64(ring->handle);
5496                         req.enables |= cpu_to_le32(
5497                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5498                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5499                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5500                 }
5501                 break;
5502         case HWRM_RING_ALLOC_NQ:
5503                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5504                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5505                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5506                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5507                 break;
5508         default:
5509                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5510                            ring_type);
5511                 return -1;
5512         }
5513
5514         mutex_lock(&bp->hwrm_cmd_lock);
5515         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5516         err = le16_to_cpu(resp->error_code);
5517         ring_id = le16_to_cpu(resp->ring_id);
5518         mutex_unlock(&bp->hwrm_cmd_lock);
5519
5520         if (rc || err) {
5521                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5522                            ring_type, rc, err);
5523                 return -EIO;
5524         }
5525         ring->fw_ring_id = ring_id;
5526         return rc;
5527 }
5528
5529 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5530 {
5531         int rc;
5532
5533         if (BNXT_PF(bp)) {
5534                 struct hwrm_func_cfg_input req = {0};
5535
5536                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5537                 req.fid = cpu_to_le16(0xffff);
5538                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5539                 req.async_event_cr = cpu_to_le16(idx);
5540                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5541         } else {
5542                 struct hwrm_func_vf_cfg_input req = {0};
5543
5544                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5545                 req.enables =
5546                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5547                 req.async_event_cr = cpu_to_le16(idx);
5548                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5549         }
5550         return rc;
5551 }
5552
5553 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5554                         u32 map_idx, u32 xid)
5555 {
5556         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5557                 if (BNXT_PF(bp))
5558                         db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5559                 else
5560                         db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5561                 switch (ring_type) {
5562                 case HWRM_RING_ALLOC_TX:
5563                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5564                         break;
5565                 case HWRM_RING_ALLOC_RX:
5566                 case HWRM_RING_ALLOC_AGG:
5567                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5568                         break;
5569                 case HWRM_RING_ALLOC_CMPL:
5570                         db->db_key64 = DBR_PATH_L2;
5571                         break;
5572                 case HWRM_RING_ALLOC_NQ:
5573                         db->db_key64 = DBR_PATH_L2;
5574                         break;
5575                 }
5576                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5577         } else {
5578                 db->doorbell = bp->bar1 + map_idx * 0x80;
5579                 switch (ring_type) {
5580                 case HWRM_RING_ALLOC_TX:
5581                         db->db_key32 = DB_KEY_TX;
5582                         break;
5583                 case HWRM_RING_ALLOC_RX:
5584                 case HWRM_RING_ALLOC_AGG:
5585                         db->db_key32 = DB_KEY_RX;
5586                         break;
5587                 case HWRM_RING_ALLOC_CMPL:
5588                         db->db_key32 = DB_KEY_CP;
5589                         break;
5590                 }
5591         }
5592 }
5593
5594 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5595 {
5596         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5597         int i, rc = 0;
5598         u32 type;
5599
5600         if (bp->flags & BNXT_FLAG_CHIP_P5)
5601                 type = HWRM_RING_ALLOC_NQ;
5602         else
5603                 type = HWRM_RING_ALLOC_CMPL;
5604         for (i = 0; i < bp->cp_nr_rings; i++) {
5605                 struct bnxt_napi *bnapi = bp->bnapi[i];
5606                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5607                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5608                 u32 map_idx = ring->map_idx;
5609                 unsigned int vector;
5610
5611                 vector = bp->irq_tbl[map_idx].vector;
5612                 disable_irq_nosync(vector);
5613                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5614                 if (rc) {
5615                         enable_irq(vector);
5616                         goto err_out;
5617                 }
5618                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5619                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5620                 enable_irq(vector);
5621                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5622
5623                 if (!i) {
5624                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5625                         if (rc)
5626                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5627                 }
5628         }
5629
5630         type = HWRM_RING_ALLOC_TX;
5631         for (i = 0; i < bp->tx_nr_rings; i++) {
5632                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5633                 struct bnxt_ring_struct *ring;
5634                 u32 map_idx;
5635
5636                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5637                         struct bnxt_napi *bnapi = txr->bnapi;
5638                         struct bnxt_cp_ring_info *cpr, *cpr2;
5639                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5640
5641                         cpr = &bnapi->cp_ring;
5642                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5643                         ring = &cpr2->cp_ring_struct;
5644                         ring->handle = BNXT_TX_HDL;
5645                         map_idx = bnapi->index;
5646                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5647                         if (rc)
5648                                 goto err_out;
5649                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5650                                     ring->fw_ring_id);
5651                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5652                 }
5653                 ring = &txr->tx_ring_struct;
5654                 map_idx = i;
5655                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5656                 if (rc)
5657                         goto err_out;
5658                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5659         }
5660
5661         type = HWRM_RING_ALLOC_RX;
5662         for (i = 0; i < bp->rx_nr_rings; i++) {
5663                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5664                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5665                 struct bnxt_napi *bnapi = rxr->bnapi;
5666                 u32 map_idx = bnapi->index;
5667
5668                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5669                 if (rc)
5670                         goto err_out;
5671                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5672                 /* If we have agg rings, post agg buffers first. */
5673                 if (!agg_rings)
5674                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5675                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5676                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5677                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5678                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5679                         struct bnxt_cp_ring_info *cpr2;
5680
5681                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5682                         ring = &cpr2->cp_ring_struct;
5683                         ring->handle = BNXT_RX_HDL;
5684                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5685                         if (rc)
5686                                 goto err_out;
5687                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5688                                     ring->fw_ring_id);
5689                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5690                 }
5691         }
5692
5693         if (agg_rings) {
5694                 type = HWRM_RING_ALLOC_AGG;
5695                 for (i = 0; i < bp->rx_nr_rings; i++) {
5696                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5697                         struct bnxt_ring_struct *ring =
5698                                                 &rxr->rx_agg_ring_struct;
5699                         u32 grp_idx = ring->grp_idx;
5700                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5701
5702                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5703                         if (rc)
5704                                 goto err_out;
5705
5706                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5707                                     ring->fw_ring_id);
5708                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5709                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5710                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5711                 }
5712         }
5713 err_out:
5714         return rc;
5715 }
5716
5717 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5718                                    struct bnxt_ring_struct *ring,
5719                                    u32 ring_type, int cmpl_ring_id)
5720 {
5721         int rc;
5722         struct hwrm_ring_free_input req = {0};
5723         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5724         u16 error_code;
5725
5726         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
5727                 return 0;
5728
5729         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5730         req.ring_type = ring_type;
5731         req.ring_id = cpu_to_le16(ring->fw_ring_id);
5732
5733         mutex_lock(&bp->hwrm_cmd_lock);
5734         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5735         error_code = le16_to_cpu(resp->error_code);
5736         mutex_unlock(&bp->hwrm_cmd_lock);
5737
5738         if (rc || error_code) {
5739                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5740                            ring_type, rc, error_code);
5741                 return -EIO;
5742         }
5743         return 0;
5744 }
5745
5746 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5747 {
5748         u32 type;
5749         int i;
5750
5751         if (!bp->bnapi)
5752                 return;
5753
5754         for (i = 0; i < bp->tx_nr_rings; i++) {
5755                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5756                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5757
5758                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5759                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5760
5761                         hwrm_ring_free_send_msg(bp, ring,
5762                                                 RING_FREE_REQ_RING_TYPE_TX,
5763                                                 close_path ? cmpl_ring_id :
5764                                                 INVALID_HW_RING_ID);
5765                         ring->fw_ring_id = INVALID_HW_RING_ID;
5766                 }
5767         }
5768
5769         for (i = 0; i < bp->rx_nr_rings; i++) {
5770                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5771                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5772                 u32 grp_idx = rxr->bnapi->index;
5773
5774                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5775                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5776
5777                         hwrm_ring_free_send_msg(bp, ring,
5778                                                 RING_FREE_REQ_RING_TYPE_RX,
5779                                                 close_path ? cmpl_ring_id :
5780                                                 INVALID_HW_RING_ID);
5781                         ring->fw_ring_id = INVALID_HW_RING_ID;
5782                         bp->grp_info[grp_idx].rx_fw_ring_id =
5783                                 INVALID_HW_RING_ID;
5784                 }
5785         }
5786
5787         if (bp->flags & BNXT_FLAG_CHIP_P5)
5788                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5789         else
5790                 type = RING_FREE_REQ_RING_TYPE_RX;
5791         for (i = 0; i < bp->rx_nr_rings; i++) {
5792                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5793                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5794                 u32 grp_idx = rxr->bnapi->index;
5795
5796                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5797                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5798
5799                         hwrm_ring_free_send_msg(bp, ring, type,
5800                                                 close_path ? cmpl_ring_id :
5801                                                 INVALID_HW_RING_ID);
5802                         ring->fw_ring_id = INVALID_HW_RING_ID;
5803                         bp->grp_info[grp_idx].agg_fw_ring_id =
5804                                 INVALID_HW_RING_ID;
5805                 }
5806         }
5807
5808         /* The completion rings are about to be freed.  After that the
5809          * IRQ doorbell will not work anymore.  So we need to disable
5810          * IRQ here.
5811          */
5812         bnxt_disable_int_sync(bp);
5813
5814         if (bp->flags & BNXT_FLAG_CHIP_P5)
5815                 type = RING_FREE_REQ_RING_TYPE_NQ;
5816         else
5817                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5818         for (i = 0; i < bp->cp_nr_rings; i++) {
5819                 struct bnxt_napi *bnapi = bp->bnapi[i];
5820                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5821                 struct bnxt_ring_struct *ring;
5822                 int j;
5823
5824                 for (j = 0; j < 2; j++) {
5825                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5826
5827                         if (cpr2) {
5828                                 ring = &cpr2->cp_ring_struct;
5829                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5830                                         continue;
5831                                 hwrm_ring_free_send_msg(bp, ring,
5832                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
5833                                         INVALID_HW_RING_ID);
5834                                 ring->fw_ring_id = INVALID_HW_RING_ID;
5835                         }
5836                 }
5837                 ring = &cpr->cp_ring_struct;
5838                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5839                         hwrm_ring_free_send_msg(bp, ring, type,
5840                                                 INVALID_HW_RING_ID);
5841                         ring->fw_ring_id = INVALID_HW_RING_ID;
5842                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5843                 }
5844         }
5845 }
5846
5847 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5848                            bool shared);
5849
5850 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5851 {
5852         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5853         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5854         struct hwrm_func_qcfg_input req = {0};
5855         int rc;
5856
5857         if (bp->hwrm_spec_code < 0x10601)
5858                 return 0;
5859
5860         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5861         req.fid = cpu_to_le16(0xffff);
5862         mutex_lock(&bp->hwrm_cmd_lock);
5863         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5864         if (rc) {
5865                 mutex_unlock(&bp->hwrm_cmd_lock);
5866                 return rc;
5867         }
5868
5869         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5870         if (BNXT_NEW_RM(bp)) {
5871                 u16 cp, stats;
5872
5873                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5874                 hw_resc->resv_hw_ring_grps =
5875                         le32_to_cpu(resp->alloc_hw_ring_grps);
5876                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5877                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5878                 stats = le16_to_cpu(resp->alloc_stat_ctx);
5879                 hw_resc->resv_irqs = cp;
5880                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5881                         int rx = hw_resc->resv_rx_rings;
5882                         int tx = hw_resc->resv_tx_rings;
5883
5884                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5885                                 rx >>= 1;
5886                         if (cp < (rx + tx)) {
5887                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5888                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5889                                         rx <<= 1;
5890                                 hw_resc->resv_rx_rings = rx;
5891                                 hw_resc->resv_tx_rings = tx;
5892                         }
5893                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5894                         hw_resc->resv_hw_ring_grps = rx;
5895                 }
5896                 hw_resc->resv_cp_rings = cp;
5897                 hw_resc->resv_stat_ctxs = stats;
5898         }
5899         mutex_unlock(&bp->hwrm_cmd_lock);
5900         return 0;
5901 }
5902
5903 /* Caller must hold bp->hwrm_cmd_lock */
5904 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5905 {
5906         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5907         struct hwrm_func_qcfg_input req = {0};
5908         int rc;
5909
5910         if (bp->hwrm_spec_code < 0x10601)
5911                 return 0;
5912
5913         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5914         req.fid = cpu_to_le16(fid);
5915         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5916         if (!rc)
5917                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5918
5919         return rc;
5920 }
5921
5922 static bool bnxt_rfs_supported(struct bnxt *bp);
5923
5924 static void
5925 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5926                              int tx_rings, int rx_rings, int ring_grps,
5927                              int cp_rings, int stats, int vnics)
5928 {
5929         u32 enables = 0;
5930
5931         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5932         req->fid = cpu_to_le16(0xffff);
5933         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5934         req->num_tx_rings = cpu_to_le16(tx_rings);
5935         if (BNXT_NEW_RM(bp)) {
5936                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5937                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5938                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5939                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5940                         enables |= tx_rings + ring_grps ?
5941                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5942                         enables |= rx_rings ?
5943                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5944                 } else {
5945                         enables |= cp_rings ?
5946                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5947                         enables |= ring_grps ?
5948                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
5949                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5950                 }
5951                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
5952
5953                 req->num_rx_rings = cpu_to_le16(rx_rings);
5954                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5955                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5956                         req->num_msix = cpu_to_le16(cp_rings);
5957                         req->num_rsscos_ctxs =
5958                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5959                 } else {
5960                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
5961                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5962                         req->num_rsscos_ctxs = cpu_to_le16(1);
5963                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
5964                             bnxt_rfs_supported(bp))
5965                                 req->num_rsscos_ctxs =
5966                                         cpu_to_le16(ring_grps + 1);
5967                 }
5968                 req->num_stat_ctxs = cpu_to_le16(stats);
5969                 req->num_vnics = cpu_to_le16(vnics);
5970         }
5971         req->enables = cpu_to_le32(enables);
5972 }
5973
5974 static void
5975 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
5976                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
5977                              int rx_rings, int ring_grps, int cp_rings,
5978                              int stats, int vnics)
5979 {
5980         u32 enables = 0;
5981
5982         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
5983         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5984         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
5985                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5986         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5987         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5988                 enables |= tx_rings + ring_grps ?
5989                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5990         } else {
5991                 enables |= cp_rings ?
5992                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5993                 enables |= ring_grps ?
5994                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
5995         }
5996         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
5997         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
5998
5999         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6000         req->num_tx_rings = cpu_to_le16(tx_rings);
6001         req->num_rx_rings = cpu_to_le16(rx_rings);
6002         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6003                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6004                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6005         } else {
6006                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
6007                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6008                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6009         }
6010         req->num_stat_ctxs = cpu_to_le16(stats);
6011         req->num_vnics = cpu_to_le16(vnics);
6012
6013         req->enables = cpu_to_le32(enables);
6014 }
6015
6016 static int
6017 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6018                            int ring_grps, int cp_rings, int stats, int vnics)
6019 {
6020         struct hwrm_func_cfg_input req = {0};
6021         int rc;
6022
6023         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6024                                      cp_rings, stats, vnics);
6025         if (!req.enables)
6026                 return 0;
6027
6028         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6029         if (rc)
6030                 return rc;
6031
6032         if (bp->hwrm_spec_code < 0x10601)
6033                 bp->hw_resc.resv_tx_rings = tx_rings;
6034
6035         return bnxt_hwrm_get_rings(bp);
6036 }
6037
6038 static int
6039 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6040                            int ring_grps, int cp_rings, int stats, int vnics)
6041 {
6042         struct hwrm_func_vf_cfg_input req = {0};
6043         int rc;
6044
6045         if (!BNXT_NEW_RM(bp)) {
6046                 bp->hw_resc.resv_tx_rings = tx_rings;
6047                 return 0;
6048         }
6049
6050         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6051                                      cp_rings, stats, vnics);
6052         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6053         if (rc)
6054                 return rc;
6055
6056         return bnxt_hwrm_get_rings(bp);
6057 }
6058
6059 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6060                                    int cp, int stat, int vnic)
6061 {
6062         if (BNXT_PF(bp))
6063                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6064                                                   vnic);
6065         else
6066                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6067                                                   vnic);
6068 }
6069
6070 int bnxt_nq_rings_in_use(struct bnxt *bp)
6071 {
6072         int cp = bp->cp_nr_rings;
6073         int ulp_msix, ulp_base;
6074
6075         ulp_msix = bnxt_get_ulp_msix_num(bp);
6076         if (ulp_msix) {
6077                 ulp_base = bnxt_get_ulp_msix_base(bp);
6078                 cp += ulp_msix;
6079                 if ((ulp_base + ulp_msix) > cp)
6080                         cp = ulp_base + ulp_msix;
6081         }
6082         return cp;
6083 }
6084
6085 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6086 {
6087         int cp;
6088
6089         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6090                 return bnxt_nq_rings_in_use(bp);
6091
6092         cp = bp->tx_nr_rings + bp->rx_nr_rings;
6093         return cp;
6094 }
6095
6096 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6097 {
6098         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6099         int cp = bp->cp_nr_rings;
6100
6101         if (!ulp_stat)
6102                 return cp;
6103
6104         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6105                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6106
6107         return cp + ulp_stat;
6108 }
6109
6110 /* Check if a default RSS map needs to be setup.  This function is only
6111  * used on older firmware that does not require reserving RX rings.
6112  */
6113 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6114 {
6115         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6116
6117         /* The RSS map is valid for RX rings set to resv_rx_rings */
6118         if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6119                 hw_resc->resv_rx_rings = bp->rx_nr_rings;
6120                 if (!netif_is_rxfh_configured(bp->dev))
6121                         bnxt_set_dflt_rss_indir_tbl(bp);
6122         }
6123 }
6124
6125 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6126 {
6127         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6128         int cp = bnxt_cp_rings_in_use(bp);
6129         int nq = bnxt_nq_rings_in_use(bp);
6130         int rx = bp->rx_nr_rings, stat;
6131         int vnic = 1, grp = rx;
6132
6133         if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6134             bp->hwrm_spec_code >= 0x10601)
6135                 return true;
6136
6137         /* Old firmware does not need RX ring reservations but we still
6138          * need to setup a default RSS map when needed.  With new firmware
6139          * we go through RX ring reservations first and then set up the
6140          * RSS map for the successfully reserved RX rings when needed.
6141          */
6142         if (!BNXT_NEW_RM(bp)) {
6143                 bnxt_check_rss_tbl_no_rmgr(bp);
6144                 return false;
6145         }
6146         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6147                 vnic = rx + 1;
6148         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6149                 rx <<= 1;
6150         stat = bnxt_get_func_stat_ctxs(bp);
6151         if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6152             hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6153             (hw_resc->resv_hw_ring_grps != grp &&
6154              !(bp->flags & BNXT_FLAG_CHIP_P5)))
6155                 return true;
6156         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6157             hw_resc->resv_irqs != nq)
6158                 return true;
6159         return false;
6160 }
6161
6162 static int __bnxt_reserve_rings(struct bnxt *bp)
6163 {
6164         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6165         int cp = bnxt_nq_rings_in_use(bp);
6166         int tx = bp->tx_nr_rings;
6167         int rx = bp->rx_nr_rings;
6168         int grp, rx_rings, rc;
6169         int vnic = 1, stat;
6170         bool sh = false;
6171
6172         if (!bnxt_need_reserve_rings(bp))
6173                 return 0;
6174
6175         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6176                 sh = true;
6177         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6178                 vnic = rx + 1;
6179         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6180                 rx <<= 1;
6181         grp = bp->rx_nr_rings;
6182         stat = bnxt_get_func_stat_ctxs(bp);
6183
6184         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6185         if (rc)
6186                 return rc;
6187
6188         tx = hw_resc->resv_tx_rings;
6189         if (BNXT_NEW_RM(bp)) {
6190                 rx = hw_resc->resv_rx_rings;
6191                 cp = hw_resc->resv_irqs;
6192                 grp = hw_resc->resv_hw_ring_grps;
6193                 vnic = hw_resc->resv_vnics;
6194                 stat = hw_resc->resv_stat_ctxs;
6195         }
6196
6197         rx_rings = rx;
6198         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6199                 if (rx >= 2) {
6200                         rx_rings = rx >> 1;
6201                 } else {
6202                         if (netif_running(bp->dev))
6203                                 return -ENOMEM;
6204
6205                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6206                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6207                         bp->dev->hw_features &= ~NETIF_F_LRO;
6208                         bp->dev->features &= ~NETIF_F_LRO;
6209                         bnxt_set_ring_params(bp);
6210                 }
6211         }
6212         rx_rings = min_t(int, rx_rings, grp);
6213         cp = min_t(int, cp, bp->cp_nr_rings);
6214         if (stat > bnxt_get_ulp_stat_ctxs(bp))
6215                 stat -= bnxt_get_ulp_stat_ctxs(bp);
6216         cp = min_t(int, cp, stat);
6217         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6218         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6219                 rx = rx_rings << 1;
6220         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6221         bp->tx_nr_rings = tx;
6222
6223         /* If we cannot reserve all the RX rings, reset the RSS map only
6224          * if absolutely necessary
6225          */
6226         if (rx_rings != bp->rx_nr_rings) {
6227                 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6228                             rx_rings, bp->rx_nr_rings);
6229                 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) &&
6230                     (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6231                      bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6232                      bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6233                         netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6234                         bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6235                 }
6236         }
6237         bp->rx_nr_rings = rx_rings;
6238         bp->cp_nr_rings = cp;
6239
6240         if (!tx || !rx || !cp || !grp || !vnic || !stat)
6241                 return -ENOMEM;
6242
6243         if (!netif_is_rxfh_configured(bp->dev))
6244                 bnxt_set_dflt_rss_indir_tbl(bp);
6245
6246         return rc;
6247 }
6248
6249 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6250                                     int ring_grps, int cp_rings, int stats,
6251                                     int vnics)
6252 {
6253         struct hwrm_func_vf_cfg_input req = {0};
6254         u32 flags;
6255
6256         if (!BNXT_NEW_RM(bp))
6257                 return 0;
6258
6259         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6260                                      cp_rings, stats, vnics);
6261         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6262                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6263                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6264                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6265                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6266                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6267         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6268                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6269
6270         req.flags = cpu_to_le32(flags);
6271         return hwrm_send_message_silent(bp, &req, sizeof(req),
6272                                         HWRM_CMD_TIMEOUT);
6273 }
6274
6275 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6276                                     int ring_grps, int cp_rings, int stats,
6277                                     int vnics)
6278 {
6279         struct hwrm_func_cfg_input req = {0};
6280         u32 flags;
6281
6282         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6283                                      cp_rings, stats, vnics);
6284         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6285         if (BNXT_NEW_RM(bp)) {
6286                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6287                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6288                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6289                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6290                 if (bp->flags & BNXT_FLAG_CHIP_P5)
6291                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6292                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6293                 else
6294                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6295         }
6296
6297         req.flags = cpu_to_le32(flags);
6298         return hwrm_send_message_silent(bp, &req, sizeof(req),
6299                                         HWRM_CMD_TIMEOUT);
6300 }
6301
6302 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6303                                  int ring_grps, int cp_rings, int stats,
6304                                  int vnics)
6305 {
6306         if (bp->hwrm_spec_code < 0x10801)
6307                 return 0;
6308
6309         if (BNXT_PF(bp))
6310                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6311                                                 ring_grps, cp_rings, stats,
6312                                                 vnics);
6313
6314         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6315                                         cp_rings, stats, vnics);
6316 }
6317
6318 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6319 {
6320         struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6321         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6322         struct hwrm_ring_aggint_qcaps_input req = {0};
6323         int rc;
6324
6325         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6326         coal_cap->num_cmpl_dma_aggr_max = 63;
6327         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6328         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6329         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6330         coal_cap->int_lat_tmr_min_max = 65535;
6331         coal_cap->int_lat_tmr_max_max = 65535;
6332         coal_cap->num_cmpl_aggr_int_max = 65535;
6333         coal_cap->timer_units = 80;
6334
6335         if (bp->hwrm_spec_code < 0x10902)
6336                 return;
6337
6338         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6339         mutex_lock(&bp->hwrm_cmd_lock);
6340         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6341         if (!rc) {
6342                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6343                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6344                 coal_cap->num_cmpl_dma_aggr_max =
6345                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6346                 coal_cap->num_cmpl_dma_aggr_during_int_max =
6347                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6348                 coal_cap->cmpl_aggr_dma_tmr_max =
6349                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6350                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6351                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6352                 coal_cap->int_lat_tmr_min_max =
6353                         le16_to_cpu(resp->int_lat_tmr_min_max);
6354                 coal_cap->int_lat_tmr_max_max =
6355                         le16_to_cpu(resp->int_lat_tmr_max_max);
6356                 coal_cap->num_cmpl_aggr_int_max =
6357                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
6358                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6359         }
6360         mutex_unlock(&bp->hwrm_cmd_lock);
6361 }
6362
6363 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6364 {
6365         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6366
6367         return usec * 1000 / coal_cap->timer_units;
6368 }
6369
6370 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6371         struct bnxt_coal *hw_coal,
6372         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6373 {
6374         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6375         u32 cmpl_params = coal_cap->cmpl_params;
6376         u16 val, tmr, max, flags = 0;
6377
6378         max = hw_coal->bufs_per_record * 128;
6379         if (hw_coal->budget)
6380                 max = hw_coal->bufs_per_record * hw_coal->budget;
6381         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6382
6383         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6384         req->num_cmpl_aggr_int = cpu_to_le16(val);
6385
6386         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6387         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6388
6389         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6390                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6391         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6392
6393         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6394         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6395         req->int_lat_tmr_max = cpu_to_le16(tmr);
6396
6397         /* min timer set to 1/2 of interrupt timer */
6398         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6399                 val = tmr / 2;
6400                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6401                 req->int_lat_tmr_min = cpu_to_le16(val);
6402                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6403         }
6404
6405         /* buf timer set to 1/4 of interrupt timer */
6406         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6407         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6408
6409         if (cmpl_params &
6410             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6411                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6412                 val = clamp_t(u16, tmr, 1,
6413                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6414                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6415                 req->enables |=
6416                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6417         }
6418
6419         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6420                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6421         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6422             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6423                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6424         req->flags = cpu_to_le16(flags);
6425         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6426 }
6427
6428 /* Caller holds bp->hwrm_cmd_lock */
6429 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6430                                    struct bnxt_coal *hw_coal)
6431 {
6432         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6433         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6434         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6435         u32 nq_params = coal_cap->nq_params;
6436         u16 tmr;
6437
6438         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6439                 return 0;
6440
6441         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6442                                -1, -1);
6443         req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6444         req.flags =
6445                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6446
6447         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6448         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6449         req.int_lat_tmr_min = cpu_to_le16(tmr);
6450         req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6451         return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6452 }
6453
6454 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6455 {
6456         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6457         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6458         struct bnxt_coal coal;
6459
6460         /* Tick values in micro seconds.
6461          * 1 coal_buf x bufs_per_record = 1 completion record.
6462          */
6463         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6464
6465         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6466         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6467
6468         if (!bnapi->rx_ring)
6469                 return -ENODEV;
6470
6471         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6472                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6473
6474         bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6475
6476         req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6477
6478         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6479                                  HWRM_CMD_TIMEOUT);
6480 }
6481
6482 int bnxt_hwrm_set_coal(struct bnxt *bp)
6483 {
6484         int i, rc = 0;
6485         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6486                                                            req_tx = {0}, *req;
6487
6488         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6489                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6490         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6491                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6492
6493         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6494         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6495
6496         mutex_lock(&bp->hwrm_cmd_lock);
6497         for (i = 0; i < bp->cp_nr_rings; i++) {
6498                 struct bnxt_napi *bnapi = bp->bnapi[i];
6499                 struct bnxt_coal *hw_coal;
6500                 u16 ring_id;
6501
6502                 req = &req_rx;
6503                 if (!bnapi->rx_ring) {
6504                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6505                         req = &req_tx;
6506                 } else {
6507                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6508                 }
6509                 req->ring_id = cpu_to_le16(ring_id);
6510
6511                 rc = _hwrm_send_message(bp, req, sizeof(*req),
6512                                         HWRM_CMD_TIMEOUT);
6513                 if (rc)
6514                         break;
6515
6516                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6517                         continue;
6518
6519                 if (bnapi->rx_ring && bnapi->tx_ring) {
6520                         req = &req_tx;
6521                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6522                         req->ring_id = cpu_to_le16(ring_id);
6523                         rc = _hwrm_send_message(bp, req, sizeof(*req),
6524                                                 HWRM_CMD_TIMEOUT);
6525                         if (rc)
6526                                 break;
6527                 }
6528                 if (bnapi->rx_ring)
6529                         hw_coal = &bp->rx_coal;
6530                 else
6531                         hw_coal = &bp->tx_coal;
6532                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6533         }
6534         mutex_unlock(&bp->hwrm_cmd_lock);
6535         return rc;
6536 }
6537
6538 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6539 {
6540         struct hwrm_stat_ctx_clr_stats_input req0 = {0};
6541         struct hwrm_stat_ctx_free_input req = {0};
6542         int i;
6543
6544         if (!bp->bnapi)
6545                 return;
6546
6547         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6548                 return;
6549
6550         bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1);
6551         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6552
6553         mutex_lock(&bp->hwrm_cmd_lock);
6554         for (i = 0; i < bp->cp_nr_rings; i++) {
6555                 struct bnxt_napi *bnapi = bp->bnapi[i];
6556                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6557
6558                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6559                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6560                         if (BNXT_FW_MAJ(bp) <= 20) {
6561                                 req0.stat_ctx_id = req.stat_ctx_id;
6562                                 _hwrm_send_message(bp, &req0, sizeof(req0),
6563                                                    HWRM_CMD_TIMEOUT);
6564                         }
6565                         _hwrm_send_message(bp, &req, sizeof(req),
6566                                            HWRM_CMD_TIMEOUT);
6567
6568                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6569                 }
6570         }
6571         mutex_unlock(&bp->hwrm_cmd_lock);
6572 }
6573
6574 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6575 {
6576         int rc = 0, i;
6577         struct hwrm_stat_ctx_alloc_input req = {0};
6578         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6579
6580         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6581                 return 0;
6582
6583         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6584
6585         req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6586         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6587
6588         mutex_lock(&bp->hwrm_cmd_lock);
6589         for (i = 0; i < bp->cp_nr_rings; i++) {
6590                 struct bnxt_napi *bnapi = bp->bnapi[i];
6591                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6592
6593                 req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6594
6595                 rc = _hwrm_send_message(bp, &req, sizeof(req),
6596                                         HWRM_CMD_TIMEOUT);
6597                 if (rc)
6598                         break;
6599
6600                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6601
6602                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6603         }
6604         mutex_unlock(&bp->hwrm_cmd_lock);
6605         return rc;
6606 }
6607
6608 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6609 {
6610         struct hwrm_func_qcfg_input req = {0};
6611         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6612         u32 min_db_offset = 0;
6613         u16 flags;
6614         int rc;
6615
6616         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6617         req.fid = cpu_to_le16(0xffff);
6618         mutex_lock(&bp->hwrm_cmd_lock);
6619         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6620         if (rc)
6621                 goto func_qcfg_exit;
6622
6623 #ifdef CONFIG_BNXT_SRIOV
6624         if (BNXT_VF(bp)) {
6625                 struct bnxt_vf_info *vf = &bp->vf;
6626
6627                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6628         } else {
6629                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6630         }
6631 #endif
6632         flags = le16_to_cpu(resp->flags);
6633         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6634                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6635                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6636                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6637                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6638         }
6639         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6640                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6641
6642         switch (resp->port_partition_type) {
6643         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6644         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6645         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6646                 bp->port_partition_type = resp->port_partition_type;
6647                 break;
6648         }
6649         if (bp->hwrm_spec_code < 0x10707 ||
6650             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6651                 bp->br_mode = BRIDGE_MODE_VEB;
6652         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6653                 bp->br_mode = BRIDGE_MODE_VEPA;
6654         else
6655                 bp->br_mode = BRIDGE_MODE_UNDEF;
6656
6657         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6658         if (!bp->max_mtu)
6659                 bp->max_mtu = BNXT_MAX_MTU;
6660
6661         if (bp->db_size)
6662                 goto func_qcfg_exit;
6663
6664         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6665                 if (BNXT_PF(bp))
6666                         min_db_offset = DB_PF_OFFSET_P5;
6667                 else
6668                         min_db_offset = DB_VF_OFFSET_P5;
6669         }
6670         bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
6671                                  1024);
6672         if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
6673             bp->db_size <= min_db_offset)
6674                 bp->db_size = pci_resource_len(bp->pdev, 2);
6675
6676 func_qcfg_exit:
6677         mutex_unlock(&bp->hwrm_cmd_lock);
6678         return rc;
6679 }
6680
6681 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6682 {
6683         struct hwrm_func_backing_store_qcaps_input req = {0};
6684         struct hwrm_func_backing_store_qcaps_output *resp =
6685                 bp->hwrm_cmd_resp_addr;
6686         int rc;
6687
6688         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6689                 return 0;
6690
6691         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6692         mutex_lock(&bp->hwrm_cmd_lock);
6693         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6694         if (!rc) {
6695                 struct bnxt_ctx_pg_info *ctx_pg;
6696                 struct bnxt_ctx_mem_info *ctx;
6697                 int i, tqm_rings;
6698
6699                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6700                 if (!ctx) {
6701                         rc = -ENOMEM;
6702                         goto ctx_err;
6703                 }
6704                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6705                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6706                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6707                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6708                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6709                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6710                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6711                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6712                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6713                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6714                 ctx->vnic_max_vnic_entries =
6715                         le16_to_cpu(resp->vnic_max_vnic_entries);
6716                 ctx->vnic_max_ring_table_entries =
6717                         le16_to_cpu(resp->vnic_max_ring_table_entries);
6718                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6719                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6720                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6721                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6722                 ctx->tqm_min_entries_per_ring =
6723                         le32_to_cpu(resp->tqm_min_entries_per_ring);
6724                 ctx->tqm_max_entries_per_ring =
6725                         le32_to_cpu(resp->tqm_max_entries_per_ring);
6726                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6727                 if (!ctx->tqm_entries_multiple)
6728                         ctx->tqm_entries_multiple = 1;
6729                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6730                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6731                 ctx->mrav_num_entries_units =
6732                         le16_to_cpu(resp->mrav_num_entries_units);
6733                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6734                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6735                 ctx->ctx_kind_initializer = resp->ctx_kind_initializer;
6736                 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
6737                 if (!ctx->tqm_fp_rings_count)
6738                         ctx->tqm_fp_rings_count = bp->max_q;
6739
6740                 tqm_rings = ctx->tqm_fp_rings_count + 1;
6741                 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
6742                 if (!ctx_pg) {
6743                         kfree(ctx);
6744                         rc = -ENOMEM;
6745                         goto ctx_err;
6746                 }
6747                 for (i = 0; i < tqm_rings; i++, ctx_pg++)
6748                         ctx->tqm_mem[i] = ctx_pg;
6749                 bp->ctx = ctx;
6750         } else {
6751                 rc = 0;
6752         }
6753 ctx_err:
6754         mutex_unlock(&bp->hwrm_cmd_lock);
6755         return rc;
6756 }
6757
6758 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6759                                   __le64 *pg_dir)
6760 {
6761         u8 pg_size = 0;
6762
6763         if (BNXT_PAGE_SHIFT == 13)
6764                 pg_size = 1 << 4;
6765         else if (BNXT_PAGE_SIZE == 16)
6766                 pg_size = 2 << 4;
6767
6768         *pg_attr = pg_size;
6769         if (rmem->depth >= 1) {
6770                 if (rmem->depth == 2)
6771                         *pg_attr |= 2;
6772                 else
6773                         *pg_attr |= 1;
6774                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6775         } else {
6776                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6777         }
6778 }
6779
6780 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
6781         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
6782          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
6783          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
6784          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
6785          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6786
6787 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6788 {
6789         struct hwrm_func_backing_store_cfg_input req = {0};
6790         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6791         struct bnxt_ctx_pg_info *ctx_pg;
6792         __le32 *num_entries;
6793         __le64 *pg_dir;
6794         u32 flags = 0;
6795         u8 *pg_attr;
6796         u32 ena;
6797         int i;
6798
6799         if (!ctx)
6800                 return 0;
6801
6802         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6803         req.enables = cpu_to_le32(enables);
6804
6805         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6806                 ctx_pg = &ctx->qp_mem;
6807                 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6808                 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6809                 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6810                 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6811                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6812                                       &req.qpc_pg_size_qpc_lvl,
6813                                       &req.qpc_page_dir);
6814         }
6815         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6816                 ctx_pg = &ctx->srq_mem;
6817                 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6818                 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6819                 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6820                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6821                                       &req.srq_pg_size_srq_lvl,
6822                                       &req.srq_page_dir);
6823         }
6824         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6825                 ctx_pg = &ctx->cq_mem;
6826                 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6827                 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6828                 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6829                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6830                                       &req.cq_page_dir);
6831         }
6832         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6833                 ctx_pg = &ctx->vnic_mem;
6834                 req.vnic_num_vnic_entries =
6835                         cpu_to_le16(ctx->vnic_max_vnic_entries);
6836                 req.vnic_num_ring_table_entries =
6837                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
6838                 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6839                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6840                                       &req.vnic_pg_size_vnic_lvl,
6841                                       &req.vnic_page_dir);
6842         }
6843         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6844                 ctx_pg = &ctx->stat_mem;
6845                 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6846                 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6847                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6848                                       &req.stat_pg_size_stat_lvl,
6849                                       &req.stat_page_dir);
6850         }
6851         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6852                 ctx_pg = &ctx->mrav_mem;
6853                 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6854                 if (ctx->mrav_num_entries_units)
6855                         flags |=
6856                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
6857                 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6858                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6859                                       &req.mrav_pg_size_mrav_lvl,
6860                                       &req.mrav_page_dir);
6861         }
6862         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6863                 ctx_pg = &ctx->tim_mem;
6864                 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6865                 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6866                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6867                                       &req.tim_pg_size_tim_lvl,
6868                                       &req.tim_page_dir);
6869         }
6870         for (i = 0, num_entries = &req.tqm_sp_num_entries,
6871              pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6872              pg_dir = &req.tqm_sp_page_dir,
6873              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6874              i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6875                 if (!(enables & ena))
6876                         continue;
6877
6878                 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6879                 ctx_pg = ctx->tqm_mem[i];
6880                 *num_entries = cpu_to_le32(ctx_pg->entries);
6881                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6882         }
6883         req.flags = cpu_to_le32(flags);
6884         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6885 }
6886
6887 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6888                                   struct bnxt_ctx_pg_info *ctx_pg)
6889 {
6890         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6891
6892         rmem->page_size = BNXT_PAGE_SIZE;
6893         rmem->pg_arr = ctx_pg->ctx_pg_arr;
6894         rmem->dma_arr = ctx_pg->ctx_dma_arr;
6895         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6896         if (rmem->depth >= 1)
6897                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6898         return bnxt_alloc_ring(bp, rmem);
6899 }
6900
6901 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6902                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6903                                   u8 depth, bool use_init_val)
6904 {
6905         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6906         int rc;
6907
6908         if (!mem_size)
6909                 return -EINVAL;
6910
6911         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6912         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6913                 ctx_pg->nr_pages = 0;
6914                 return -EINVAL;
6915         }
6916         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6917                 int nr_tbls, i;
6918
6919                 rmem->depth = 2;
6920                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6921                                              GFP_KERNEL);
6922                 if (!ctx_pg->ctx_pg_tbl)
6923                         return -ENOMEM;
6924                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6925                 rmem->nr_pages = nr_tbls;
6926                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6927                 if (rc)
6928                         return rc;
6929                 for (i = 0; i < nr_tbls; i++) {
6930                         struct bnxt_ctx_pg_info *pg_tbl;
6931
6932                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6933                         if (!pg_tbl)
6934                                 return -ENOMEM;
6935                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6936                         rmem = &pg_tbl->ring_mem;
6937                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6938                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6939                         rmem->depth = 1;
6940                         rmem->nr_pages = MAX_CTX_PAGES;
6941                         if (use_init_val)
6942                                 rmem->init_val = bp->ctx->ctx_kind_initializer;
6943                         if (i == (nr_tbls - 1)) {
6944                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6945
6946                                 if (rem)
6947                                         rmem->nr_pages = rem;
6948                         }
6949                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
6950                         if (rc)
6951                                 break;
6952                 }
6953         } else {
6954                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6955                 if (rmem->nr_pages > 1 || depth)
6956                         rmem->depth = 1;
6957                 if (use_init_val)
6958                         rmem->init_val = bp->ctx->ctx_kind_initializer;
6959                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6960         }
6961         return rc;
6962 }
6963
6964 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
6965                                   struct bnxt_ctx_pg_info *ctx_pg)
6966 {
6967         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6968
6969         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
6970             ctx_pg->ctx_pg_tbl) {
6971                 int i, nr_tbls = rmem->nr_pages;
6972
6973                 for (i = 0; i < nr_tbls; i++) {
6974                         struct bnxt_ctx_pg_info *pg_tbl;
6975                         struct bnxt_ring_mem_info *rmem2;
6976
6977                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
6978                         if (!pg_tbl)
6979                                 continue;
6980                         rmem2 = &pg_tbl->ring_mem;
6981                         bnxt_free_ring(bp, rmem2);
6982                         ctx_pg->ctx_pg_arr[i] = NULL;
6983                         kfree(pg_tbl);
6984                         ctx_pg->ctx_pg_tbl[i] = NULL;
6985                 }
6986                 kfree(ctx_pg->ctx_pg_tbl);
6987                 ctx_pg->ctx_pg_tbl = NULL;
6988         }
6989         bnxt_free_ring(bp, rmem);
6990         ctx_pg->nr_pages = 0;
6991 }
6992
6993 static void bnxt_free_ctx_mem(struct bnxt *bp)
6994 {
6995         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6996         int i;
6997
6998         if (!ctx)
6999                 return;
7000
7001         if (ctx->tqm_mem[0]) {
7002                 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7003                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7004                 kfree(ctx->tqm_mem[0]);
7005                 ctx->tqm_mem[0] = NULL;
7006         }
7007
7008         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7009         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7010         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7011         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7012         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7013         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7014         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7015         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7016 }
7017
7018 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7019 {
7020         struct bnxt_ctx_pg_info *ctx_pg;
7021         struct bnxt_ctx_mem_info *ctx;
7022         u32 mem_size, ena, entries;
7023         u32 entries_sp, min;
7024         u32 num_mr, num_ah;
7025         u32 extra_srqs = 0;
7026         u32 extra_qps = 0;
7027         u8 pg_lvl = 1;
7028         int i, rc;
7029
7030         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7031         if (rc) {
7032                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7033                            rc);
7034                 return rc;
7035         }
7036         ctx = bp->ctx;
7037         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7038                 return 0;
7039
7040         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7041                 pg_lvl = 2;
7042                 extra_qps = 65536;
7043                 extra_srqs = 8192;
7044         }
7045
7046         ctx_pg = &ctx->qp_mem;
7047         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7048                           extra_qps;
7049         mem_size = ctx->qp_entry_size * ctx_pg->entries;
7050         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7051         if (rc)
7052                 return rc;
7053
7054         ctx_pg = &ctx->srq_mem;
7055         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7056         mem_size = ctx->srq_entry_size * ctx_pg->entries;
7057         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7058         if (rc)
7059                 return rc;
7060
7061         ctx_pg = &ctx->cq_mem;
7062         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7063         mem_size = ctx->cq_entry_size * ctx_pg->entries;
7064         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7065         if (rc)
7066                 return rc;
7067
7068         ctx_pg = &ctx->vnic_mem;
7069         ctx_pg->entries = ctx->vnic_max_vnic_entries +
7070                           ctx->vnic_max_ring_table_entries;
7071         mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7072         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7073         if (rc)
7074                 return rc;
7075
7076         ctx_pg = &ctx->stat_mem;
7077         ctx_pg->entries = ctx->stat_max_entries;
7078         mem_size = ctx->stat_entry_size * ctx_pg->entries;
7079         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7080         if (rc)
7081                 return rc;
7082
7083         ena = 0;
7084         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7085                 goto skip_rdma;
7086
7087         ctx_pg = &ctx->mrav_mem;
7088         /* 128K extra is needed to accommodate static AH context
7089          * allocation by f/w.
7090          */
7091         num_mr = 1024 * 256;
7092         num_ah = 1024 * 128;
7093         ctx_pg->entries = num_mr + num_ah;
7094         mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7095         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, true);
7096         if (rc)
7097                 return rc;
7098         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7099         if (ctx->mrav_num_entries_units)
7100                 ctx_pg->entries =
7101                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
7102                          (num_ah / ctx->mrav_num_entries_units);
7103
7104         ctx_pg = &ctx->tim_mem;
7105         ctx_pg->entries = ctx->qp_mem.entries;
7106         mem_size = ctx->tim_entry_size * ctx_pg->entries;
7107         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7108         if (rc)
7109                 return rc;
7110         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7111
7112 skip_rdma:
7113         min = ctx->tqm_min_entries_per_ring;
7114         entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7115                      2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7116         entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7117         entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries;
7118         entries = roundup(entries, ctx->tqm_entries_multiple);
7119         entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7120         for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7121                 ctx_pg = ctx->tqm_mem[i];
7122                 ctx_pg->entries = i ? entries : entries_sp;
7123                 mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7124                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7125                 if (rc)
7126                         return rc;
7127                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7128         }
7129         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7130         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7131         if (rc) {
7132                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7133                            rc);
7134                 return rc;
7135         }
7136         ctx->flags |= BNXT_CTX_FLAG_INITED;
7137         return 0;
7138 }
7139
7140 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7141 {
7142         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7143         struct hwrm_func_resource_qcaps_input req = {0};
7144         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7145         int rc;
7146
7147         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
7148         req.fid = cpu_to_le16(0xffff);
7149
7150         mutex_lock(&bp->hwrm_cmd_lock);
7151         rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
7152                                        HWRM_CMD_TIMEOUT);
7153         if (rc)
7154                 goto hwrm_func_resc_qcaps_exit;
7155
7156         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7157         if (!all)
7158                 goto hwrm_func_resc_qcaps_exit;
7159
7160         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7161         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7162         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7163         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7164         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7165         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7166         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7167         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7168         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7169         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7170         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7171         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7172         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7173         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7174         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7175         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7176
7177         if (bp->flags & BNXT_FLAG_CHIP_P5) {
7178                 u16 max_msix = le16_to_cpu(resp->max_msix);
7179
7180                 hw_resc->max_nqs = max_msix;
7181                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7182         }
7183
7184         if (BNXT_PF(bp)) {
7185                 struct bnxt_pf_info *pf = &bp->pf;
7186
7187                 pf->vf_resv_strategy =
7188                         le16_to_cpu(resp->vf_reservation_strategy);
7189                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7190                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7191         }
7192 hwrm_func_resc_qcaps_exit:
7193         mutex_unlock(&bp->hwrm_cmd_lock);
7194         return rc;
7195 }
7196
7197 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7198 {
7199         int rc = 0;
7200         struct hwrm_func_qcaps_input req = {0};
7201         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7202         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7203         u32 flags, flags_ext;
7204
7205         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
7206         req.fid = cpu_to_le16(0xffff);
7207
7208         mutex_lock(&bp->hwrm_cmd_lock);
7209         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7210         if (rc)
7211                 goto hwrm_func_qcaps_exit;
7212
7213         flags = le32_to_cpu(resp->flags);
7214         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7215                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7216         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7217                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7218         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7219                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7220         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7221                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7222         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7223                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7224         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7225                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7226         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7227                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7228         if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7229                 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7230
7231         flags_ext = le32_to_cpu(resp->flags_ext);
7232         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7233                 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7234
7235         bp->tx_push_thresh = 0;
7236         if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7237             BNXT_FW_MAJ(bp) > 217)
7238                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7239
7240         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7241         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7242         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7243         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7244         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7245         if (!hw_resc->max_hw_ring_grps)
7246                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7247         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7248         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7249         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7250
7251         if (BNXT_PF(bp)) {
7252                 struct bnxt_pf_info *pf = &bp->pf;
7253
7254                 pf->fw_fid = le16_to_cpu(resp->fid);
7255                 pf->port_id = le16_to_cpu(resp->port_id);
7256                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7257                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7258                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7259                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7260                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7261                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7262                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7263                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7264                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7265                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7266                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7267                         bp->flags |= BNXT_FLAG_WOL_CAP;
7268         } else {
7269 #ifdef CONFIG_BNXT_SRIOV
7270                 struct bnxt_vf_info *vf = &bp->vf;
7271
7272                 vf->fw_fid = le16_to_cpu(resp->fid);
7273                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7274 #endif
7275         }
7276
7277 hwrm_func_qcaps_exit:
7278         mutex_unlock(&bp->hwrm_cmd_lock);
7279         return rc;
7280 }
7281
7282 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7283
7284 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7285 {
7286         int rc;
7287
7288         rc = __bnxt_hwrm_func_qcaps(bp);
7289         if (rc)
7290                 return rc;
7291         rc = bnxt_hwrm_queue_qportcfg(bp);
7292         if (rc) {
7293                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7294                 return rc;
7295         }
7296         if (bp->hwrm_spec_code >= 0x10803) {
7297                 rc = bnxt_alloc_ctx_mem(bp);
7298                 if (rc)
7299                         return rc;
7300                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7301                 if (!rc)
7302                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7303         }
7304         return 0;
7305 }
7306
7307 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7308 {
7309         struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7310         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7311         int rc = 0;
7312         u32 flags;
7313
7314         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7315                 return 0;
7316
7317         resp = bp->hwrm_cmd_resp_addr;
7318         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7319
7320         mutex_lock(&bp->hwrm_cmd_lock);
7321         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7322         if (rc)
7323                 goto hwrm_cfa_adv_qcaps_exit;
7324
7325         flags = le32_to_cpu(resp->flags);
7326         if (flags &
7327             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7328                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7329
7330 hwrm_cfa_adv_qcaps_exit:
7331         mutex_unlock(&bp->hwrm_cmd_lock);
7332         return rc;
7333 }
7334
7335 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7336 {
7337         struct bnxt_fw_health *fw_health = bp->fw_health;
7338         u32 reg_base = 0xffffffff;
7339         int i;
7340
7341         /* Only pre-map the monitoring GRC registers using window 3 */
7342         for (i = 0; i < 4; i++) {
7343                 u32 reg = fw_health->regs[i];
7344
7345                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7346                         continue;
7347                 if (reg_base == 0xffffffff)
7348                         reg_base = reg & BNXT_GRC_BASE_MASK;
7349                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7350                         return -ERANGE;
7351                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_BASE +
7352                                             (reg & BNXT_GRC_OFFSET_MASK);
7353         }
7354         if (reg_base == 0xffffffff)
7355                 return 0;
7356
7357         writel(reg_base, bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7358                          BNXT_FW_HEALTH_WIN_MAP_OFF);
7359         return 0;
7360 }
7361
7362 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7363 {
7364         struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7365         struct bnxt_fw_health *fw_health = bp->fw_health;
7366         struct hwrm_error_recovery_qcfg_input req = {0};
7367         int rc, i;
7368
7369         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7370                 return 0;
7371
7372         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7373         mutex_lock(&bp->hwrm_cmd_lock);
7374         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7375         if (rc)
7376                 goto err_recovery_out;
7377         fw_health->flags = le32_to_cpu(resp->flags);
7378         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7379             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7380                 rc = -EINVAL;
7381                 goto err_recovery_out;
7382         }
7383         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7384         fw_health->master_func_wait_dsecs =
7385                 le32_to_cpu(resp->master_func_wait_period);
7386         fw_health->normal_func_wait_dsecs =
7387                 le32_to_cpu(resp->normal_func_wait_period);
7388         fw_health->post_reset_wait_dsecs =
7389                 le32_to_cpu(resp->master_func_wait_period_after_reset);
7390         fw_health->post_reset_max_wait_dsecs =
7391                 le32_to_cpu(resp->max_bailout_time_after_reset);
7392         fw_health->regs[BNXT_FW_HEALTH_REG] =
7393                 le32_to_cpu(resp->fw_health_status_reg);
7394         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7395                 le32_to_cpu(resp->fw_heartbeat_reg);
7396         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7397                 le32_to_cpu(resp->fw_reset_cnt_reg);
7398         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7399                 le32_to_cpu(resp->reset_inprogress_reg);
7400         fw_health->fw_reset_inprog_reg_mask =
7401                 le32_to_cpu(resp->reset_inprogress_reg_mask);
7402         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7403         if (fw_health->fw_reset_seq_cnt >= 16) {
7404                 rc = -EINVAL;
7405                 goto err_recovery_out;
7406         }
7407         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7408                 fw_health->fw_reset_seq_regs[i] =
7409                         le32_to_cpu(resp->reset_reg[i]);
7410                 fw_health->fw_reset_seq_vals[i] =
7411                         le32_to_cpu(resp->reset_reg_val[i]);
7412                 fw_health->fw_reset_seq_delay_msec[i] =
7413                         resp->delay_after_reset[i];
7414         }
7415 err_recovery_out:
7416         mutex_unlock(&bp->hwrm_cmd_lock);
7417         if (!rc)
7418                 rc = bnxt_map_fw_health_regs(bp);
7419         if (rc)
7420                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7421         return rc;
7422 }
7423
7424 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7425 {
7426         struct hwrm_func_reset_input req = {0};
7427
7428         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7429         req.enables = 0;
7430
7431         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7432 }
7433
7434 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7435 {
7436         int rc = 0;
7437         struct hwrm_queue_qportcfg_input req = {0};
7438         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7439         u8 i, j, *qptr;
7440         bool no_rdma;
7441
7442         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7443
7444         mutex_lock(&bp->hwrm_cmd_lock);
7445         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7446         if (rc)
7447                 goto qportcfg_exit;
7448
7449         if (!resp->max_configurable_queues) {
7450                 rc = -EINVAL;
7451                 goto qportcfg_exit;
7452         }
7453         bp->max_tc = resp->max_configurable_queues;
7454         bp->max_lltc = resp->max_configurable_lossless_queues;
7455         if (bp->max_tc > BNXT_MAX_QUEUE)
7456                 bp->max_tc = BNXT_MAX_QUEUE;
7457
7458         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7459         qptr = &resp->queue_id0;
7460         for (i = 0, j = 0; i < bp->max_tc; i++) {
7461                 bp->q_info[j].queue_id = *qptr;
7462                 bp->q_ids[i] = *qptr++;
7463                 bp->q_info[j].queue_profile = *qptr++;
7464                 bp->tc_to_qidx[j] = j;
7465                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7466                     (no_rdma && BNXT_PF(bp)))
7467                         j++;
7468         }
7469         bp->max_q = bp->max_tc;
7470         bp->max_tc = max_t(u8, j, 1);
7471
7472         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7473                 bp->max_tc = 1;
7474
7475         if (bp->max_lltc > bp->max_tc)
7476                 bp->max_lltc = bp->max_tc;
7477
7478 qportcfg_exit:
7479         mutex_unlock(&bp->hwrm_cmd_lock);
7480         return rc;
7481 }
7482
7483 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7484 {
7485         struct hwrm_ver_get_input req = {0};
7486         int rc;
7487
7488         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7489         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7490         req.hwrm_intf_min = HWRM_VERSION_MINOR;
7491         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7492
7493         rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7494                                    silent);
7495         return rc;
7496 }
7497
7498 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7499 {
7500         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7501         u16 fw_maj, fw_min, fw_bld, fw_rsv;
7502         u32 dev_caps_cfg, hwrm_ver;
7503         int rc, len;
7504
7505         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7506         mutex_lock(&bp->hwrm_cmd_lock);
7507         rc = __bnxt_hwrm_ver_get(bp, false);
7508         if (rc)
7509                 goto hwrm_ver_get_exit;
7510
7511         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7512
7513         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7514                              resp->hwrm_intf_min_8b << 8 |
7515                              resp->hwrm_intf_upd_8b;
7516         if (resp->hwrm_intf_maj_8b < 1) {
7517                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7518                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7519                             resp->hwrm_intf_upd_8b);
7520                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7521         }
7522
7523         hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
7524                         HWRM_VERSION_UPDATE;
7525
7526         if (bp->hwrm_spec_code > hwrm_ver)
7527                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7528                          HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
7529                          HWRM_VERSION_UPDATE);
7530         else
7531                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7532                          resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7533                          resp->hwrm_intf_upd_8b);
7534
7535         fw_maj = le16_to_cpu(resp->hwrm_fw_major);
7536         if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
7537                 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
7538                 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
7539                 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
7540                 len = FW_VER_STR_LEN;
7541         } else {
7542                 fw_maj = resp->hwrm_fw_maj_8b;
7543                 fw_min = resp->hwrm_fw_min_8b;
7544                 fw_bld = resp->hwrm_fw_bld_8b;
7545                 fw_rsv = resp->hwrm_fw_rsvd_8b;
7546                 len = BC_HWRM_STR_LEN;
7547         }
7548         bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
7549         snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
7550                  fw_rsv);
7551
7552         if (strlen(resp->active_pkg_name)) {
7553                 int fw_ver_len = strlen(bp->fw_ver_str);
7554
7555                 snprintf(bp->fw_ver_str + fw_ver_len,
7556                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7557                          resp->active_pkg_name);
7558                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7559         }
7560
7561         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7562         if (!bp->hwrm_cmd_timeout)
7563                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7564
7565         if (resp->hwrm_intf_maj_8b >= 1) {
7566                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7567                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7568         }
7569         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7570                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7571
7572         bp->chip_num = le16_to_cpu(resp->chip_num);
7573         bp->chip_rev = resp->chip_rev;
7574         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7575             !resp->chip_metal)
7576                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7577
7578         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7579         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7580             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7581                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7582
7583         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7584                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7585
7586         if (dev_caps_cfg &
7587             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7588                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7589
7590         if (dev_caps_cfg &
7591             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7592                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7593
7594         if (dev_caps_cfg &
7595             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7596                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7597
7598 hwrm_ver_get_exit:
7599         mutex_unlock(&bp->hwrm_cmd_lock);
7600         return rc;
7601 }
7602
7603 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7604 {
7605         struct hwrm_fw_set_time_input req = {0};
7606         struct tm tm;
7607         time64_t now = ktime_get_real_seconds();
7608
7609         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7610             bp->hwrm_spec_code < 0x10400)
7611                 return -EOPNOTSUPP;
7612
7613         time64_to_tm(now, 0, &tm);
7614         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7615         req.year = cpu_to_le16(1900 + tm.tm_year);
7616         req.month = 1 + tm.tm_mon;
7617         req.day = tm.tm_mday;
7618         req.hour = tm.tm_hour;
7619         req.minute = tm.tm_min;
7620         req.second = tm.tm_sec;
7621         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7622 }
7623
7624 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
7625 {
7626         u64 sw_tmp;
7627
7628         sw_tmp = (*sw & ~mask) | hw;
7629         if (hw < (*sw & mask))
7630                 sw_tmp += mask + 1;
7631         WRITE_ONCE(*sw, sw_tmp);
7632 }
7633
7634 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
7635                                     int count, bool ignore_zero)
7636 {
7637         int i;
7638
7639         for (i = 0; i < count; i++) {
7640                 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
7641
7642                 if (ignore_zero && !hw)
7643                         continue;
7644
7645                 if (masks[i] == -1ULL)
7646                         sw_stats[i] = hw;
7647                 else
7648                         bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
7649         }
7650 }
7651
7652 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
7653 {
7654         if (!stats->hw_stats)
7655                 return;
7656
7657         __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7658                                 stats->hw_masks, stats->len / 8, false);
7659 }
7660
7661 static void bnxt_accumulate_all_stats(struct bnxt *bp)
7662 {
7663         struct bnxt_stats_mem *ring0_stats;
7664         bool ignore_zero = false;
7665         int i;
7666
7667         /* Chip bug.  Counter intermittently becomes 0. */
7668         if (bp->flags & BNXT_FLAG_CHIP_P5)
7669                 ignore_zero = true;
7670
7671         for (i = 0; i < bp->cp_nr_rings; i++) {
7672                 struct bnxt_napi *bnapi = bp->bnapi[i];
7673                 struct bnxt_cp_ring_info *cpr;
7674                 struct bnxt_stats_mem *stats;
7675
7676                 cpr = &bnapi->cp_ring;
7677                 stats = &cpr->stats;
7678                 if (!i)
7679                         ring0_stats = stats;
7680                 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7681                                         ring0_stats->hw_masks,
7682                                         ring0_stats->len / 8, ignore_zero);
7683         }
7684         if (bp->flags & BNXT_FLAG_PORT_STATS) {
7685                 struct bnxt_stats_mem *stats = &bp->port_stats;
7686                 __le64 *hw_stats = stats->hw_stats;
7687                 u64 *sw_stats = stats->sw_stats;
7688                 u64 *masks = stats->hw_masks;
7689                 int cnt;
7690
7691                 cnt = sizeof(struct rx_port_stats) / 8;
7692                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7693
7694                 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7695                 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7696                 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7697                 cnt = sizeof(struct tx_port_stats) / 8;
7698                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7699         }
7700         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
7701                 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
7702                 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
7703         }
7704 }
7705
7706 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
7707 {
7708         struct bnxt_pf_info *pf = &bp->pf;
7709         struct hwrm_port_qstats_input req = {0};
7710
7711         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7712                 return 0;
7713
7714         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7715                 return -EOPNOTSUPP;
7716
7717         req.flags = flags;
7718         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7719         req.port_id = cpu_to_le16(pf->port_id);
7720         req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
7721                                             BNXT_TX_PORT_STATS_BYTE_OFFSET);
7722         req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
7723         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7724 }
7725
7726 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
7727 {
7728         struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7729         struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7730         struct hwrm_port_qstats_ext_input req = {0};
7731         struct bnxt_pf_info *pf = &bp->pf;
7732         u32 tx_stat_size;
7733         int rc;
7734
7735         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7736                 return 0;
7737
7738         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7739                 return -EOPNOTSUPP;
7740
7741         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7742         req.flags = flags;
7743         req.port_id = cpu_to_le16(pf->port_id);
7744         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7745         req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
7746         tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
7747                        sizeof(struct tx_port_stats_ext) : 0;
7748         req.tx_stat_size = cpu_to_le16(tx_stat_size);
7749         req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
7750         mutex_lock(&bp->hwrm_cmd_lock);
7751         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7752         if (!rc) {
7753                 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
7754                 bp->fw_tx_stats_ext_size = tx_stat_size ?
7755                         le16_to_cpu(resp->tx_stat_size) / 8 : 0;
7756         } else {
7757                 bp->fw_rx_stats_ext_size = 0;
7758                 bp->fw_tx_stats_ext_size = 0;
7759         }
7760         if (flags)
7761                 goto qstats_done;
7762
7763         if (bp->fw_tx_stats_ext_size <=
7764             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7765                 mutex_unlock(&bp->hwrm_cmd_lock);
7766                 bp->pri2cos_valid = 0;
7767                 return rc;
7768         }
7769
7770         bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7771         req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7772
7773         rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7774         if (!rc) {
7775                 struct hwrm_queue_pri2cos_qcfg_output *resp2;
7776                 u8 *pri2cos;
7777                 int i, j;
7778
7779                 resp2 = bp->hwrm_cmd_resp_addr;
7780                 pri2cos = &resp2->pri0_cos_queue_id;
7781                 for (i = 0; i < 8; i++) {
7782                         u8 queue_id = pri2cos[i];
7783                         u8 queue_idx;
7784
7785                         /* Per port queue IDs start from 0, 10, 20, etc */
7786                         queue_idx = queue_id % 10;
7787                         if (queue_idx > BNXT_MAX_QUEUE) {
7788                                 bp->pri2cos_valid = false;
7789                                 goto qstats_done;
7790                         }
7791                         for (j = 0; j < bp->max_q; j++) {
7792                                 if (bp->q_ids[j] == queue_id)
7793                                         bp->pri2cos_idx[i] = queue_idx;
7794                         }
7795                 }
7796                 bp->pri2cos_valid = 1;
7797         }
7798 qstats_done:
7799         mutex_unlock(&bp->hwrm_cmd_lock);
7800         return rc;
7801 }
7802
7803 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7804 {
7805         if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID)
7806                 bnxt_hwrm_tunnel_dst_port_free(
7807                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7808         if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID)
7809                 bnxt_hwrm_tunnel_dst_port_free(
7810                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7811 }
7812
7813 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7814 {
7815         int rc, i;
7816         u32 tpa_flags = 0;
7817
7818         if (set_tpa)
7819                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
7820         else if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
7821                 return 0;
7822         for (i = 0; i < bp->nr_vnics; i++) {
7823                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7824                 if (rc) {
7825                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
7826                                    i, rc);
7827                         return rc;
7828                 }
7829         }
7830         return 0;
7831 }
7832
7833 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7834 {
7835         int i;
7836
7837         for (i = 0; i < bp->nr_vnics; i++)
7838                 bnxt_hwrm_vnic_set_rss(bp, i, false);
7839 }
7840
7841 static void bnxt_clear_vnic(struct bnxt *bp)
7842 {
7843         if (!bp->vnic_info)
7844                 return;
7845
7846         bnxt_hwrm_clear_vnic_filter(bp);
7847         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
7848                 /* clear all RSS setting before free vnic ctx */
7849                 bnxt_hwrm_clear_vnic_rss(bp);
7850                 bnxt_hwrm_vnic_ctx_free(bp);
7851         }
7852         /* before free the vnic, undo the vnic tpa settings */
7853         if (bp->flags & BNXT_FLAG_TPA)
7854                 bnxt_set_tpa(bp, false);
7855         bnxt_hwrm_vnic_free(bp);
7856         if (bp->flags & BNXT_FLAG_CHIP_P5)
7857                 bnxt_hwrm_vnic_ctx_free(bp);
7858 }
7859
7860 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7861                                     bool irq_re_init)
7862 {
7863         bnxt_clear_vnic(bp);
7864         bnxt_hwrm_ring_free(bp, close_path);
7865         bnxt_hwrm_ring_grp_free(bp);
7866         if (irq_re_init) {
7867                 bnxt_hwrm_stat_ctx_free(bp);
7868                 bnxt_hwrm_free_tunnel_ports(bp);
7869         }
7870 }
7871
7872 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
7873 {
7874         struct hwrm_func_cfg_input req = {0};
7875
7876         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7877         req.fid = cpu_to_le16(0xffff);
7878         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
7879         if (br_mode == BRIDGE_MODE_VEB)
7880                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
7881         else if (br_mode == BRIDGE_MODE_VEPA)
7882                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
7883         else
7884                 return -EINVAL;
7885         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7886 }
7887
7888 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
7889 {
7890         struct hwrm_func_cfg_input req = {0};
7891
7892         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
7893                 return 0;
7894
7895         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7896         req.fid = cpu_to_le16(0xffff);
7897         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
7898         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
7899         if (size == 128)
7900                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
7901
7902         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7903 }
7904
7905 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7906 {
7907         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
7908         int rc;
7909
7910         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
7911                 goto skip_rss_ctx;
7912
7913         /* allocate context for vnic */
7914         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
7915         if (rc) {
7916                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7917                            vnic_id, rc);
7918                 goto vnic_setup_err;
7919         }
7920         bp->rsscos_nr_ctxs++;
7921
7922         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7923                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
7924                 if (rc) {
7925                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
7926                                    vnic_id, rc);
7927                         goto vnic_setup_err;
7928                 }
7929                 bp->rsscos_nr_ctxs++;
7930         }
7931
7932 skip_rss_ctx:
7933         /* configure default vnic, ring grp */
7934         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7935         if (rc) {
7936                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7937                            vnic_id, rc);
7938                 goto vnic_setup_err;
7939         }
7940
7941         /* Enable RSS hashing on vnic */
7942         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
7943         if (rc) {
7944                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
7945                            vnic_id, rc);
7946                 goto vnic_setup_err;
7947         }
7948
7949         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7950                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7951                 if (rc) {
7952                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7953                                    vnic_id, rc);
7954                 }
7955         }
7956
7957 vnic_setup_err:
7958         return rc;
7959 }
7960
7961 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
7962 {
7963         int rc, i, nr_ctxs;
7964
7965         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
7966         for (i = 0; i < nr_ctxs; i++) {
7967                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
7968                 if (rc) {
7969                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
7970                                    vnic_id, i, rc);
7971                         break;
7972                 }
7973                 bp->rsscos_nr_ctxs++;
7974         }
7975         if (i < nr_ctxs)
7976                 return -ENOMEM;
7977
7978         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
7979         if (rc) {
7980                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
7981                            vnic_id, rc);
7982                 return rc;
7983         }
7984         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7985         if (rc) {
7986                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7987                            vnic_id, rc);
7988                 return rc;
7989         }
7990         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7991                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7992                 if (rc) {
7993                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7994                                    vnic_id, rc);
7995                 }
7996         }
7997         return rc;
7998 }
7999
8000 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8001 {
8002         if (bp->flags & BNXT_FLAG_CHIP_P5)
8003                 return __bnxt_setup_vnic_p5(bp, vnic_id);
8004         else
8005                 return __bnxt_setup_vnic(bp, vnic_id);
8006 }
8007
8008 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8009 {
8010 #ifdef CONFIG_RFS_ACCEL
8011         int i, rc = 0;
8012
8013         if (bp->flags & BNXT_FLAG_CHIP_P5)
8014                 return 0;
8015
8016         for (i = 0; i < bp->rx_nr_rings; i++) {
8017                 struct bnxt_vnic_info *vnic;
8018                 u16 vnic_id = i + 1;
8019                 u16 ring_id = i;
8020
8021                 if (vnic_id >= bp->nr_vnics)
8022                         break;
8023
8024                 vnic = &bp->vnic_info[vnic_id];
8025                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
8026                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8027                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8028                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8029                 if (rc) {
8030                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8031                                    vnic_id, rc);
8032                         break;
8033                 }
8034                 rc = bnxt_setup_vnic(bp, vnic_id);
8035                 if (rc)
8036                         break;
8037         }
8038         return rc;
8039 #else
8040         return 0;
8041 #endif
8042 }
8043
8044 /* Allow PF and VF with default VLAN to be in promiscuous mode */
8045 static bool bnxt_promisc_ok(struct bnxt *bp)
8046 {
8047 #ifdef CONFIG_BNXT_SRIOV
8048         if (BNXT_VF(bp) && !bp->vf.vlan)
8049                 return false;
8050 #endif
8051         return true;
8052 }
8053
8054 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8055 {
8056         unsigned int rc = 0;
8057
8058         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8059         if (rc) {
8060                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8061                            rc);
8062                 return rc;
8063         }
8064
8065         rc = bnxt_hwrm_vnic_cfg(bp, 1);
8066         if (rc) {
8067                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8068                            rc);
8069                 return rc;
8070         }
8071         return rc;
8072 }
8073
8074 static int bnxt_cfg_rx_mode(struct bnxt *);
8075 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8076
8077 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8078 {
8079         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8080         int rc = 0;
8081         unsigned int rx_nr_rings = bp->rx_nr_rings;
8082
8083         if (irq_re_init) {
8084                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
8085                 if (rc) {
8086                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8087                                    rc);
8088                         goto err_out;
8089                 }
8090         }
8091
8092         rc = bnxt_hwrm_ring_alloc(bp);
8093         if (rc) {
8094                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8095                 goto err_out;
8096         }
8097
8098         rc = bnxt_hwrm_ring_grp_alloc(bp);
8099         if (rc) {
8100                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8101                 goto err_out;
8102         }
8103
8104         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8105                 rx_nr_rings--;
8106
8107         /* default vnic 0 */
8108         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8109         if (rc) {
8110                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8111                 goto err_out;
8112         }
8113
8114         rc = bnxt_setup_vnic(bp, 0);
8115         if (rc)
8116                 goto err_out;
8117
8118         if (bp->flags & BNXT_FLAG_RFS) {
8119                 rc = bnxt_alloc_rfs_vnics(bp);
8120                 if (rc)
8121                         goto err_out;
8122         }
8123
8124         if (bp->flags & BNXT_FLAG_TPA) {
8125                 rc = bnxt_set_tpa(bp, true);
8126                 if (rc)
8127                         goto err_out;
8128         }
8129
8130         if (BNXT_VF(bp))
8131                 bnxt_update_vf_mac(bp);
8132
8133         /* Filter for default vnic 0 */
8134         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8135         if (rc) {
8136                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8137                 goto err_out;
8138         }
8139         vnic->uc_filter_count = 1;
8140
8141         vnic->rx_mask = 0;
8142         if (bp->dev->flags & IFF_BROADCAST)
8143                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8144
8145         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
8146                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8147
8148         if (bp->dev->flags & IFF_ALLMULTI) {
8149                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8150                 vnic->mc_list_count = 0;
8151         } else {
8152                 u32 mask = 0;
8153
8154                 bnxt_mc_list_updated(bp, &mask);
8155                 vnic->rx_mask |= mask;
8156         }
8157
8158         rc = bnxt_cfg_rx_mode(bp);
8159         if (rc)
8160                 goto err_out;
8161
8162         rc = bnxt_hwrm_set_coal(bp);
8163         if (rc)
8164                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8165                                 rc);
8166
8167         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8168                 rc = bnxt_setup_nitroa0_vnic(bp);
8169                 if (rc)
8170                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8171                                    rc);
8172         }
8173
8174         if (BNXT_VF(bp)) {
8175                 bnxt_hwrm_func_qcfg(bp);
8176                 netdev_update_features(bp->dev);
8177         }
8178
8179         return 0;
8180
8181 err_out:
8182         bnxt_hwrm_resource_free(bp, 0, true);
8183
8184         return rc;
8185 }
8186
8187 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8188 {
8189         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8190         return 0;
8191 }
8192
8193 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8194 {
8195         bnxt_init_cp_rings(bp);
8196         bnxt_init_rx_rings(bp);
8197         bnxt_init_tx_rings(bp);
8198         bnxt_init_ring_grps(bp, irq_re_init);
8199         bnxt_init_vnics(bp);
8200
8201         return bnxt_init_chip(bp, irq_re_init);
8202 }
8203
8204 static int bnxt_set_real_num_queues(struct bnxt *bp)
8205 {
8206         int rc;
8207         struct net_device *dev = bp->dev;
8208
8209         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8210                                           bp->tx_nr_rings_xdp);
8211         if (rc)
8212                 return rc;
8213
8214         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8215         if (rc)
8216                 return rc;
8217
8218 #ifdef CONFIG_RFS_ACCEL
8219         if (bp->flags & BNXT_FLAG_RFS)
8220                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8221 #endif
8222
8223         return rc;
8224 }
8225
8226 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8227                            bool shared)
8228 {
8229         int _rx = *rx, _tx = *tx;
8230
8231         if (shared) {
8232                 *rx = min_t(int, _rx, max);
8233                 *tx = min_t(int, _tx, max);
8234         } else {
8235                 if (max < 2)
8236                         return -ENOMEM;
8237
8238                 while (_rx + _tx > max) {
8239                         if (_rx > _tx && _rx > 1)
8240                                 _rx--;
8241                         else if (_tx > 1)
8242                                 _tx--;
8243                 }
8244                 *rx = _rx;
8245                 *tx = _tx;
8246         }
8247         return 0;
8248 }
8249
8250 static void bnxt_setup_msix(struct bnxt *bp)
8251 {
8252         const int len = sizeof(bp->irq_tbl[0].name);
8253         struct net_device *dev = bp->dev;
8254         int tcs, i;
8255
8256         tcs = netdev_get_num_tc(dev);
8257         if (tcs) {
8258                 int i, off, count;
8259
8260                 for (i = 0; i < tcs; i++) {
8261                         count = bp->tx_nr_rings_per_tc;
8262                         off = i * count;
8263                         netdev_set_tc_queue(dev, i, count, off);
8264                 }
8265         }
8266
8267         for (i = 0; i < bp->cp_nr_rings; i++) {
8268                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8269                 char *attr;
8270
8271                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8272                         attr = "TxRx";
8273                 else if (i < bp->rx_nr_rings)
8274                         attr = "rx";
8275                 else
8276                         attr = "tx";
8277
8278                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8279                          attr, i);
8280                 bp->irq_tbl[map_idx].handler = bnxt_msix;
8281         }
8282 }
8283
8284 static void bnxt_setup_inta(struct bnxt *bp)
8285 {
8286         const int len = sizeof(bp->irq_tbl[0].name);
8287
8288         if (netdev_get_num_tc(bp->dev))
8289                 netdev_reset_tc(bp->dev);
8290
8291         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
8292                  0);
8293         bp->irq_tbl[0].handler = bnxt_inta;
8294 }
8295
8296 static int bnxt_setup_int_mode(struct bnxt *bp)
8297 {
8298         int rc;
8299
8300         if (bp->flags & BNXT_FLAG_USING_MSIX)
8301                 bnxt_setup_msix(bp);
8302         else
8303                 bnxt_setup_inta(bp);
8304
8305         rc = bnxt_set_real_num_queues(bp);
8306         return rc;
8307 }
8308
8309 #ifdef CONFIG_RFS_ACCEL
8310 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
8311 {
8312         return bp->hw_resc.max_rsscos_ctxs;
8313 }
8314
8315 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
8316 {
8317         return bp->hw_resc.max_vnics;
8318 }
8319 #endif
8320
8321 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
8322 {
8323         return bp->hw_resc.max_stat_ctxs;
8324 }
8325
8326 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
8327 {
8328         return bp->hw_resc.max_cp_rings;
8329 }
8330
8331 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
8332 {
8333         unsigned int cp = bp->hw_resc.max_cp_rings;
8334
8335         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8336                 cp -= bnxt_get_ulp_msix_num(bp);
8337
8338         return cp;
8339 }
8340
8341 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
8342 {
8343         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8344
8345         if (bp->flags & BNXT_FLAG_CHIP_P5)
8346                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
8347
8348         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
8349 }
8350
8351 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
8352 {
8353         bp->hw_resc.max_irqs = max_irqs;
8354 }
8355
8356 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
8357 {
8358         unsigned int cp;
8359
8360         cp = bnxt_get_max_func_cp_rings_for_en(bp);
8361         if (bp->flags & BNXT_FLAG_CHIP_P5)
8362                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
8363         else
8364                 return cp - bp->cp_nr_rings;
8365 }
8366
8367 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
8368 {
8369         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8370 }
8371
8372 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8373 {
8374         int max_cp = bnxt_get_max_func_cp_rings(bp);
8375         int max_irq = bnxt_get_max_func_irqs(bp);
8376         int total_req = bp->cp_nr_rings + num;
8377         int max_idx, avail_msix;
8378
8379         max_idx = bp->total_irqs;
8380         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8381                 max_idx = min_t(int, bp->total_irqs, max_cp);
8382         avail_msix = max_idx - bp->cp_nr_rings;
8383         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8384                 return avail_msix;
8385
8386         if (max_irq < total_req) {
8387                 num = max_irq - bp->cp_nr_rings;
8388                 if (num <= 0)
8389                         return 0;
8390         }
8391         return num;
8392 }
8393
8394 static int bnxt_get_num_msix(struct bnxt *bp)
8395 {
8396         if (!BNXT_NEW_RM(bp))
8397                 return bnxt_get_max_func_irqs(bp);
8398
8399         return bnxt_nq_rings_in_use(bp);
8400 }
8401
8402 static int bnxt_init_msix(struct bnxt *bp)
8403 {
8404         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8405         struct msix_entry *msix_ent;
8406
8407         total_vecs = bnxt_get_num_msix(bp);
8408         max = bnxt_get_max_func_irqs(bp);
8409         if (total_vecs > max)
8410                 total_vecs = max;
8411
8412         if (!total_vecs)
8413                 return 0;
8414
8415         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8416         if (!msix_ent)
8417                 return -ENOMEM;
8418
8419         for (i = 0; i < total_vecs; i++) {
8420                 msix_ent[i].entry = i;
8421                 msix_ent[i].vector = 0;
8422         }
8423
8424         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8425                 min = 2;
8426
8427         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8428         ulp_msix = bnxt_get_ulp_msix_num(bp);
8429         if (total_vecs < 0 || total_vecs < ulp_msix) {
8430                 rc = -ENODEV;
8431                 goto msix_setup_exit;
8432         }
8433
8434         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8435         if (bp->irq_tbl) {
8436                 for (i = 0; i < total_vecs; i++)
8437                         bp->irq_tbl[i].vector = msix_ent[i].vector;
8438
8439                 bp->total_irqs = total_vecs;
8440                 /* Trim rings based upon num of vectors allocated */
8441                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8442                                      total_vecs - ulp_msix, min == 1);
8443                 if (rc)
8444                         goto msix_setup_exit;
8445
8446                 bp->cp_nr_rings = (min == 1) ?
8447                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8448                                   bp->tx_nr_rings + bp->rx_nr_rings;
8449
8450         } else {
8451                 rc = -ENOMEM;
8452                 goto msix_setup_exit;
8453         }
8454         bp->flags |= BNXT_FLAG_USING_MSIX;
8455         kfree(msix_ent);
8456         return 0;
8457
8458 msix_setup_exit:
8459         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8460         kfree(bp->irq_tbl);
8461         bp->irq_tbl = NULL;
8462         pci_disable_msix(bp->pdev);
8463         kfree(msix_ent);
8464         return rc;
8465 }
8466
8467 static int bnxt_init_inta(struct bnxt *bp)
8468 {
8469         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
8470         if (!bp->irq_tbl)
8471                 return -ENOMEM;
8472
8473         bp->total_irqs = 1;
8474         bp->rx_nr_rings = 1;
8475         bp->tx_nr_rings = 1;
8476         bp->cp_nr_rings = 1;
8477         bp->flags |= BNXT_FLAG_SHARED_RINGS;
8478         bp->irq_tbl[0].vector = bp->pdev->irq;
8479         return 0;
8480 }
8481
8482 static int bnxt_init_int_mode(struct bnxt *bp)
8483 {
8484         int rc = 0;
8485
8486         if (bp->flags & BNXT_FLAG_MSIX_CAP)
8487                 rc = bnxt_init_msix(bp);
8488
8489         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8490                 /* fallback to INTA */
8491                 rc = bnxt_init_inta(bp);
8492         }
8493         return rc;
8494 }
8495
8496 static void bnxt_clear_int_mode(struct bnxt *bp)
8497 {
8498         if (bp->flags & BNXT_FLAG_USING_MSIX)
8499                 pci_disable_msix(bp->pdev);
8500
8501         kfree(bp->irq_tbl);
8502         bp->irq_tbl = NULL;
8503         bp->flags &= ~BNXT_FLAG_USING_MSIX;
8504 }
8505
8506 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8507 {
8508         int tcs = netdev_get_num_tc(bp->dev);
8509         bool irq_cleared = false;
8510         int rc;
8511
8512         if (!bnxt_need_reserve_rings(bp))
8513                 return 0;
8514
8515         if (irq_re_init && BNXT_NEW_RM(bp) &&
8516             bnxt_get_num_msix(bp) != bp->total_irqs) {
8517                 bnxt_ulp_irq_stop(bp);
8518                 bnxt_clear_int_mode(bp);
8519                 irq_cleared = true;
8520         }
8521         rc = __bnxt_reserve_rings(bp);
8522         if (irq_cleared) {
8523                 if (!rc)
8524                         rc = bnxt_init_int_mode(bp);
8525                 bnxt_ulp_irq_restart(bp, rc);
8526         }
8527         if (rc) {
8528                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8529                 return rc;
8530         }
8531         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8532                 netdev_err(bp->dev, "tx ring reservation failure\n");
8533                 netdev_reset_tc(bp->dev);
8534                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8535                 return -ENOMEM;
8536         }
8537         return 0;
8538 }
8539
8540 static void bnxt_free_irq(struct bnxt *bp)
8541 {
8542         struct bnxt_irq *irq;
8543         int i;
8544
8545 #ifdef CONFIG_RFS_ACCEL
8546         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8547         bp->dev->rx_cpu_rmap = NULL;
8548 #endif
8549         if (!bp->irq_tbl || !bp->bnapi)
8550                 return;
8551
8552         for (i = 0; i < bp->cp_nr_rings; i++) {
8553                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8554
8555                 irq = &bp->irq_tbl[map_idx];
8556                 if (irq->requested) {
8557                         if (irq->have_cpumask) {
8558                                 irq_set_affinity_hint(irq->vector, NULL);
8559                                 free_cpumask_var(irq->cpu_mask);
8560                                 irq->have_cpumask = 0;
8561                         }
8562                         free_irq(irq->vector, bp->bnapi[i]);
8563                 }
8564
8565                 irq->requested = 0;
8566         }
8567 }
8568
8569 static int bnxt_request_irq(struct bnxt *bp)
8570 {
8571         int i, j, rc = 0;
8572         unsigned long flags = 0;
8573 #ifdef CONFIG_RFS_ACCEL
8574         struct cpu_rmap *rmap;
8575 #endif
8576
8577         rc = bnxt_setup_int_mode(bp);
8578         if (rc) {
8579                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8580                            rc);
8581                 return rc;
8582         }
8583 #ifdef CONFIG_RFS_ACCEL
8584         rmap = bp->dev->rx_cpu_rmap;
8585 #endif
8586         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8587                 flags = IRQF_SHARED;
8588
8589         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8590                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8591                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8592
8593 #ifdef CONFIG_RFS_ACCEL
8594                 if (rmap && bp->bnapi[i]->rx_ring) {
8595                         rc = irq_cpu_rmap_add(rmap, irq->vector);
8596                         if (rc)
8597                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8598                                             j);
8599                         j++;
8600                 }
8601 #endif
8602                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8603                                  bp->bnapi[i]);
8604                 if (rc)
8605                         break;
8606
8607                 irq->requested = 1;
8608
8609                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8610                         int numa_node = dev_to_node(&bp->pdev->dev);
8611
8612                         irq->have_cpumask = 1;
8613                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8614                                         irq->cpu_mask);
8615                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8616                         if (rc) {
8617                                 netdev_warn(bp->dev,
8618                                             "Set affinity failed, IRQ = %d\n",
8619                                             irq->vector);
8620                                 break;
8621                         }
8622                 }
8623         }
8624         return rc;
8625 }
8626
8627 static void bnxt_del_napi(struct bnxt *bp)
8628 {
8629         int i;
8630
8631         if (!bp->bnapi)
8632                 return;
8633
8634         for (i = 0; i < bp->cp_nr_rings; i++) {
8635                 struct bnxt_napi *bnapi = bp->bnapi[i];
8636
8637                 napi_hash_del(&bnapi->napi);
8638                 netif_napi_del(&bnapi->napi);
8639         }
8640         /* We called napi_hash_del() before netif_napi_del(), we need
8641          * to respect an RCU grace period before freeing napi structures.
8642          */
8643         synchronize_net();
8644 }
8645
8646 static void bnxt_init_napi(struct bnxt *bp)
8647 {
8648         int i;
8649         unsigned int cp_nr_rings = bp->cp_nr_rings;
8650         struct bnxt_napi *bnapi;
8651
8652         if (bp->flags & BNXT_FLAG_USING_MSIX) {
8653                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8654
8655                 if (bp->flags & BNXT_FLAG_CHIP_P5)
8656                         poll_fn = bnxt_poll_p5;
8657                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8658                         cp_nr_rings--;
8659                 for (i = 0; i < cp_nr_rings; i++) {
8660                         bnapi = bp->bnapi[i];
8661                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8662                 }
8663                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8664                         bnapi = bp->bnapi[cp_nr_rings];
8665                         netif_napi_add(bp->dev, &bnapi->napi,
8666                                        bnxt_poll_nitroa0, 64);
8667                 }
8668         } else {
8669                 bnapi = bp->bnapi[0];
8670                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8671         }
8672 }
8673
8674 static void bnxt_disable_napi(struct bnxt *bp)
8675 {
8676         int i;
8677
8678         if (!bp->bnapi)
8679                 return;
8680
8681         for (i = 0; i < bp->cp_nr_rings; i++) {
8682                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8683
8684                 if (bp->bnapi[i]->rx_ring)
8685                         cancel_work_sync(&cpr->dim.work);
8686
8687                 napi_disable(&bp->bnapi[i]->napi);
8688         }
8689 }
8690
8691 static void bnxt_enable_napi(struct bnxt *bp)
8692 {
8693         int i;
8694
8695         for (i = 0; i < bp->cp_nr_rings; i++) {
8696                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8697                 bp->bnapi[i]->in_reset = false;
8698
8699                 if (bp->bnapi[i]->rx_ring) {
8700                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8701                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8702                 }
8703                 napi_enable(&bp->bnapi[i]->napi);
8704         }
8705 }
8706
8707 void bnxt_tx_disable(struct bnxt *bp)
8708 {
8709         int i;
8710         struct bnxt_tx_ring_info *txr;
8711
8712         if (bp->tx_ring) {
8713                 for (i = 0; i < bp->tx_nr_rings; i++) {
8714                         txr = &bp->tx_ring[i];
8715                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
8716                 }
8717         }
8718         /* Stop all TX queues */
8719         netif_tx_disable(bp->dev);
8720         netif_carrier_off(bp->dev);
8721 }
8722
8723 void bnxt_tx_enable(struct bnxt *bp)
8724 {
8725         int i;
8726         struct bnxt_tx_ring_info *txr;
8727
8728         for (i = 0; i < bp->tx_nr_rings; i++) {
8729                 txr = &bp->tx_ring[i];
8730                 txr->dev_state = 0;
8731         }
8732         netif_tx_wake_all_queues(bp->dev);
8733         if (bp->link_info.link_up)
8734                 netif_carrier_on(bp->dev);
8735 }
8736
8737 static void bnxt_report_link(struct bnxt *bp)
8738 {
8739         if (bp->link_info.link_up) {
8740                 const char *duplex;
8741                 const char *flow_ctrl;
8742                 u32 speed;
8743                 u16 fec;
8744
8745                 netif_carrier_on(bp->dev);
8746                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8747                         duplex = "full";
8748                 else
8749                         duplex = "half";
8750                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8751                         flow_ctrl = "ON - receive & transmit";
8752                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8753                         flow_ctrl = "ON - transmit";
8754                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8755                         flow_ctrl = "ON - receive";
8756                 else
8757                         flow_ctrl = "none";
8758                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
8759                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
8760                             speed, duplex, flow_ctrl);
8761                 if (bp->flags & BNXT_FLAG_EEE_CAP)
8762                         netdev_info(bp->dev, "EEE is %s\n",
8763                                     bp->eee.eee_active ? "active" :
8764                                                          "not active");
8765                 fec = bp->link_info.fec_cfg;
8766                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8767                         netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
8768                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8769                                     (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
8770                                      (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
8771         } else {
8772                 netif_carrier_off(bp->dev);
8773                 netdev_err(bp->dev, "NIC Link is Down\n");
8774         }
8775 }
8776
8777 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8778 {
8779         int rc = 0;
8780         struct hwrm_port_phy_qcaps_input req = {0};
8781         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8782         struct bnxt_link_info *link_info = &bp->link_info;
8783
8784         bp->flags &= ~BNXT_FLAG_EEE_CAP;
8785         if (bp->test_info)
8786                 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK |
8787                                           BNXT_TEST_FL_AN_PHY_LPBK);
8788         if (bp->hwrm_spec_code < 0x10201)
8789                 return 0;
8790
8791         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8792
8793         mutex_lock(&bp->hwrm_cmd_lock);
8794         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8795         if (rc)
8796                 goto hwrm_phy_qcaps_exit;
8797
8798         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
8799                 struct ethtool_eee *eee = &bp->eee;
8800                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8801
8802                 bp->flags |= BNXT_FLAG_EEE_CAP;
8803                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8804                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8805                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8806                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8807                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8808         }
8809         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8810                 if (bp->test_info)
8811                         bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8812         }
8813         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) {
8814                 if (bp->test_info)
8815                         bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK;
8816         }
8817         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) {
8818                 if (BNXT_PF(bp))
8819                         bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG;
8820         }
8821         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET)
8822                 bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET;
8823
8824         if (resp->supported_speeds_auto_mode)
8825                 link_info->support_auto_speeds =
8826                         le16_to_cpu(resp->supported_speeds_auto_mode);
8827
8828         bp->port_count = resp->port_cnt;
8829
8830 hwrm_phy_qcaps_exit:
8831         mutex_unlock(&bp->hwrm_cmd_lock);
8832         return rc;
8833 }
8834
8835 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
8836 {
8837         int rc = 0;
8838         struct bnxt_link_info *link_info = &bp->link_info;
8839         struct hwrm_port_phy_qcfg_input req = {0};
8840         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8841         u8 link_up = link_info->link_up;
8842         u16 diff;
8843
8844         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
8845
8846         mutex_lock(&bp->hwrm_cmd_lock);
8847         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8848         if (rc) {
8849                 mutex_unlock(&bp->hwrm_cmd_lock);
8850                 return rc;
8851         }
8852
8853         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
8854         link_info->phy_link_status = resp->link;
8855         link_info->duplex = resp->duplex_cfg;
8856         if (bp->hwrm_spec_code >= 0x10800)
8857                 link_info->duplex = resp->duplex_state;
8858         link_info->pause = resp->pause;
8859         link_info->auto_mode = resp->auto_mode;
8860         link_info->auto_pause_setting = resp->auto_pause;
8861         link_info->lp_pause = resp->link_partner_adv_pause;
8862         link_info->force_pause_setting = resp->force_pause;
8863         link_info->duplex_setting = resp->duplex_cfg;
8864         if (link_info->phy_link_status == BNXT_LINK_LINK)
8865                 link_info->link_speed = le16_to_cpu(resp->link_speed);
8866         else
8867                 link_info->link_speed = 0;
8868         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
8869         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
8870         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
8871         link_info->lp_auto_link_speeds =
8872                 le16_to_cpu(resp->link_partner_adv_speeds);
8873         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
8874         link_info->phy_ver[0] = resp->phy_maj;
8875         link_info->phy_ver[1] = resp->phy_min;
8876         link_info->phy_ver[2] = resp->phy_bld;
8877         link_info->media_type = resp->media_type;
8878         link_info->phy_type = resp->phy_type;
8879         link_info->transceiver = resp->xcvr_pkg_type;
8880         link_info->phy_addr = resp->eee_config_phy_addr &
8881                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
8882         link_info->module_status = resp->module_status;
8883
8884         if (bp->flags & BNXT_FLAG_EEE_CAP) {
8885                 struct ethtool_eee *eee = &bp->eee;
8886                 u16 fw_speeds;
8887
8888                 eee->eee_active = 0;
8889                 if (resp->eee_config_phy_addr &
8890                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
8891                         eee->eee_active = 1;
8892                         fw_speeds = le16_to_cpu(
8893                                 resp->link_partner_adv_eee_link_speed_mask);
8894                         eee->lp_advertised =
8895                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8896                 }
8897
8898                 /* Pull initial EEE config */
8899                 if (!chng_link_state) {
8900                         if (resp->eee_config_phy_addr &
8901                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
8902                                 eee->eee_enabled = 1;
8903
8904                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
8905                         eee->advertised =
8906                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8907
8908                         if (resp->eee_config_phy_addr &
8909                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
8910                                 __le32 tmr;
8911
8912                                 eee->tx_lpi_enabled = 1;
8913                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
8914                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
8915                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
8916                         }
8917                 }
8918         }
8919
8920         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
8921         if (bp->hwrm_spec_code >= 0x10504)
8922                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
8923
8924         /* TODO: need to add more logic to report VF link */
8925         if (chng_link_state) {
8926                 if (link_info->phy_link_status == BNXT_LINK_LINK)
8927                         link_info->link_up = 1;
8928                 else
8929                         link_info->link_up = 0;
8930                 if (link_up != link_info->link_up)
8931                         bnxt_report_link(bp);
8932         } else {
8933                 /* alwasy link down if not require to update link state */
8934                 link_info->link_up = 0;
8935         }
8936         mutex_unlock(&bp->hwrm_cmd_lock);
8937
8938         if (!BNXT_PHY_CFG_ABLE(bp))
8939                 return 0;
8940
8941         diff = link_info->support_auto_speeds ^ link_info->advertising;
8942         if ((link_info->support_auto_speeds | diff) !=
8943             link_info->support_auto_speeds) {
8944                 /* An advertised speed is no longer supported, so we need to
8945                  * update the advertisement settings.  Caller holds RTNL
8946                  * so we can modify link settings.
8947                  */
8948                 link_info->advertising = link_info->support_auto_speeds;
8949                 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
8950                         bnxt_hwrm_set_link_setting(bp, true, false);
8951         }
8952         return 0;
8953 }
8954
8955 static void bnxt_get_port_module_status(struct bnxt *bp)
8956 {
8957         struct bnxt_link_info *link_info = &bp->link_info;
8958         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
8959         u8 module_status;
8960
8961         if (bnxt_update_link(bp, true))
8962                 return;
8963
8964         module_status = link_info->module_status;
8965         switch (module_status) {
8966         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
8967         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
8968         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
8969                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
8970                             bp->pf.port_id);
8971                 if (bp->hwrm_spec_code >= 0x10201) {
8972                         netdev_warn(bp->dev, "Module part number %s\n",
8973                                     resp->phy_vendor_partnumber);
8974                 }
8975                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
8976                         netdev_warn(bp->dev, "TX is disabled\n");
8977                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
8978                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
8979         }
8980 }
8981
8982 static void
8983 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
8984 {
8985         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
8986                 if (bp->hwrm_spec_code >= 0x10201)
8987                         req->auto_pause =
8988                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
8989                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8990                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
8991                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8992                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
8993                 req->enables |=
8994                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8995         } else {
8996                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8997                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
8998                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8999                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9000                 req->enables |=
9001                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9002                 if (bp->hwrm_spec_code >= 0x10201) {
9003                         req->auto_pause = req->force_pause;
9004                         req->enables |= cpu_to_le32(
9005                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9006                 }
9007         }
9008 }
9009
9010 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
9011                                       struct hwrm_port_phy_cfg_input *req)
9012 {
9013         u8 autoneg = bp->link_info.autoneg;
9014         u16 fw_link_speed = bp->link_info.req_link_speed;
9015         u16 advertising = bp->link_info.advertising;
9016
9017         if (autoneg & BNXT_AUTONEG_SPEED) {
9018                 req->auto_mode |=
9019                         PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9020
9021                 req->enables |= cpu_to_le32(
9022                         PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9023                 req->auto_link_speed_mask = cpu_to_le16(advertising);
9024
9025                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9026                 req->flags |=
9027                         cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9028         } else {
9029                 req->force_link_speed = cpu_to_le16(fw_link_speed);
9030                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9031         }
9032
9033         /* tell chimp that the setting takes effect immediately */
9034         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9035 }
9036
9037 int bnxt_hwrm_set_pause(struct bnxt *bp)
9038 {
9039         struct hwrm_port_phy_cfg_input req = {0};
9040         int rc;
9041
9042         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9043         bnxt_hwrm_set_pause_common(bp, &req);
9044
9045         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9046             bp->link_info.force_link_chng)
9047                 bnxt_hwrm_set_link_common(bp, &req);
9048
9049         mutex_lock(&bp->hwrm_cmd_lock);
9050         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9051         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9052                 /* since changing of pause setting doesn't trigger any link
9053                  * change event, the driver needs to update the current pause
9054                  * result upon successfully return of the phy_cfg command
9055                  */
9056                 bp->link_info.pause =
9057                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9058                 bp->link_info.auto_pause_setting = 0;
9059                 if (!bp->link_info.force_link_chng)
9060                         bnxt_report_link(bp);
9061         }
9062         bp->link_info.force_link_chng = false;
9063         mutex_unlock(&bp->hwrm_cmd_lock);
9064         return rc;
9065 }
9066
9067 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9068                               struct hwrm_port_phy_cfg_input *req)
9069 {
9070         struct ethtool_eee *eee = &bp->eee;
9071
9072         if (eee->eee_enabled) {
9073                 u16 eee_speeds;
9074                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9075
9076                 if (eee->tx_lpi_enabled)
9077                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9078                 else
9079                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9080
9081                 req->flags |= cpu_to_le32(flags);
9082                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9083                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9084                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9085         } else {
9086                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9087         }
9088 }
9089
9090 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9091 {
9092         struct hwrm_port_phy_cfg_input req = {0};
9093
9094         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9095         if (set_pause)
9096                 bnxt_hwrm_set_pause_common(bp, &req);
9097
9098         bnxt_hwrm_set_link_common(bp, &req);
9099
9100         if (set_eee)
9101                 bnxt_hwrm_set_eee(bp, &req);
9102         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9103 }
9104
9105 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9106 {
9107         struct hwrm_port_phy_cfg_input req = {0};
9108
9109         if (!BNXT_SINGLE_PF(bp))
9110                 return 0;
9111
9112         if (pci_num_vf(bp->pdev))
9113                 return 0;
9114
9115         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9116         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9117         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9118 }
9119
9120 static int bnxt_fw_init_one(struct bnxt *bp);
9121
9122 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
9123 {
9124         struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
9125         struct hwrm_func_drv_if_change_input req = {0};
9126         bool resc_reinit = false, fw_reset = false;
9127         u32 flags = 0;
9128         int rc;
9129
9130         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
9131                 return 0;
9132
9133         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
9134         if (up)
9135                 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
9136         mutex_lock(&bp->hwrm_cmd_lock);
9137         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9138         if (!rc)
9139                 flags = le32_to_cpu(resp->flags);
9140         mutex_unlock(&bp->hwrm_cmd_lock);
9141         if (rc)
9142                 return rc;
9143
9144         if (!up)
9145                 return 0;
9146
9147         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
9148                 resc_reinit = true;
9149         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
9150                 fw_reset = true;
9151
9152         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
9153                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
9154                 return -ENODEV;
9155         }
9156         if (resc_reinit || fw_reset) {
9157                 if (fw_reset) {
9158                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9159                                 bnxt_ulp_stop(bp);
9160                         bnxt_free_ctx_mem(bp);
9161                         kfree(bp->ctx);
9162                         bp->ctx = NULL;
9163                         bnxt_dcb_free(bp);
9164                         rc = bnxt_fw_init_one(bp);
9165                         if (rc) {
9166                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9167                                 return rc;
9168                         }
9169                         bnxt_clear_int_mode(bp);
9170                         rc = bnxt_init_int_mode(bp);
9171                         if (rc) {
9172                                 netdev_err(bp->dev, "init int mode failed\n");
9173                                 return rc;
9174                         }
9175                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9176                 }
9177                 if (BNXT_NEW_RM(bp)) {
9178                         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9179
9180                         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9181                         hw_resc->resv_cp_rings = 0;
9182                         hw_resc->resv_stat_ctxs = 0;
9183                         hw_resc->resv_irqs = 0;
9184                         hw_resc->resv_tx_rings = 0;
9185                         hw_resc->resv_rx_rings = 0;
9186                         hw_resc->resv_hw_ring_grps = 0;
9187                         hw_resc->resv_vnics = 0;
9188                         if (!fw_reset) {
9189                                 bp->tx_nr_rings = 0;
9190                                 bp->rx_nr_rings = 0;
9191                         }
9192                 }
9193         }
9194         return 0;
9195 }
9196
9197 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
9198 {
9199         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9200         struct hwrm_port_led_qcaps_input req = {0};
9201         struct bnxt_pf_info *pf = &bp->pf;
9202         int rc;
9203
9204         bp->num_leds = 0;
9205         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
9206                 return 0;
9207
9208         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
9209         req.port_id = cpu_to_le16(pf->port_id);
9210         mutex_lock(&bp->hwrm_cmd_lock);
9211         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9212         if (rc) {
9213                 mutex_unlock(&bp->hwrm_cmd_lock);
9214                 return rc;
9215         }
9216         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
9217                 int i;
9218
9219                 bp->num_leds = resp->num_leds;
9220                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
9221                                                  bp->num_leds);
9222                 for (i = 0; i < bp->num_leds; i++) {
9223                         struct bnxt_led_info *led = &bp->leds[i];
9224                         __le16 caps = led->led_state_caps;
9225
9226                         if (!led->led_group_id ||
9227                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
9228                                 bp->num_leds = 0;
9229                                 break;
9230                         }
9231                 }
9232         }
9233         mutex_unlock(&bp->hwrm_cmd_lock);
9234         return 0;
9235 }
9236
9237 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
9238 {
9239         struct hwrm_wol_filter_alloc_input req = {0};
9240         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
9241         int rc;
9242
9243         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
9244         req.port_id = cpu_to_le16(bp->pf.port_id);
9245         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
9246         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
9247         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
9248         mutex_lock(&bp->hwrm_cmd_lock);
9249         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9250         if (!rc)
9251                 bp->wol_filter_id = resp->wol_filter_id;
9252         mutex_unlock(&bp->hwrm_cmd_lock);
9253         return rc;
9254 }
9255
9256 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
9257 {
9258         struct hwrm_wol_filter_free_input req = {0};
9259
9260         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
9261         req.port_id = cpu_to_le16(bp->pf.port_id);
9262         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
9263         req.wol_filter_id = bp->wol_filter_id;
9264         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9265 }
9266
9267 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
9268 {
9269         struct hwrm_wol_filter_qcfg_input req = {0};
9270         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9271         u16 next_handle = 0;
9272         int rc;
9273
9274         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
9275         req.port_id = cpu_to_le16(bp->pf.port_id);
9276         req.handle = cpu_to_le16(handle);
9277         mutex_lock(&bp->hwrm_cmd_lock);
9278         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9279         if (!rc) {
9280                 next_handle = le16_to_cpu(resp->next_handle);
9281                 if (next_handle != 0) {
9282                         if (resp->wol_type ==
9283                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
9284                                 bp->wol = 1;
9285                                 bp->wol_filter_id = resp->wol_filter_id;
9286                         }
9287                 }
9288         }
9289         mutex_unlock(&bp->hwrm_cmd_lock);
9290         return next_handle;
9291 }
9292
9293 static void bnxt_get_wol_settings(struct bnxt *bp)
9294 {
9295         u16 handle = 0;
9296
9297         bp->wol = 0;
9298         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
9299                 return;
9300
9301         do {
9302                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
9303         } while (handle && handle != 0xffff);
9304 }
9305
9306 #ifdef CONFIG_BNXT_HWMON
9307 static ssize_t bnxt_show_temp(struct device *dev,
9308                               struct device_attribute *devattr, char *buf)
9309 {
9310         struct hwrm_temp_monitor_query_input req = {0};
9311         struct hwrm_temp_monitor_query_output *resp;
9312         struct bnxt *bp = dev_get_drvdata(dev);
9313         u32 len = 0;
9314
9315         resp = bp->hwrm_cmd_resp_addr;
9316         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9317         mutex_lock(&bp->hwrm_cmd_lock);
9318         if (!_hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT))
9319                 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
9320         mutex_unlock(&bp->hwrm_cmd_lock);
9321
9322         if (len)
9323                 return len;
9324
9325         return sprintf(buf, "unknown\n");
9326 }
9327 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
9328
9329 static struct attribute *bnxt_attrs[] = {
9330         &sensor_dev_attr_temp1_input.dev_attr.attr,
9331         NULL
9332 };
9333 ATTRIBUTE_GROUPS(bnxt);
9334
9335 static void bnxt_hwmon_close(struct bnxt *bp)
9336 {
9337         if (bp->hwmon_dev) {
9338                 hwmon_device_unregister(bp->hwmon_dev);
9339                 bp->hwmon_dev = NULL;
9340         }
9341 }
9342
9343 static void bnxt_hwmon_open(struct bnxt *bp)
9344 {
9345         struct pci_dev *pdev = bp->pdev;
9346
9347         if (bp->hwmon_dev)
9348                 return;
9349
9350         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
9351                                                           DRV_MODULE_NAME, bp,
9352                                                           bnxt_groups);
9353         if (IS_ERR(bp->hwmon_dev)) {
9354                 bp->hwmon_dev = NULL;
9355                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
9356         }
9357 }
9358 #else
9359 static void bnxt_hwmon_close(struct bnxt *bp)
9360 {
9361 }
9362
9363 static void bnxt_hwmon_open(struct bnxt *bp)
9364 {
9365 }
9366 #endif
9367
9368 static bool bnxt_eee_config_ok(struct bnxt *bp)
9369 {
9370         struct ethtool_eee *eee = &bp->eee;
9371         struct bnxt_link_info *link_info = &bp->link_info;
9372
9373         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
9374                 return true;
9375
9376         if (eee->eee_enabled) {
9377                 u32 advertising =
9378                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
9379
9380                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9381                         eee->eee_enabled = 0;
9382                         return false;
9383                 }
9384                 if (eee->advertised & ~advertising) {
9385                         eee->advertised = advertising & eee->supported;
9386                         return false;
9387                 }
9388         }
9389         return true;
9390 }
9391
9392 static int bnxt_update_phy_setting(struct bnxt *bp)
9393 {
9394         int rc;
9395         bool update_link = false;
9396         bool update_pause = false;
9397         bool update_eee = false;
9398         struct bnxt_link_info *link_info = &bp->link_info;
9399
9400         rc = bnxt_update_link(bp, true);
9401         if (rc) {
9402                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9403                            rc);
9404                 return rc;
9405         }
9406         if (!BNXT_SINGLE_PF(bp))
9407                 return 0;
9408
9409         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9410             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9411             link_info->req_flow_ctrl)
9412                 update_pause = true;
9413         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9414             link_info->force_pause_setting != link_info->req_flow_ctrl)
9415                 update_pause = true;
9416         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9417                 if (BNXT_AUTO_MODE(link_info->auto_mode))
9418                         update_link = true;
9419                 if (link_info->req_link_speed != link_info->force_link_speed)
9420                         update_link = true;
9421                 if (link_info->req_duplex != link_info->duplex_setting)
9422                         update_link = true;
9423         } else {
9424                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9425                         update_link = true;
9426                 if (link_info->advertising != link_info->auto_link_speeds)
9427                         update_link = true;
9428         }
9429
9430         /* The last close may have shutdown the link, so need to call
9431          * PHY_CFG to bring it back up.
9432          */
9433         if (!bp->link_info.link_up)
9434                 update_link = true;
9435
9436         if (!bnxt_eee_config_ok(bp))
9437                 update_eee = true;
9438
9439         if (update_link)
9440                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9441         else if (update_pause)
9442                 rc = bnxt_hwrm_set_pause(bp);
9443         if (rc) {
9444                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9445                            rc);
9446                 return rc;
9447         }
9448
9449         return rc;
9450 }
9451
9452 /* Common routine to pre-map certain register block to different GRC window.
9453  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
9454  * in PF and 3 windows in VF that can be customized to map in different
9455  * register blocks.
9456  */
9457 static void bnxt_preset_reg_win(struct bnxt *bp)
9458 {
9459         if (BNXT_PF(bp)) {
9460                 /* CAG registers map to GRC window #4 */
9461                 writel(BNXT_CAG_REG_BASE,
9462                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
9463         }
9464 }
9465
9466 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
9467
9468 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9469 {
9470         int rc = 0;
9471
9472         bnxt_preset_reg_win(bp);
9473         netif_carrier_off(bp->dev);
9474         if (irq_re_init) {
9475                 /* Reserve rings now if none were reserved at driver probe. */
9476                 rc = bnxt_init_dflt_ring_mode(bp);
9477                 if (rc) {
9478                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
9479                         return rc;
9480                 }
9481         }
9482         rc = bnxt_reserve_rings(bp, irq_re_init);
9483         if (rc)
9484                 return rc;
9485         if ((bp->flags & BNXT_FLAG_RFS) &&
9486             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
9487                 /* disable RFS if falling back to INTA */
9488                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
9489                 bp->flags &= ~BNXT_FLAG_RFS;
9490         }
9491
9492         rc = bnxt_alloc_mem(bp, irq_re_init);
9493         if (rc) {
9494                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9495                 goto open_err_free_mem;
9496         }
9497
9498         if (irq_re_init) {
9499                 bnxt_init_napi(bp);
9500                 rc = bnxt_request_irq(bp);
9501                 if (rc) {
9502                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
9503                         goto open_err_irq;
9504                 }
9505         }
9506
9507         rc = bnxt_init_nic(bp, irq_re_init);
9508         if (rc) {
9509                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9510                 goto open_err_irq;
9511         }
9512
9513         bnxt_enable_napi(bp);
9514         bnxt_debug_dev_init(bp);
9515
9516         if (link_re_init) {
9517                 mutex_lock(&bp->link_lock);
9518                 rc = bnxt_update_phy_setting(bp);
9519                 mutex_unlock(&bp->link_lock);
9520                 if (rc) {
9521                         netdev_warn(bp->dev, "failed to update phy settings\n");
9522                         if (BNXT_SINGLE_PF(bp)) {
9523                                 bp->link_info.phy_retry = true;
9524                                 bp->link_info.phy_retry_expires =
9525                                         jiffies + 5 * HZ;
9526                         }
9527                 }
9528         }
9529
9530         if (irq_re_init)
9531                 udp_tunnel_nic_reset_ntf(bp->dev);
9532
9533         set_bit(BNXT_STATE_OPEN, &bp->state);
9534         bnxt_enable_int(bp);
9535         /* Enable TX queues */
9536         bnxt_tx_enable(bp);
9537         mod_timer(&bp->timer, jiffies + bp->current_interval);
9538         /* Poll link status and check for SFP+ module status */
9539         bnxt_get_port_module_status(bp);
9540
9541         /* VF-reps may need to be re-opened after the PF is re-opened */
9542         if (BNXT_PF(bp))
9543                 bnxt_vf_reps_open(bp);
9544         return 0;
9545
9546 open_err_irq:
9547         bnxt_del_napi(bp);
9548
9549 open_err_free_mem:
9550         bnxt_free_skbs(bp);
9551         bnxt_free_irq(bp);
9552         bnxt_free_mem(bp, true);
9553         return rc;
9554 }
9555
9556 /* rtnl_lock held */
9557 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9558 {
9559         int rc = 0;
9560
9561         rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
9562         if (rc) {
9563                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
9564                 dev_close(bp->dev);
9565         }
9566         return rc;
9567 }
9568
9569 /* rtnl_lock held, open the NIC half way by allocating all resources, but
9570  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
9571  * self tests.
9572  */
9573 int bnxt_half_open_nic(struct bnxt *bp)
9574 {
9575         int rc = 0;
9576
9577         rc = bnxt_alloc_mem(bp, false);
9578         if (rc) {
9579                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9580                 goto half_open_err;
9581         }
9582         rc = bnxt_init_nic(bp, false);
9583         if (rc) {
9584                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9585                 goto half_open_err;
9586         }
9587         return 0;
9588
9589 half_open_err:
9590         bnxt_free_skbs(bp);
9591         bnxt_free_mem(bp, false);
9592         dev_close(bp->dev);
9593         return rc;
9594 }
9595
9596 /* rtnl_lock held, this call can only be made after a previous successful
9597  * call to bnxt_half_open_nic().
9598  */
9599 void bnxt_half_close_nic(struct bnxt *bp)
9600 {
9601         bnxt_hwrm_resource_free(bp, false, false);
9602         bnxt_free_skbs(bp);
9603         bnxt_free_mem(bp, false);
9604 }
9605
9606 static void bnxt_reenable_sriov(struct bnxt *bp)
9607 {
9608         if (BNXT_PF(bp)) {
9609                 struct bnxt_pf_info *pf = &bp->pf;
9610                 int n = pf->active_vfs;
9611
9612                 if (n)
9613                         bnxt_cfg_hw_sriov(bp, &n, true);
9614         }
9615 }
9616
9617 static int bnxt_open(struct net_device *dev)
9618 {
9619         struct bnxt *bp = netdev_priv(dev);
9620         int rc;
9621
9622         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
9623                 netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n");
9624                 return -ENODEV;
9625         }
9626
9627         rc = bnxt_hwrm_if_change(bp, true);
9628         if (rc)
9629                 return rc;
9630         rc = __bnxt_open_nic(bp, true, true);
9631         if (rc) {
9632                 bnxt_hwrm_if_change(bp, false);
9633         } else {
9634                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
9635                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9636                                 bnxt_ulp_start(bp, 0);
9637                                 bnxt_reenable_sriov(bp);
9638                         }
9639                 }
9640                 bnxt_hwmon_open(bp);
9641         }
9642
9643         return rc;
9644 }
9645
9646 static bool bnxt_drv_busy(struct bnxt *bp)
9647 {
9648         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
9649                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
9650 }
9651
9652 static void bnxt_get_ring_stats(struct bnxt *bp,
9653                                 struct rtnl_link_stats64 *stats);
9654
9655 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
9656                              bool link_re_init)
9657 {
9658         /* Close the VF-reps before closing PF */
9659         if (BNXT_PF(bp))
9660                 bnxt_vf_reps_close(bp);
9661
9662         /* Change device state to avoid TX queue wake up's */
9663         bnxt_tx_disable(bp);
9664
9665         clear_bit(BNXT_STATE_OPEN, &bp->state);
9666         smp_mb__after_atomic();
9667         while (bnxt_drv_busy(bp))
9668                 msleep(20);
9669
9670         /* Flush rings and and disable interrupts */
9671         bnxt_shutdown_nic(bp, irq_re_init);
9672
9673         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
9674
9675         bnxt_debug_dev_exit(bp);
9676         bnxt_disable_napi(bp);
9677         del_timer_sync(&bp->timer);
9678         bnxt_free_skbs(bp);
9679
9680         /* Save ring stats before shutdown */
9681         if (bp->bnapi && irq_re_init)
9682                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
9683         if (irq_re_init) {
9684                 bnxt_free_irq(bp);
9685                 bnxt_del_napi(bp);
9686         }
9687         bnxt_free_mem(bp, irq_re_init);
9688 }
9689
9690 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9691 {
9692         int rc = 0;
9693
9694         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9695                 /* If we get here, it means firmware reset is in progress
9696                  * while we are trying to close.  We can safely proceed with
9697                  * the close because we are holding rtnl_lock().  Some firmware
9698                  * messages may fail as we proceed to close.  We set the
9699                  * ABORT_ERR flag here so that the FW reset thread will later
9700                  * abort when it gets the rtnl_lock() and sees the flag.
9701                  */
9702                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
9703                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9704         }
9705
9706 #ifdef CONFIG_BNXT_SRIOV
9707         if (bp->sriov_cfg) {
9708                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
9709                                                       !bp->sriov_cfg,
9710                                                       BNXT_SRIOV_CFG_WAIT_TMO);
9711                 if (rc)
9712                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
9713         }
9714 #endif
9715         __bnxt_close_nic(bp, irq_re_init, link_re_init);
9716         return rc;
9717 }
9718
9719 static int bnxt_close(struct net_device *dev)
9720 {
9721         struct bnxt *bp = netdev_priv(dev);
9722
9723         bnxt_hwmon_close(bp);
9724         bnxt_close_nic(bp, true, true);
9725         bnxt_hwrm_shutdown_link(bp);
9726         bnxt_hwrm_if_change(bp, false);
9727         return 0;
9728 }
9729
9730 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9731                                    u16 *val)
9732 {
9733         struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9734         struct hwrm_port_phy_mdio_read_input req = {0};
9735         int rc;
9736
9737         if (bp->hwrm_spec_code < 0x10a00)
9738                 return -EOPNOTSUPP;
9739
9740         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9741         req.port_id = cpu_to_le16(bp->pf.port_id);
9742         req.phy_addr = phy_addr;
9743         req.reg_addr = cpu_to_le16(reg & 0x1f);
9744         if (mdio_phy_id_is_c45(phy_addr)) {
9745                 req.cl45_mdio = 1;
9746                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9747                 req.dev_addr = mdio_phy_id_devad(phy_addr);
9748                 req.reg_addr = cpu_to_le16(reg);
9749         }
9750
9751         mutex_lock(&bp->hwrm_cmd_lock);
9752         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9753         if (!rc)
9754                 *val = le16_to_cpu(resp->reg_data);
9755         mutex_unlock(&bp->hwrm_cmd_lock);
9756         return rc;
9757 }
9758
9759 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9760                                     u16 val)
9761 {
9762         struct hwrm_port_phy_mdio_write_input req = {0};
9763
9764         if (bp->hwrm_spec_code < 0x10a00)
9765                 return -EOPNOTSUPP;
9766
9767         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9768         req.port_id = cpu_to_le16(bp->pf.port_id);
9769         req.phy_addr = phy_addr;
9770         req.reg_addr = cpu_to_le16(reg & 0x1f);
9771         if (mdio_phy_id_is_c45(phy_addr)) {
9772                 req.cl45_mdio = 1;
9773                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9774                 req.dev_addr = mdio_phy_id_devad(phy_addr);
9775                 req.reg_addr = cpu_to_le16(reg);
9776         }
9777         req.reg_data = cpu_to_le16(val);
9778
9779         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9780 }
9781
9782 /* rtnl_lock held */
9783 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
9784 {
9785         struct mii_ioctl_data *mdio = if_mii(ifr);
9786         struct bnxt *bp = netdev_priv(dev);
9787         int rc;
9788
9789         switch (cmd) {
9790         case SIOCGMIIPHY:
9791                 mdio->phy_id = bp->link_info.phy_addr;
9792
9793                 fallthrough;
9794         case SIOCGMIIREG: {
9795                 u16 mii_regval = 0;
9796
9797                 if (!netif_running(dev))
9798                         return -EAGAIN;
9799
9800                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
9801                                              &mii_regval);
9802                 mdio->val_out = mii_regval;
9803                 return rc;
9804         }
9805
9806         case SIOCSMIIREG:
9807                 if (!netif_running(dev))
9808                         return -EAGAIN;
9809
9810                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
9811                                                 mdio->val_in);
9812
9813         default:
9814                 /* do nothing */
9815                 break;
9816         }
9817         return -EOPNOTSUPP;
9818 }
9819
9820 static void bnxt_get_ring_stats(struct bnxt *bp,
9821                                 struct rtnl_link_stats64 *stats)
9822 {
9823         int i;
9824
9825         for (i = 0; i < bp->cp_nr_rings; i++) {
9826                 struct bnxt_napi *bnapi = bp->bnapi[i];
9827                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9828                 u64 *sw = cpr->stats.sw_stats;
9829
9830                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
9831                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
9832                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
9833
9834                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
9835                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
9836                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
9837
9838                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
9839                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
9840                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
9841
9842                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
9843                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
9844                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
9845
9846                 stats->rx_missed_errors +=
9847                         BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
9848
9849                 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
9850
9851                 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
9852         }
9853 }
9854
9855 static void bnxt_add_prev_stats(struct bnxt *bp,
9856                                 struct rtnl_link_stats64 *stats)
9857 {
9858         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
9859
9860         stats->rx_packets += prev_stats->rx_packets;
9861         stats->tx_packets += prev_stats->tx_packets;
9862         stats->rx_bytes += prev_stats->rx_bytes;
9863         stats->tx_bytes += prev_stats->tx_bytes;
9864         stats->rx_missed_errors += prev_stats->rx_missed_errors;
9865         stats->multicast += prev_stats->multicast;
9866         stats->tx_dropped += prev_stats->tx_dropped;
9867 }
9868
9869 static void
9870 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
9871 {
9872         struct bnxt *bp = netdev_priv(dev);
9873
9874         set_bit(BNXT_STATE_READ_STATS, &bp->state);
9875         /* Make sure bnxt_close_nic() sees that we are reading stats before
9876          * we check the BNXT_STATE_OPEN flag.
9877          */
9878         smp_mb__after_atomic();
9879         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9880                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9881                 *stats = bp->net_stats_prev;
9882                 return;
9883         }
9884
9885         bnxt_get_ring_stats(bp, stats);
9886         bnxt_add_prev_stats(bp, stats);
9887
9888         if (bp->flags & BNXT_FLAG_PORT_STATS) {
9889                 u64 *rx = bp->port_stats.sw_stats;
9890                 u64 *tx = bp->port_stats.sw_stats +
9891                           BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9892
9893                 stats->rx_crc_errors =
9894                         BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
9895                 stats->rx_frame_errors =
9896                         BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
9897                 stats->rx_length_errors =
9898                         BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
9899                         BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
9900                         BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
9901                 stats->rx_errors =
9902                         BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
9903                         BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
9904                 stats->collisions =
9905                         BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
9906                 stats->tx_fifo_errors =
9907                         BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
9908                 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
9909         }
9910         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9911 }
9912
9913 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
9914 {
9915         struct net_device *dev = bp->dev;
9916         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9917         struct netdev_hw_addr *ha;
9918         u8 *haddr;
9919         int mc_count = 0;
9920         bool update = false;
9921         int off = 0;
9922
9923         netdev_for_each_mc_addr(ha, dev) {
9924                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
9925                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9926                         vnic->mc_list_count = 0;
9927                         return false;
9928                 }
9929                 haddr = ha->addr;
9930                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
9931                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
9932                         update = true;
9933                 }
9934                 off += ETH_ALEN;
9935                 mc_count++;
9936         }
9937         if (mc_count)
9938                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
9939
9940         if (mc_count != vnic->mc_list_count) {
9941                 vnic->mc_list_count = mc_count;
9942                 update = true;
9943         }
9944         return update;
9945 }
9946
9947 static bool bnxt_uc_list_updated(struct bnxt *bp)
9948 {
9949         struct net_device *dev = bp->dev;
9950         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9951         struct netdev_hw_addr *ha;
9952         int off = 0;
9953
9954         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
9955                 return true;
9956
9957         netdev_for_each_uc_addr(ha, dev) {
9958                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
9959                         return true;
9960
9961                 off += ETH_ALEN;
9962         }
9963         return false;
9964 }
9965
9966 static void bnxt_set_rx_mode(struct net_device *dev)
9967 {
9968         struct bnxt *bp = netdev_priv(dev);
9969         struct bnxt_vnic_info *vnic;
9970         bool mc_update = false;
9971         bool uc_update;
9972         u32 mask;
9973
9974         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
9975                 return;
9976
9977         vnic = &bp->vnic_info[0];
9978         mask = vnic->rx_mask;
9979         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
9980                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
9981                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
9982                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
9983
9984         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
9985                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9986
9987         uc_update = bnxt_uc_list_updated(bp);
9988
9989         if (dev->flags & IFF_BROADCAST)
9990                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
9991         if (dev->flags & IFF_ALLMULTI) {
9992                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9993                 vnic->mc_list_count = 0;
9994         } else {
9995                 mc_update = bnxt_mc_list_updated(bp, &mask);
9996         }
9997
9998         if (mask != vnic->rx_mask || uc_update || mc_update) {
9999                 vnic->rx_mask = mask;
10000
10001                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
10002                 bnxt_queue_sp_work(bp);
10003         }
10004 }
10005
10006 static int bnxt_cfg_rx_mode(struct bnxt *bp)
10007 {
10008         struct net_device *dev = bp->dev;
10009         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10010         struct netdev_hw_addr *ha;
10011         int i, off = 0, rc;
10012         bool uc_update;
10013
10014         netif_addr_lock_bh(dev);
10015         uc_update = bnxt_uc_list_updated(bp);
10016         netif_addr_unlock_bh(dev);
10017
10018         if (!uc_update)
10019                 goto skip_uc;
10020
10021         mutex_lock(&bp->hwrm_cmd_lock);
10022         for (i = 1; i < vnic->uc_filter_count; i++) {
10023                 struct hwrm_cfa_l2_filter_free_input req = {0};
10024
10025                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
10026                                        -1);
10027
10028                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
10029
10030                 rc = _hwrm_send_message(bp, &req, sizeof(req),
10031                                         HWRM_CMD_TIMEOUT);
10032         }
10033         mutex_unlock(&bp->hwrm_cmd_lock);
10034
10035         vnic->uc_filter_count = 1;
10036
10037         netif_addr_lock_bh(dev);
10038         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
10039                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10040         } else {
10041                 netdev_for_each_uc_addr(ha, dev) {
10042                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
10043                         off += ETH_ALEN;
10044                         vnic->uc_filter_count++;
10045                 }
10046         }
10047         netif_addr_unlock_bh(dev);
10048
10049         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
10050                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
10051                 if (rc) {
10052                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
10053                                    rc);
10054                         vnic->uc_filter_count = i;
10055                         return rc;
10056                 }
10057         }
10058
10059 skip_uc:
10060         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10061         if (rc && vnic->mc_list_count) {
10062                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
10063                             rc);
10064                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10065                 vnic->mc_list_count = 0;
10066                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10067         }
10068         if (rc)
10069                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
10070                            rc);
10071
10072         return rc;
10073 }
10074
10075 static bool bnxt_can_reserve_rings(struct bnxt *bp)
10076 {
10077 #ifdef CONFIG_BNXT_SRIOV
10078         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
10079                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10080
10081                 /* No minimum rings were provisioned by the PF.  Don't
10082                  * reserve rings by default when device is down.
10083                  */
10084                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
10085                         return true;
10086
10087                 if (!netif_running(bp->dev))
10088                         return false;
10089         }
10090 #endif
10091         return true;
10092 }
10093
10094 /* If the chip and firmware supports RFS */
10095 static bool bnxt_rfs_supported(struct bnxt *bp)
10096 {
10097         if (bp->flags & BNXT_FLAG_CHIP_P5) {
10098                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
10099                         return true;
10100                 return false;
10101         }
10102         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
10103                 return true;
10104         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10105                 return true;
10106         return false;
10107 }
10108
10109 /* If runtime conditions support RFS */
10110 static bool bnxt_rfs_capable(struct bnxt *bp)
10111 {
10112 #ifdef CONFIG_RFS_ACCEL
10113         int vnics, max_vnics, max_rss_ctxs;
10114
10115         if (bp->flags & BNXT_FLAG_CHIP_P5)
10116                 return bnxt_rfs_supported(bp);
10117         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
10118                 return false;
10119
10120         vnics = 1 + bp->rx_nr_rings;
10121         max_vnics = bnxt_get_max_func_vnics(bp);
10122         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
10123
10124         /* RSS contexts not a limiting factor */
10125         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10126                 max_rss_ctxs = max_vnics;
10127         if (vnics > max_vnics || vnics > max_rss_ctxs) {
10128                 if (bp->rx_nr_rings > 1)
10129                         netdev_warn(bp->dev,
10130                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
10131                                     min(max_rss_ctxs - 1, max_vnics - 1));
10132                 return false;
10133         }
10134
10135         if (!BNXT_NEW_RM(bp))
10136                 return true;
10137
10138         if (vnics == bp->hw_resc.resv_vnics)
10139                 return true;
10140
10141         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
10142         if (vnics <= bp->hw_resc.resv_vnics)
10143                 return true;
10144
10145         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
10146         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
10147         return false;
10148 #else
10149         return false;
10150 #endif
10151 }
10152
10153 static netdev_features_t bnxt_fix_features(struct net_device *dev,
10154                                            netdev_features_t features)
10155 {
10156         struct bnxt *bp = netdev_priv(dev);
10157         netdev_features_t vlan_features;
10158
10159         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
10160                 features &= ~NETIF_F_NTUPLE;
10161
10162         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10163                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10164
10165         if (!(features & NETIF_F_GRO))
10166                 features &= ~NETIF_F_GRO_HW;
10167
10168         if (features & NETIF_F_GRO_HW)
10169                 features &= ~NETIF_F_LRO;
10170
10171         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
10172          * turned on or off together.
10173          */
10174         vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
10175         if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
10176                 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10177                         features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10178                 else if (vlan_features)
10179                         features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
10180         }
10181 #ifdef CONFIG_BNXT_SRIOV
10182         if (BNXT_VF(bp) && bp->vf.vlan)
10183                 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10184 #endif
10185         return features;
10186 }
10187
10188 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
10189 {
10190         struct bnxt *bp = netdev_priv(dev);
10191         u32 flags = bp->flags;
10192         u32 changes;
10193         int rc = 0;
10194         bool re_init = false;
10195         bool update_tpa = false;
10196
10197         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
10198         if (features & NETIF_F_GRO_HW)
10199                 flags |= BNXT_FLAG_GRO;
10200         else if (features & NETIF_F_LRO)
10201                 flags |= BNXT_FLAG_LRO;
10202
10203         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10204                 flags &= ~BNXT_FLAG_TPA;
10205
10206         if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10207                 flags |= BNXT_FLAG_STRIP_VLAN;
10208
10209         if (features & NETIF_F_NTUPLE)
10210                 flags |= BNXT_FLAG_RFS;
10211
10212         changes = flags ^ bp->flags;
10213         if (changes & BNXT_FLAG_TPA) {
10214                 update_tpa = true;
10215                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
10216                     (flags & BNXT_FLAG_TPA) == 0 ||
10217                     (bp->flags & BNXT_FLAG_CHIP_P5))
10218                         re_init = true;
10219         }
10220
10221         if (changes & ~BNXT_FLAG_TPA)
10222                 re_init = true;
10223
10224         if (flags != bp->flags) {
10225                 u32 old_flags = bp->flags;
10226
10227                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10228                         bp->flags = flags;
10229                         if (update_tpa)
10230                                 bnxt_set_ring_params(bp);
10231                         return rc;
10232                 }
10233
10234                 if (re_init) {
10235                         bnxt_close_nic(bp, false, false);
10236                         bp->flags = flags;
10237                         if (update_tpa)
10238                                 bnxt_set_ring_params(bp);
10239
10240                         return bnxt_open_nic(bp, false, false);
10241                 }
10242                 if (update_tpa) {
10243                         bp->flags = flags;
10244                         rc = bnxt_set_tpa(bp,
10245                                           (flags & BNXT_FLAG_TPA) ?
10246                                           true : false);
10247                         if (rc)
10248                                 bp->flags = old_flags;
10249                 }
10250         }
10251         return rc;
10252 }
10253
10254 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
10255                          u32 *reg_buf)
10256 {
10257         struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr;
10258         struct hwrm_dbg_read_direct_input req = {0};
10259         __le32 *dbg_reg_buf;
10260         dma_addr_t mapping;
10261         int rc, i;
10262
10263         dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4,
10264                                          &mapping, GFP_KERNEL);
10265         if (!dbg_reg_buf)
10266                 return -ENOMEM;
10267         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1);
10268         req.host_dest_addr = cpu_to_le64(mapping);
10269         req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
10270         req.read_len32 = cpu_to_le32(num_words);
10271         mutex_lock(&bp->hwrm_cmd_lock);
10272         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10273         if (rc || resp->error_code) {
10274                 rc = -EIO;
10275                 goto dbg_rd_reg_exit;
10276         }
10277         for (i = 0; i < num_words; i++)
10278                 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
10279
10280 dbg_rd_reg_exit:
10281         mutex_unlock(&bp->hwrm_cmd_lock);
10282         dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping);
10283         return rc;
10284 }
10285
10286 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
10287                                        u32 ring_id, u32 *prod, u32 *cons)
10288 {
10289         struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
10290         struct hwrm_dbg_ring_info_get_input req = {0};
10291         int rc;
10292
10293         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
10294         req.ring_type = ring_type;
10295         req.fw_ring_id = cpu_to_le32(ring_id);
10296         mutex_lock(&bp->hwrm_cmd_lock);
10297         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10298         if (!rc) {
10299                 *prod = le32_to_cpu(resp->producer_index);
10300                 *cons = le32_to_cpu(resp->consumer_index);
10301         }
10302         mutex_unlock(&bp->hwrm_cmd_lock);
10303         return rc;
10304 }
10305
10306 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
10307 {
10308         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
10309         int i = bnapi->index;
10310
10311         if (!txr)
10312                 return;
10313
10314         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
10315                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
10316                     txr->tx_cons);
10317 }
10318
10319 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
10320 {
10321         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
10322         int i = bnapi->index;
10323
10324         if (!rxr)
10325                 return;
10326
10327         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
10328                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
10329                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
10330                     rxr->rx_sw_agg_prod);
10331 }
10332
10333 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
10334 {
10335         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10336         int i = bnapi->index;
10337
10338         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
10339                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
10340 }
10341
10342 static void bnxt_dbg_dump_states(struct bnxt *bp)
10343 {
10344         int i;
10345         struct bnxt_napi *bnapi;
10346
10347         for (i = 0; i < bp->cp_nr_rings; i++) {
10348                 bnapi = bp->bnapi[i];
10349                 if (netif_msg_drv(bp)) {
10350                         bnxt_dump_tx_sw_state(bnapi);
10351                         bnxt_dump_rx_sw_state(bnapi);
10352                         bnxt_dump_cp_sw_state(bnapi);
10353                 }
10354         }
10355 }
10356
10357 static void bnxt_reset_task(struct bnxt *bp, bool silent)
10358 {
10359         if (!silent)
10360                 bnxt_dbg_dump_states(bp);
10361         if (netif_running(bp->dev)) {
10362                 int rc;
10363
10364                 if (silent) {
10365                         bnxt_close_nic(bp, false, false);
10366                         bnxt_open_nic(bp, false, false);
10367                 } else {
10368                         bnxt_ulp_stop(bp);
10369                         bnxt_close_nic(bp, true, false);
10370                         rc = bnxt_open_nic(bp, true, false);
10371                         bnxt_ulp_start(bp, rc);
10372                 }
10373         }
10374 }
10375
10376 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
10377 {
10378         struct bnxt *bp = netdev_priv(dev);
10379
10380         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
10381         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
10382         bnxt_queue_sp_work(bp);
10383 }
10384
10385 static void bnxt_fw_health_check(struct bnxt *bp)
10386 {
10387         struct bnxt_fw_health *fw_health = bp->fw_health;
10388         u32 val;
10389
10390         if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10391                 return;
10392
10393         if (fw_health->tmr_counter) {
10394                 fw_health->tmr_counter--;
10395                 return;
10396         }
10397
10398         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10399         if (val == fw_health->last_fw_heartbeat)
10400                 goto fw_reset;
10401
10402         fw_health->last_fw_heartbeat = val;
10403
10404         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10405         if (val != fw_health->last_fw_reset_cnt)
10406                 goto fw_reset;
10407
10408         fw_health->tmr_counter = fw_health->tmr_multiplier;
10409         return;
10410
10411 fw_reset:
10412         set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
10413         bnxt_queue_sp_work(bp);
10414 }
10415
10416 static void bnxt_timer(struct timer_list *t)
10417 {
10418         struct bnxt *bp = from_timer(bp, t, timer);
10419         struct net_device *dev = bp->dev;
10420
10421         if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
10422                 return;
10423
10424         if (atomic_read(&bp->intr_sem) != 0)
10425                 goto bnxt_restart_timer;
10426
10427         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
10428                 bnxt_fw_health_check(bp);
10429
10430         if (bp->link_info.link_up && bp->stats_coal_ticks) {
10431                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
10432                 bnxt_queue_sp_work(bp);
10433         }
10434
10435         if (bnxt_tc_flower_enabled(bp)) {
10436                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
10437                 bnxt_queue_sp_work(bp);
10438         }
10439
10440 #ifdef CONFIG_RFS_ACCEL
10441         if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
10442                 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
10443                 bnxt_queue_sp_work(bp);
10444         }
10445 #endif /*CONFIG_RFS_ACCEL*/
10446
10447         if (bp->link_info.phy_retry) {
10448                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
10449                         bp->link_info.phy_retry = false;
10450                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
10451                 } else {
10452                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
10453                         bnxt_queue_sp_work(bp);
10454                 }
10455         }
10456
10457         if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
10458             netif_carrier_ok(dev)) {
10459                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
10460                 bnxt_queue_sp_work(bp);
10461         }
10462 bnxt_restart_timer:
10463         mod_timer(&bp->timer, jiffies + bp->current_interval);
10464 }
10465
10466 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
10467 {
10468         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
10469          * set.  If the device is being closed, bnxt_close() may be holding
10470          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
10471          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
10472          */
10473         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10474         rtnl_lock();
10475 }
10476
10477 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
10478 {
10479         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10480         rtnl_unlock();
10481 }
10482
10483 /* Only called from bnxt_sp_task() */
10484 static void bnxt_reset(struct bnxt *bp, bool silent)
10485 {
10486         bnxt_rtnl_lock_sp(bp);
10487         if (test_bit(BNXT_STATE_OPEN, &bp->state))
10488                 bnxt_reset_task(bp, silent);
10489         bnxt_rtnl_unlock_sp(bp);
10490 }
10491
10492 static void bnxt_fw_reset_close(struct bnxt *bp)
10493 {
10494         bnxt_ulp_stop(bp);
10495         /* When firmware is fatal state, disable PCI device to prevent
10496          * any potential bad DMAs before freeing kernel memory.
10497          */
10498         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10499                 pci_disable_device(bp->pdev);
10500         __bnxt_close_nic(bp, true, false);
10501         bnxt_clear_int_mode(bp);
10502         bnxt_hwrm_func_drv_unrgtr(bp);
10503         if (pci_is_enabled(bp->pdev))
10504                 pci_disable_device(bp->pdev);
10505         bnxt_free_ctx_mem(bp);
10506         kfree(bp->ctx);
10507         bp->ctx = NULL;
10508 }
10509
10510 static bool is_bnxt_fw_ok(struct bnxt *bp)
10511 {
10512         struct bnxt_fw_health *fw_health = bp->fw_health;
10513         bool no_heartbeat = false, has_reset = false;
10514         u32 val;
10515
10516         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10517         if (val == fw_health->last_fw_heartbeat)
10518                 no_heartbeat = true;
10519
10520         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10521         if (val != fw_health->last_fw_reset_cnt)
10522                 has_reset = true;
10523
10524         if (!no_heartbeat && has_reset)
10525                 return true;
10526
10527         return false;
10528 }
10529
10530 /* rtnl_lock is acquired before calling this function */
10531 static void bnxt_force_fw_reset(struct bnxt *bp)
10532 {
10533         struct bnxt_fw_health *fw_health = bp->fw_health;
10534         u32 wait_dsecs;
10535
10536         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
10537             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10538                 return;
10539
10540         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10541         bnxt_fw_reset_close(bp);
10542         wait_dsecs = fw_health->master_func_wait_dsecs;
10543         if (fw_health->master) {
10544                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
10545                         wait_dsecs = 0;
10546                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10547         } else {
10548                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
10549                 wait_dsecs = fw_health->normal_func_wait_dsecs;
10550                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10551         }
10552
10553         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
10554         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
10555         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10556 }
10557
10558 void bnxt_fw_exception(struct bnxt *bp)
10559 {
10560         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
10561         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10562         bnxt_rtnl_lock_sp(bp);
10563         bnxt_force_fw_reset(bp);
10564         bnxt_rtnl_unlock_sp(bp);
10565 }
10566
10567 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
10568  * < 0 on error.
10569  */
10570 static int bnxt_get_registered_vfs(struct bnxt *bp)
10571 {
10572 #ifdef CONFIG_BNXT_SRIOV
10573         int rc;
10574
10575         if (!BNXT_PF(bp))
10576                 return 0;
10577
10578         rc = bnxt_hwrm_func_qcfg(bp);
10579         if (rc) {
10580                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
10581                 return rc;
10582         }
10583         if (bp->pf.registered_vfs)
10584                 return bp->pf.registered_vfs;
10585         if (bp->sriov_cfg)
10586                 return 1;
10587 #endif
10588         return 0;
10589 }
10590
10591 void bnxt_fw_reset(struct bnxt *bp)
10592 {
10593         bnxt_rtnl_lock_sp(bp);
10594         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
10595             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10596                 int n = 0, tmo;
10597
10598                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10599                 if (bp->pf.active_vfs &&
10600                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10601                         n = bnxt_get_registered_vfs(bp);
10602                 if (n < 0) {
10603                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
10604                                    n);
10605                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10606                         dev_close(bp->dev);
10607                         goto fw_reset_exit;
10608                 } else if (n > 0) {
10609                         u16 vf_tmo_dsecs = n * 10;
10610
10611                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
10612                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
10613                         bp->fw_reset_state =
10614                                 BNXT_FW_RESET_STATE_POLL_VF;
10615                         bnxt_queue_fw_reset_work(bp, HZ / 10);
10616                         goto fw_reset_exit;
10617                 }
10618                 bnxt_fw_reset_close(bp);
10619                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10620                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10621                         tmo = HZ / 10;
10622                 } else {
10623                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10624                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
10625                 }
10626                 bnxt_queue_fw_reset_work(bp, tmo);
10627         }
10628 fw_reset_exit:
10629         bnxt_rtnl_unlock_sp(bp);
10630 }
10631
10632 static void bnxt_chk_missed_irq(struct bnxt *bp)
10633 {
10634         int i;
10635
10636         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10637                 return;
10638
10639         for (i = 0; i < bp->cp_nr_rings; i++) {
10640                 struct bnxt_napi *bnapi = bp->bnapi[i];
10641                 struct bnxt_cp_ring_info *cpr;
10642                 u32 fw_ring_id;
10643                 int j;
10644
10645                 if (!bnapi)
10646                         continue;
10647
10648                 cpr = &bnapi->cp_ring;
10649                 for (j = 0; j < 2; j++) {
10650                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
10651                         u32 val[2];
10652
10653                         if (!cpr2 || cpr2->has_more_work ||
10654                             !bnxt_has_work(bp, cpr2))
10655                                 continue;
10656
10657                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
10658                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
10659                                 continue;
10660                         }
10661                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
10662                         bnxt_dbg_hwrm_ring_info_get(bp,
10663                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
10664                                 fw_ring_id, &val[0], &val[1]);
10665                         cpr->sw_stats.cmn.missed_irqs++;
10666                 }
10667         }
10668 }
10669
10670 static void bnxt_cfg_ntp_filters(struct bnxt *);
10671
10672 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
10673 {
10674         struct bnxt_link_info *link_info = &bp->link_info;
10675
10676         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
10677                 link_info->autoneg = BNXT_AUTONEG_SPEED;
10678                 if (bp->hwrm_spec_code >= 0x10201) {
10679                         if (link_info->auto_pause_setting &
10680                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10681                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10682                 } else {
10683                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10684                 }
10685                 link_info->advertising = link_info->auto_link_speeds;
10686         } else {
10687                 link_info->req_link_speed = link_info->force_link_speed;
10688                 link_info->req_duplex = link_info->duplex_setting;
10689         }
10690         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10691                 link_info->req_flow_ctrl =
10692                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10693         else
10694                 link_info->req_flow_ctrl = link_info->force_pause_setting;
10695 }
10696
10697 static void bnxt_sp_task(struct work_struct *work)
10698 {
10699         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
10700
10701         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10702         smp_mb__after_atomic();
10703         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10704                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10705                 return;
10706         }
10707
10708         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
10709                 bnxt_cfg_rx_mode(bp);
10710
10711         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
10712                 bnxt_cfg_ntp_filters(bp);
10713         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
10714                 bnxt_hwrm_exec_fwd_req(bp);
10715         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
10716                 bnxt_hwrm_port_qstats(bp, 0);
10717                 bnxt_hwrm_port_qstats_ext(bp, 0);
10718                 bnxt_accumulate_all_stats(bp);
10719         }
10720
10721         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
10722                 int rc;
10723
10724                 mutex_lock(&bp->link_lock);
10725                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
10726                                        &bp->sp_event))
10727                         bnxt_hwrm_phy_qcaps(bp);
10728
10729                 rc = bnxt_update_link(bp, true);
10730                 if (rc)
10731                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
10732                                    rc);
10733
10734                 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
10735                                        &bp->sp_event))
10736                         bnxt_init_ethtool_link_settings(bp);
10737                 mutex_unlock(&bp->link_lock);
10738         }
10739         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
10740                 int rc;
10741
10742                 mutex_lock(&bp->link_lock);
10743                 rc = bnxt_update_phy_setting(bp);
10744                 mutex_unlock(&bp->link_lock);
10745                 if (rc) {
10746                         netdev_warn(bp->dev, "update phy settings retry failed\n");
10747                 } else {
10748                         bp->link_info.phy_retry = false;
10749                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
10750                 }
10751         }
10752         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
10753                 mutex_lock(&bp->link_lock);
10754                 bnxt_get_port_module_status(bp);
10755                 mutex_unlock(&bp->link_lock);
10756         }
10757
10758         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
10759                 bnxt_tc_flow_stats_work(bp);
10760
10761         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
10762                 bnxt_chk_missed_irq(bp);
10763
10764         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
10765          * must be the last functions to be called before exiting.
10766          */
10767         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
10768                 bnxt_reset(bp, false);
10769
10770         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
10771                 bnxt_reset(bp, true);
10772
10773         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
10774                 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
10775
10776         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
10777                 if (!is_bnxt_fw_ok(bp))
10778                         bnxt_devlink_health_report(bp,
10779                                                    BNXT_FW_EXCEPTION_SP_EVENT);
10780         }
10781
10782         smp_mb__before_atomic();
10783         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10784 }
10785
10786 /* Under rtnl_lock */
10787 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
10788                      int tx_xdp)
10789 {
10790         int max_rx, max_tx, tx_sets = 1;
10791         int tx_rings_needed, stats;
10792         int rx_rings = rx;
10793         int cp, vnics, rc;
10794
10795         if (tcs)
10796                 tx_sets = tcs;
10797
10798         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
10799         if (rc)
10800                 return rc;
10801
10802         if (max_rx < rx)
10803                 return -ENOMEM;
10804
10805         tx_rings_needed = tx * tx_sets + tx_xdp;
10806         if (max_tx < tx_rings_needed)
10807                 return -ENOMEM;
10808
10809         vnics = 1;
10810         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
10811                 vnics += rx_rings;
10812
10813         if (bp->flags & BNXT_FLAG_AGG_RINGS)
10814                 rx_rings <<= 1;
10815         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
10816         stats = cp;
10817         if (BNXT_NEW_RM(bp)) {
10818                 cp += bnxt_get_ulp_msix_num(bp);
10819                 stats += bnxt_get_ulp_stat_ctxs(bp);
10820         }
10821         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
10822                                      stats, vnics);
10823 }
10824
10825 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
10826 {
10827         if (bp->bar2) {
10828                 pci_iounmap(pdev, bp->bar2);
10829                 bp->bar2 = NULL;
10830         }
10831
10832         if (bp->bar1) {
10833                 pci_iounmap(pdev, bp->bar1);
10834                 bp->bar1 = NULL;
10835         }
10836
10837         if (bp->bar0) {
10838                 pci_iounmap(pdev, bp->bar0);
10839                 bp->bar0 = NULL;
10840         }
10841 }
10842
10843 static void bnxt_cleanup_pci(struct bnxt *bp)
10844 {
10845         bnxt_unmap_bars(bp, bp->pdev);
10846         pci_release_regions(bp->pdev);
10847         if (pci_is_enabled(bp->pdev))
10848                 pci_disable_device(bp->pdev);
10849 }
10850
10851 static void bnxt_init_dflt_coal(struct bnxt *bp)
10852 {
10853         struct bnxt_coal *coal;
10854
10855         /* Tick values in micro seconds.
10856          * 1 coal_buf x bufs_per_record = 1 completion record.
10857          */
10858         coal = &bp->rx_coal;
10859         coal->coal_ticks = 10;
10860         coal->coal_bufs = 30;
10861         coal->coal_ticks_irq = 1;
10862         coal->coal_bufs_irq = 2;
10863         coal->idle_thresh = 50;
10864         coal->bufs_per_record = 2;
10865         coal->budget = 64;              /* NAPI budget */
10866
10867         coal = &bp->tx_coal;
10868         coal->coal_ticks = 28;
10869         coal->coal_bufs = 30;
10870         coal->coal_ticks_irq = 2;
10871         coal->coal_bufs_irq = 2;
10872         coal->bufs_per_record = 1;
10873
10874         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
10875 }
10876
10877 static void bnxt_alloc_fw_health(struct bnxt *bp)
10878 {
10879         if (bp->fw_health)
10880                 return;
10881
10882         if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
10883             !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
10884                 return;
10885
10886         bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
10887         if (!bp->fw_health) {
10888                 netdev_warn(bp->dev, "Failed to allocate fw_health\n");
10889                 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
10890                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
10891         }
10892 }
10893
10894 static int bnxt_fw_init_one_p1(struct bnxt *bp)
10895 {
10896         int rc;
10897
10898         bp->fw_cap = 0;
10899         rc = bnxt_hwrm_ver_get(bp);
10900         if (rc)
10901                 return rc;
10902
10903         if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
10904                 rc = bnxt_alloc_kong_hwrm_resources(bp);
10905                 if (rc)
10906                         bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
10907         }
10908
10909         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
10910             bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
10911                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
10912                 if (rc)
10913                         return rc;
10914         }
10915         rc = bnxt_hwrm_func_reset(bp);
10916         if (rc)
10917                 return -ENODEV;
10918
10919         bnxt_hwrm_fw_set_time(bp);
10920         return 0;
10921 }
10922
10923 static int bnxt_fw_init_one_p2(struct bnxt *bp)
10924 {
10925         int rc;
10926
10927         /* Get the MAX capabilities for this function */
10928         rc = bnxt_hwrm_func_qcaps(bp);
10929         if (rc) {
10930                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
10931                            rc);
10932                 return -ENODEV;
10933         }
10934
10935         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
10936         if (rc)
10937                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
10938                             rc);
10939
10940         bnxt_alloc_fw_health(bp);
10941         rc = bnxt_hwrm_error_recovery_qcfg(bp);
10942         if (rc)
10943                 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
10944                             rc);
10945
10946         rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
10947         if (rc)
10948                 return -ENODEV;
10949
10950         bnxt_hwrm_func_qcfg(bp);
10951         bnxt_hwrm_vnic_qcaps(bp);
10952         bnxt_hwrm_port_led_qcaps(bp);
10953         bnxt_ethtool_init(bp);
10954         bnxt_dcb_init(bp);
10955         return 0;
10956 }
10957
10958 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
10959 {
10960         bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
10961         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
10962                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
10963                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
10964                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
10965         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
10966                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
10967                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
10968                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
10969         }
10970 }
10971
10972 static void bnxt_set_dflt_rfs(struct bnxt *bp)
10973 {
10974         struct net_device *dev = bp->dev;
10975
10976         dev->hw_features &= ~NETIF_F_NTUPLE;
10977         dev->features &= ~NETIF_F_NTUPLE;
10978         bp->flags &= ~BNXT_FLAG_RFS;
10979         if (bnxt_rfs_supported(bp)) {
10980                 dev->hw_features |= NETIF_F_NTUPLE;
10981                 if (bnxt_rfs_capable(bp)) {
10982                         bp->flags |= BNXT_FLAG_RFS;
10983                         dev->features |= NETIF_F_NTUPLE;
10984                 }
10985         }
10986 }
10987
10988 static void bnxt_fw_init_one_p3(struct bnxt *bp)
10989 {
10990         struct pci_dev *pdev = bp->pdev;
10991
10992         bnxt_set_dflt_rss_hash_type(bp);
10993         bnxt_set_dflt_rfs(bp);
10994
10995         bnxt_get_wol_settings(bp);
10996         if (bp->flags & BNXT_FLAG_WOL_CAP)
10997                 device_set_wakeup_enable(&pdev->dev, bp->wol);
10998         else
10999                 device_set_wakeup_capable(&pdev->dev, false);
11000
11001         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
11002         bnxt_hwrm_coal_params_qcaps(bp);
11003 }
11004
11005 static int bnxt_fw_init_one(struct bnxt *bp)
11006 {
11007         int rc;
11008
11009         rc = bnxt_fw_init_one_p1(bp);
11010         if (rc) {
11011                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
11012                 return rc;
11013         }
11014         rc = bnxt_fw_init_one_p2(bp);
11015         if (rc) {
11016                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
11017                 return rc;
11018         }
11019         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
11020         if (rc)
11021                 return rc;
11022
11023         /* In case fw capabilities have changed, destroy the unneeded
11024          * reporters and create newly capable ones.
11025          */
11026         bnxt_dl_fw_reporters_destroy(bp, false);
11027         bnxt_dl_fw_reporters_create(bp);
11028         bnxt_fw_init_one_p3(bp);
11029         return 0;
11030 }
11031
11032 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
11033 {
11034         struct bnxt_fw_health *fw_health = bp->fw_health;
11035         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
11036         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
11037         u32 reg_type, reg_off, delay_msecs;
11038
11039         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
11040         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
11041         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
11042         switch (reg_type) {
11043         case BNXT_FW_HEALTH_REG_TYPE_CFG:
11044                 pci_write_config_dword(bp->pdev, reg_off, val);
11045                 break;
11046         case BNXT_FW_HEALTH_REG_TYPE_GRC:
11047                 writel(reg_off & BNXT_GRC_BASE_MASK,
11048                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
11049                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
11050                 fallthrough;
11051         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
11052                 writel(val, bp->bar0 + reg_off);
11053                 break;
11054         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
11055                 writel(val, bp->bar1 + reg_off);
11056                 break;
11057         }
11058         if (delay_msecs) {
11059                 pci_read_config_dword(bp->pdev, 0, &val);
11060                 msleep(delay_msecs);
11061         }
11062 }
11063
11064 static void bnxt_reset_all(struct bnxt *bp)
11065 {
11066         struct bnxt_fw_health *fw_health = bp->fw_health;
11067         int i, rc;
11068
11069         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11070 #ifdef CONFIG_TEE_BNXT_FW
11071                 rc = tee_bnxt_fw_load();
11072                 if (rc)
11073                         netdev_err(bp->dev, "Unable to reset FW rc=%d\n", rc);
11074                 bp->fw_reset_timestamp = jiffies;
11075 #endif
11076                 return;
11077         }
11078
11079         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
11080                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
11081                         bnxt_fw_reset_writel(bp, i);
11082         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
11083                 struct hwrm_fw_reset_input req = {0};
11084
11085                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
11086                 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
11087                 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
11088                 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
11089                 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
11090                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11091                 if (rc)
11092                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
11093         }
11094         bp->fw_reset_timestamp = jiffies;
11095 }
11096
11097 static void bnxt_fw_reset_task(struct work_struct *work)
11098 {
11099         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
11100         int rc;
11101
11102         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11103                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
11104                 return;
11105         }
11106
11107         switch (bp->fw_reset_state) {
11108         case BNXT_FW_RESET_STATE_POLL_VF: {
11109                 int n = bnxt_get_registered_vfs(bp);
11110                 int tmo;
11111
11112                 if (n < 0) {
11113                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
11114                                    n, jiffies_to_msecs(jiffies -
11115                                    bp->fw_reset_timestamp));
11116                         goto fw_reset_abort;
11117                 } else if (n > 0) {
11118                         if (time_after(jiffies, bp->fw_reset_timestamp +
11119                                        (bp->fw_reset_max_dsecs * HZ / 10))) {
11120                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11121                                 bp->fw_reset_state = 0;
11122                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
11123                                            n);
11124                                 return;
11125                         }
11126                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11127                         return;
11128                 }
11129                 bp->fw_reset_timestamp = jiffies;
11130                 rtnl_lock();
11131                 bnxt_fw_reset_close(bp);
11132                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11133                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11134                         tmo = HZ / 10;
11135                 } else {
11136                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11137                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11138                 }
11139                 rtnl_unlock();
11140                 bnxt_queue_fw_reset_work(bp, tmo);
11141                 return;
11142         }
11143         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
11144                 u32 val;
11145
11146                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11147                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
11148                     !time_after(jiffies, bp->fw_reset_timestamp +
11149                     (bp->fw_reset_max_dsecs * HZ / 10))) {
11150                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11151                         return;
11152                 }
11153
11154                 if (!bp->fw_health->master) {
11155                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
11156
11157                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11158                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11159                         return;
11160                 }
11161                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11162         }
11163                 fallthrough;
11164         case BNXT_FW_RESET_STATE_RESET_FW:
11165                 bnxt_reset_all(bp);
11166                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11167                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
11168                 return;
11169         case BNXT_FW_RESET_STATE_ENABLE_DEV:
11170                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11171                         u32 val;
11172
11173                         val = bnxt_fw_health_readl(bp,
11174                                                    BNXT_FW_RESET_INPROG_REG);
11175                         if (val)
11176                                 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n",
11177                                             val);
11178                 }
11179                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11180                 if (pci_enable_device(bp->pdev)) {
11181                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
11182                         goto fw_reset_abort;
11183                 }
11184                 pci_set_master(bp->pdev);
11185                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
11186                 fallthrough;
11187         case BNXT_FW_RESET_STATE_POLL_FW:
11188                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
11189                 rc = __bnxt_hwrm_ver_get(bp, true);
11190                 if (rc) {
11191                         if (time_after(jiffies, bp->fw_reset_timestamp +
11192                                        (bp->fw_reset_max_dsecs * HZ / 10))) {
11193                                 netdev_err(bp->dev, "Firmware reset aborted\n");
11194                                 goto fw_reset_abort;
11195                         }
11196                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11197                         return;
11198                 }
11199                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
11200                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
11201                 fallthrough;
11202         case BNXT_FW_RESET_STATE_OPENING:
11203                 while (!rtnl_trylock()) {
11204                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11205                         return;
11206                 }
11207                 rc = bnxt_open(bp->dev);
11208                 if (rc) {
11209                         netdev_err(bp->dev, "bnxt_open_nic() failed\n");
11210                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11211                         dev_close(bp->dev);
11212                 }
11213
11214                 bp->fw_reset_state = 0;
11215                 /* Make sure fw_reset_state is 0 before clearing the flag */
11216                 smp_mb__before_atomic();
11217                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11218                 bnxt_ulp_start(bp, rc);
11219                 if (!rc)
11220                         bnxt_reenable_sriov(bp);
11221                 bnxt_dl_health_recovery_done(bp);
11222                 bnxt_dl_health_status_update(bp, true);
11223                 rtnl_unlock();
11224                 break;
11225         }
11226         return;
11227
11228 fw_reset_abort:
11229         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11230         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
11231                 bnxt_dl_health_status_update(bp, false);
11232         bp->fw_reset_state = 0;
11233         rtnl_lock();
11234         dev_close(bp->dev);
11235         rtnl_unlock();
11236 }
11237
11238 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
11239 {
11240         int rc;
11241         struct bnxt *bp = netdev_priv(dev);
11242
11243         SET_NETDEV_DEV(dev, &pdev->dev);
11244
11245         /* enable device (incl. PCI PM wakeup), and bus-mastering */
11246         rc = pci_enable_device(pdev);
11247         if (rc) {
11248                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
11249                 goto init_err;
11250         }
11251
11252         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
11253                 dev_err(&pdev->dev,
11254                         "Cannot find PCI device base address, aborting\n");
11255                 rc = -ENODEV;
11256                 goto init_err_disable;
11257         }
11258
11259         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
11260         if (rc) {
11261                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
11262                 goto init_err_disable;
11263         }
11264
11265         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
11266             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
11267                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
11268                 goto init_err_disable;
11269         }
11270
11271         pci_set_master(pdev);
11272
11273         bp->dev = dev;
11274         bp->pdev = pdev;
11275
11276         /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
11277          * determines the BAR size.
11278          */
11279         bp->bar0 = pci_ioremap_bar(pdev, 0);
11280         if (!bp->bar0) {
11281                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
11282                 rc = -ENOMEM;
11283                 goto init_err_release;
11284         }
11285
11286         bp->bar2 = pci_ioremap_bar(pdev, 4);
11287         if (!bp->bar2) {
11288                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
11289                 rc = -ENOMEM;
11290                 goto init_err_release;
11291         }
11292
11293         pci_enable_pcie_error_reporting(pdev);
11294
11295         INIT_WORK(&bp->sp_task, bnxt_sp_task);
11296         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
11297
11298         spin_lock_init(&bp->ntp_fltr_lock);
11299 #if BITS_PER_LONG == 32
11300         spin_lock_init(&bp->db_lock);
11301 #endif
11302
11303         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
11304         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
11305
11306         bnxt_init_dflt_coal(bp);
11307
11308         timer_setup(&bp->timer, bnxt_timer, 0);
11309         bp->current_interval = BNXT_TIMER_INTERVAL;
11310
11311         bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
11312         bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
11313
11314         clear_bit(BNXT_STATE_OPEN, &bp->state);
11315         return 0;
11316
11317 init_err_release:
11318         bnxt_unmap_bars(bp, pdev);
11319         pci_release_regions(pdev);
11320
11321 init_err_disable:
11322         pci_disable_device(pdev);
11323
11324 init_err:
11325         return rc;
11326 }
11327
11328 /* rtnl_lock held */
11329 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
11330 {
11331         struct sockaddr *addr = p;
11332         struct bnxt *bp = netdev_priv(dev);
11333         int rc = 0;
11334
11335         if (!is_valid_ether_addr(addr->sa_data))
11336                 return -EADDRNOTAVAIL;
11337
11338         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
11339                 return 0;
11340
11341         rc = bnxt_approve_mac(bp, addr->sa_data, true);
11342         if (rc)
11343                 return rc;
11344
11345         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
11346         if (netif_running(dev)) {
11347                 bnxt_close_nic(bp, false, false);
11348                 rc = bnxt_open_nic(bp, false, false);
11349         }
11350
11351         return rc;
11352 }
11353
11354 /* rtnl_lock held */
11355 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
11356 {
11357         struct bnxt *bp = netdev_priv(dev);
11358
11359         if (netif_running(dev))
11360                 bnxt_close_nic(bp, true, false);
11361
11362         dev->mtu = new_mtu;
11363         bnxt_set_ring_params(bp);
11364
11365         if (netif_running(dev))
11366                 return bnxt_open_nic(bp, true, false);
11367
11368         return 0;
11369 }
11370
11371 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
11372 {
11373         struct bnxt *bp = netdev_priv(dev);
11374         bool sh = false;
11375         int rc;
11376
11377         if (tc > bp->max_tc) {
11378                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
11379                            tc, bp->max_tc);
11380                 return -EINVAL;
11381         }
11382
11383         if (netdev_get_num_tc(dev) == tc)
11384                 return 0;
11385
11386         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
11387                 sh = true;
11388
11389         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
11390                               sh, tc, bp->tx_nr_rings_xdp);
11391         if (rc)
11392                 return rc;
11393
11394         /* Needs to close the device and do hw resource re-allocations */
11395         if (netif_running(bp->dev))
11396                 bnxt_close_nic(bp, true, false);
11397
11398         if (tc) {
11399                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
11400                 netdev_set_num_tc(dev, tc);
11401         } else {
11402                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11403                 netdev_reset_tc(dev);
11404         }
11405         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
11406         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
11407                                bp->tx_nr_rings + bp->rx_nr_rings;
11408
11409         if (netif_running(bp->dev))
11410                 return bnxt_open_nic(bp, true, false);
11411
11412         return 0;
11413 }
11414
11415 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
11416                                   void *cb_priv)
11417 {
11418         struct bnxt *bp = cb_priv;
11419
11420         if (!bnxt_tc_flower_enabled(bp) ||
11421             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
11422                 return -EOPNOTSUPP;
11423
11424         switch (type) {
11425         case TC_SETUP_CLSFLOWER:
11426                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
11427         default:
11428                 return -EOPNOTSUPP;
11429         }
11430 }
11431
11432 LIST_HEAD(bnxt_block_cb_list);
11433
11434 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
11435                          void *type_data)
11436 {
11437         struct bnxt *bp = netdev_priv(dev);
11438
11439         switch (type) {
11440         case TC_SETUP_BLOCK:
11441                 return flow_block_cb_setup_simple(type_data,
11442                                                   &bnxt_block_cb_list,
11443                                                   bnxt_setup_tc_block_cb,
11444                                                   bp, bp, true);
11445         case TC_SETUP_QDISC_MQPRIO: {
11446                 struct tc_mqprio_qopt *mqprio = type_data;
11447
11448                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
11449
11450                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
11451         }
11452         default:
11453                 return -EOPNOTSUPP;
11454         }
11455 }
11456
11457 #ifdef CONFIG_RFS_ACCEL
11458 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
11459                             struct bnxt_ntuple_filter *f2)
11460 {
11461         struct flow_keys *keys1 = &f1->fkeys;
11462         struct flow_keys *keys2 = &f2->fkeys;
11463
11464         if (keys1->basic.n_proto != keys2->basic.n_proto ||
11465             keys1->basic.ip_proto != keys2->basic.ip_proto)
11466                 return false;
11467
11468         if (keys1->basic.n_proto == htons(ETH_P_IP)) {
11469                 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
11470                     keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
11471                         return false;
11472         } else {
11473                 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
11474                            sizeof(keys1->addrs.v6addrs.src)) ||
11475                     memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
11476                            sizeof(keys1->addrs.v6addrs.dst)))
11477                         return false;
11478         }
11479
11480         if (keys1->ports.ports == keys2->ports.ports &&
11481             keys1->control.flags == keys2->control.flags &&
11482             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
11483             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
11484                 return true;
11485
11486         return false;
11487 }
11488
11489 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
11490                               u16 rxq_index, u32 flow_id)
11491 {
11492         struct bnxt *bp = netdev_priv(dev);
11493         struct bnxt_ntuple_filter *fltr, *new_fltr;
11494         struct flow_keys *fkeys;
11495         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
11496         int rc = 0, idx, bit_id, l2_idx = 0;
11497         struct hlist_head *head;
11498         u32 flags;
11499
11500         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
11501                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11502                 int off = 0, j;
11503
11504                 netif_addr_lock_bh(dev);
11505                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
11506                         if (ether_addr_equal(eth->h_dest,
11507                                              vnic->uc_list + off)) {
11508                                 l2_idx = j + 1;
11509                                 break;
11510                         }
11511                 }
11512                 netif_addr_unlock_bh(dev);
11513                 if (!l2_idx)
11514                         return -EINVAL;
11515         }
11516         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
11517         if (!new_fltr)
11518                 return -ENOMEM;
11519
11520         fkeys = &new_fltr->fkeys;
11521         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
11522                 rc = -EPROTONOSUPPORT;
11523                 goto err_free;
11524         }
11525
11526         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
11527              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
11528             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
11529              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
11530                 rc = -EPROTONOSUPPORT;
11531                 goto err_free;
11532         }
11533         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
11534             bp->hwrm_spec_code < 0x10601) {
11535                 rc = -EPROTONOSUPPORT;
11536                 goto err_free;
11537         }
11538         flags = fkeys->control.flags;
11539         if (((flags & FLOW_DIS_ENCAPSULATION) &&
11540              bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
11541                 rc = -EPROTONOSUPPORT;
11542                 goto err_free;
11543         }
11544
11545         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
11546         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
11547
11548         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
11549         head = &bp->ntp_fltr_hash_tbl[idx];
11550         rcu_read_lock();
11551         hlist_for_each_entry_rcu(fltr, head, hash) {
11552                 if (bnxt_fltr_match(fltr, new_fltr)) {
11553                         rcu_read_unlock();
11554                         rc = 0;
11555                         goto err_free;
11556                 }
11557         }
11558         rcu_read_unlock();
11559
11560         spin_lock_bh(&bp->ntp_fltr_lock);
11561         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
11562                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
11563         if (bit_id < 0) {
11564                 spin_unlock_bh(&bp->ntp_fltr_lock);
11565                 rc = -ENOMEM;
11566                 goto err_free;
11567         }
11568
11569         new_fltr->sw_id = (u16)bit_id;
11570         new_fltr->flow_id = flow_id;
11571         new_fltr->l2_fltr_idx = l2_idx;
11572         new_fltr->rxq = rxq_index;
11573         hlist_add_head_rcu(&new_fltr->hash, head);
11574         bp->ntp_fltr_count++;
11575         spin_unlock_bh(&bp->ntp_fltr_lock);
11576
11577         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11578         bnxt_queue_sp_work(bp);
11579
11580         return new_fltr->sw_id;
11581
11582 err_free:
11583         kfree(new_fltr);
11584         return rc;
11585 }
11586
11587 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11588 {
11589         int i;
11590
11591         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
11592                 struct hlist_head *head;
11593                 struct hlist_node *tmp;
11594                 struct bnxt_ntuple_filter *fltr;
11595                 int rc;
11596
11597                 head = &bp->ntp_fltr_hash_tbl[i];
11598                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
11599                         bool del = false;
11600
11601                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
11602                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
11603                                                         fltr->flow_id,
11604                                                         fltr->sw_id)) {
11605                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
11606                                                                          fltr);
11607                                         del = true;
11608                                 }
11609                         } else {
11610                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
11611                                                                        fltr);
11612                                 if (rc)
11613                                         del = true;
11614                                 else
11615                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
11616                         }
11617
11618                         if (del) {
11619                                 spin_lock_bh(&bp->ntp_fltr_lock);
11620                                 hlist_del_rcu(&fltr->hash);
11621                                 bp->ntp_fltr_count--;
11622                                 spin_unlock_bh(&bp->ntp_fltr_lock);
11623                                 synchronize_rcu();
11624                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
11625                                 kfree(fltr);
11626                         }
11627                 }
11628         }
11629         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
11630                 netdev_info(bp->dev, "Receive PF driver unload event!\n");
11631 }
11632
11633 #else
11634
11635 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11636 {
11637 }
11638
11639 #endif /* CONFIG_RFS_ACCEL */
11640
11641 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
11642 {
11643         struct bnxt *bp = netdev_priv(netdev);
11644         struct udp_tunnel_info ti;
11645         unsigned int cmd;
11646
11647         udp_tunnel_nic_get_port(netdev, table, 0, &ti);
11648         if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
11649                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
11650         else
11651                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
11652
11653         if (ti.port)
11654                 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
11655
11656         return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
11657 }
11658
11659 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
11660         .sync_table     = bnxt_udp_tunnel_sync,
11661         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
11662                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
11663         .tables         = {
11664                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
11665                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
11666         },
11667 };
11668
11669 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11670                                struct net_device *dev, u32 filter_mask,
11671                                int nlflags)
11672 {
11673         struct bnxt *bp = netdev_priv(dev);
11674
11675         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
11676                                        nlflags, filter_mask, NULL);
11677 }
11678
11679 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
11680                                u16 flags, struct netlink_ext_ack *extack)
11681 {
11682         struct bnxt *bp = netdev_priv(dev);
11683         struct nlattr *attr, *br_spec;
11684         int rem, rc = 0;
11685
11686         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
11687                 return -EOPNOTSUPP;
11688
11689         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
11690         if (!br_spec)
11691                 return -EINVAL;
11692
11693         nla_for_each_nested(attr, br_spec, rem) {
11694                 u16 mode;
11695
11696                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
11697                         continue;
11698
11699                 if (nla_len(attr) < sizeof(mode))
11700                         return -EINVAL;
11701
11702                 mode = nla_get_u16(attr);
11703                 if (mode == bp->br_mode)
11704                         break;
11705
11706                 rc = bnxt_hwrm_set_br_mode(bp, mode);
11707                 if (!rc)
11708                         bp->br_mode = mode;
11709                 break;
11710         }
11711         return rc;
11712 }
11713
11714 int bnxt_get_port_parent_id(struct net_device *dev,
11715                             struct netdev_phys_item_id *ppid)
11716 {
11717         struct bnxt *bp = netdev_priv(dev);
11718
11719         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
11720                 return -EOPNOTSUPP;
11721
11722         /* The PF and it's VF-reps only support the switchdev framework */
11723         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
11724                 return -EOPNOTSUPP;
11725
11726         ppid->id_len = sizeof(bp->dsn);
11727         memcpy(ppid->id, bp->dsn, ppid->id_len);
11728
11729         return 0;
11730 }
11731
11732 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
11733 {
11734         struct bnxt *bp = netdev_priv(dev);
11735
11736         return &bp->dl_port;
11737 }
11738
11739 static const struct net_device_ops bnxt_netdev_ops = {
11740         .ndo_open               = bnxt_open,
11741         .ndo_start_xmit         = bnxt_start_xmit,
11742         .ndo_stop               = bnxt_close,
11743         .ndo_get_stats64        = bnxt_get_stats64,
11744         .ndo_set_rx_mode        = bnxt_set_rx_mode,
11745         .ndo_do_ioctl           = bnxt_ioctl,
11746         .ndo_validate_addr      = eth_validate_addr,
11747         .ndo_set_mac_address    = bnxt_change_mac_addr,
11748         .ndo_change_mtu         = bnxt_change_mtu,
11749         .ndo_fix_features       = bnxt_fix_features,
11750         .ndo_set_features       = bnxt_set_features,
11751         .ndo_tx_timeout         = bnxt_tx_timeout,
11752 #ifdef CONFIG_BNXT_SRIOV
11753         .ndo_get_vf_config      = bnxt_get_vf_config,
11754         .ndo_set_vf_mac         = bnxt_set_vf_mac,
11755         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
11756         .ndo_set_vf_rate        = bnxt_set_vf_bw,
11757         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
11758         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
11759         .ndo_set_vf_trust       = bnxt_set_vf_trust,
11760 #endif
11761         .ndo_setup_tc           = bnxt_setup_tc,
11762 #ifdef CONFIG_RFS_ACCEL
11763         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
11764 #endif
11765         .ndo_udp_tunnel_add     = udp_tunnel_nic_add_port,
11766         .ndo_udp_tunnel_del     = udp_tunnel_nic_del_port,
11767         .ndo_bpf                = bnxt_xdp,
11768         .ndo_xdp_xmit           = bnxt_xdp_xmit,
11769         .ndo_bridge_getlink     = bnxt_bridge_getlink,
11770         .ndo_bridge_setlink     = bnxt_bridge_setlink,
11771         .ndo_get_devlink_port   = bnxt_get_devlink_port,
11772 };
11773
11774 static void bnxt_remove_one(struct pci_dev *pdev)
11775 {
11776         struct net_device *dev = pci_get_drvdata(pdev);
11777         struct bnxt *bp = netdev_priv(dev);
11778
11779         if (BNXT_PF(bp))
11780                 bnxt_sriov_disable(bp);
11781
11782         bnxt_dl_fw_reporters_destroy(bp, true);
11783         if (BNXT_PF(bp))
11784                 devlink_port_type_clear(&bp->dl_port);
11785         pci_disable_pcie_error_reporting(pdev);
11786         unregister_netdev(dev);
11787         bnxt_dl_unregister(bp);
11788         bnxt_shutdown_tc(bp);
11789         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11790         bnxt_cancel_sp_work(bp);
11791         bp->sp_event = 0;
11792
11793         bnxt_clear_int_mode(bp);
11794         bnxt_hwrm_func_drv_unrgtr(bp);
11795         bnxt_free_hwrm_resources(bp);
11796         bnxt_free_hwrm_short_cmd_req(bp);
11797         bnxt_ethtool_free(bp);
11798         bnxt_dcb_free(bp);
11799         kfree(bp->edev);
11800         bp->edev = NULL;
11801         kfree(bp->fw_health);
11802         bp->fw_health = NULL;
11803         bnxt_cleanup_pci(bp);
11804         bnxt_free_ctx_mem(bp);
11805         kfree(bp->ctx);
11806         bp->ctx = NULL;
11807         kfree(bp->rss_indir_tbl);
11808         bp->rss_indir_tbl = NULL;
11809         bnxt_free_port_stats(bp);
11810         free_netdev(dev);
11811 }
11812
11813 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
11814 {
11815         int rc = 0;
11816         struct bnxt_link_info *link_info = &bp->link_info;
11817
11818         rc = bnxt_hwrm_phy_qcaps(bp);
11819         if (rc) {
11820                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
11821                            rc);
11822                 return rc;
11823         }
11824         if (!fw_dflt)
11825                 return 0;
11826
11827         rc = bnxt_update_link(bp, false);
11828         if (rc) {
11829                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
11830                            rc);
11831                 return rc;
11832         }
11833
11834         /* Older firmware does not have supported_auto_speeds, so assume
11835          * that all supported speeds can be autonegotiated.
11836          */
11837         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
11838                 link_info->support_auto_speeds = link_info->support_speeds;
11839
11840         bnxt_init_ethtool_link_settings(bp);
11841         return 0;
11842 }
11843
11844 static int bnxt_get_max_irq(struct pci_dev *pdev)
11845 {
11846         u16 ctrl;
11847
11848         if (!pdev->msix_cap)
11849                 return 1;
11850
11851         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
11852         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
11853 }
11854
11855 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
11856                                 int *max_cp)
11857 {
11858         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11859         int max_ring_grps = 0, max_irq;
11860
11861         *max_tx = hw_resc->max_tx_rings;
11862         *max_rx = hw_resc->max_rx_rings;
11863         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
11864         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
11865                         bnxt_get_ulp_msix_num(bp),
11866                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
11867         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11868                 *max_cp = min_t(int, *max_cp, max_irq);
11869         max_ring_grps = hw_resc->max_hw_ring_grps;
11870         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
11871                 *max_cp -= 1;
11872                 *max_rx -= 2;
11873         }
11874         if (bp->flags & BNXT_FLAG_AGG_RINGS)
11875                 *max_rx >>= 1;
11876         if (bp->flags & BNXT_FLAG_CHIP_P5) {
11877                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
11878                 /* On P5 chips, max_cp output param should be available NQs */
11879                 *max_cp = max_irq;
11880         }
11881         *max_rx = min_t(int, *max_rx, max_ring_grps);
11882 }
11883
11884 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
11885 {
11886         int rx, tx, cp;
11887
11888         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
11889         *max_rx = rx;
11890         *max_tx = tx;
11891         if (!rx || !tx || !cp)
11892                 return -ENOMEM;
11893
11894         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
11895 }
11896
11897 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
11898                                bool shared)
11899 {
11900         int rc;
11901
11902         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
11903         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
11904                 /* Not enough rings, try disabling agg rings. */
11905                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
11906                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
11907                 if (rc) {
11908                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
11909                         bp->flags |= BNXT_FLAG_AGG_RINGS;
11910                         return rc;
11911                 }
11912                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
11913                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11914                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11915                 bnxt_set_ring_params(bp);
11916         }
11917
11918         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
11919                 int max_cp, max_stat, max_irq;
11920
11921                 /* Reserve minimum resources for RoCE */
11922                 max_cp = bnxt_get_max_func_cp_rings(bp);
11923                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
11924                 max_irq = bnxt_get_max_func_irqs(bp);
11925                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
11926                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
11927                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
11928                         return 0;
11929
11930                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
11931                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
11932                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
11933                 max_cp = min_t(int, max_cp, max_irq);
11934                 max_cp = min_t(int, max_cp, max_stat);
11935                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
11936                 if (rc)
11937                         rc = 0;
11938         }
11939         return rc;
11940 }
11941
11942 /* In initial default shared ring setting, each shared ring must have a
11943  * RX/TX ring pair.
11944  */
11945 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
11946 {
11947         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
11948         bp->rx_nr_rings = bp->cp_nr_rings;
11949         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
11950         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11951 }
11952
11953 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
11954 {
11955         int dflt_rings, max_rx_rings, max_tx_rings, rc;
11956
11957         if (!bnxt_can_reserve_rings(bp))
11958                 return 0;
11959
11960         if (sh)
11961                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
11962         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
11963         /* Reduce default rings on multi-port cards so that total default
11964          * rings do not exceed CPU count.
11965          */
11966         if (bp->port_count > 1) {
11967                 int max_rings =
11968                         max_t(int, num_online_cpus() / bp->port_count, 1);
11969
11970                 dflt_rings = min_t(int, dflt_rings, max_rings);
11971         }
11972         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
11973         if (rc)
11974                 return rc;
11975         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
11976         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
11977         if (sh)
11978                 bnxt_trim_dflt_sh_rings(bp);
11979         else
11980                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
11981         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11982
11983         rc = __bnxt_reserve_rings(bp);
11984         if (rc)
11985                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
11986         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11987         if (sh)
11988                 bnxt_trim_dflt_sh_rings(bp);
11989
11990         /* Rings may have been trimmed, re-reserve the trimmed rings. */
11991         if (bnxt_need_reserve_rings(bp)) {
11992                 rc = __bnxt_reserve_rings(bp);
11993                 if (rc)
11994                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
11995                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11996         }
11997         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
11998                 bp->rx_nr_rings++;
11999                 bp->cp_nr_rings++;
12000         }
12001         if (rc) {
12002                 bp->tx_nr_rings = 0;
12003                 bp->rx_nr_rings = 0;
12004         }
12005         return rc;
12006 }
12007
12008 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
12009 {
12010         int rc;
12011
12012         if (bp->tx_nr_rings)
12013                 return 0;
12014
12015         bnxt_ulp_irq_stop(bp);
12016         bnxt_clear_int_mode(bp);
12017         rc = bnxt_set_dflt_rings(bp, true);
12018         if (rc) {
12019                 netdev_err(bp->dev, "Not enough rings available.\n");
12020                 goto init_dflt_ring_err;
12021         }
12022         rc = bnxt_init_int_mode(bp);
12023         if (rc)
12024                 goto init_dflt_ring_err;
12025
12026         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12027         if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
12028                 bp->flags |= BNXT_FLAG_RFS;
12029                 bp->dev->features |= NETIF_F_NTUPLE;
12030         }
12031 init_dflt_ring_err:
12032         bnxt_ulp_irq_restart(bp, rc);
12033         return rc;
12034 }
12035
12036 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
12037 {
12038         int rc;
12039
12040         ASSERT_RTNL();
12041         bnxt_hwrm_func_qcaps(bp);
12042
12043         if (netif_running(bp->dev))
12044                 __bnxt_close_nic(bp, true, false);
12045
12046         bnxt_ulp_irq_stop(bp);
12047         bnxt_clear_int_mode(bp);
12048         rc = bnxt_init_int_mode(bp);
12049         bnxt_ulp_irq_restart(bp, rc);
12050
12051         if (netif_running(bp->dev)) {
12052                 if (rc)
12053                         dev_close(bp->dev);
12054                 else
12055                         rc = bnxt_open_nic(bp, true, false);
12056         }
12057
12058         return rc;
12059 }
12060
12061 static int bnxt_init_mac_addr(struct bnxt *bp)
12062 {
12063         int rc = 0;
12064
12065         if (BNXT_PF(bp)) {
12066                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
12067         } else {
12068 #ifdef CONFIG_BNXT_SRIOV
12069                 struct bnxt_vf_info *vf = &bp->vf;
12070                 bool strict_approval = true;
12071
12072                 if (is_valid_ether_addr(vf->mac_addr)) {
12073                         /* overwrite netdev dev_addr with admin VF MAC */
12074                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
12075                         /* Older PF driver or firmware may not approve this
12076                          * correctly.
12077                          */
12078                         strict_approval = false;
12079                 } else {
12080                         eth_hw_addr_random(bp->dev);
12081                 }
12082                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
12083 #endif
12084         }
12085         return rc;
12086 }
12087
12088 #define BNXT_VPD_LEN    512
12089 static void bnxt_vpd_read_info(struct bnxt *bp)
12090 {
12091         struct pci_dev *pdev = bp->pdev;
12092         int i, len, pos, ro_size;
12093         ssize_t vpd_size;
12094         u8 *vpd_data;
12095
12096         vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL);
12097         if (!vpd_data)
12098                 return;
12099
12100         vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data);
12101         if (vpd_size <= 0) {
12102                 netdev_err(bp->dev, "Unable to read VPD\n");
12103                 goto exit;
12104         }
12105
12106         i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
12107         if (i < 0) {
12108                 netdev_err(bp->dev, "VPD READ-Only not found\n");
12109                 goto exit;
12110         }
12111
12112         ro_size = pci_vpd_lrdt_size(&vpd_data[i]);
12113         i += PCI_VPD_LRDT_TAG_SIZE;
12114         if (i + ro_size > vpd_size)
12115                 goto exit;
12116
12117         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12118                                         PCI_VPD_RO_KEYWORD_PARTNO);
12119         if (pos < 0)
12120                 goto read_sn;
12121
12122         len = pci_vpd_info_field_size(&vpd_data[pos]);
12123         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12124         if (len + pos > vpd_size)
12125                 goto read_sn;
12126
12127         strlcpy(bp->board_partno, &vpd_data[pos], min(len, BNXT_VPD_FLD_LEN));
12128
12129 read_sn:
12130         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12131                                         PCI_VPD_RO_KEYWORD_SERIALNO);
12132         if (pos < 0)
12133                 goto exit;
12134
12135         len = pci_vpd_info_field_size(&vpd_data[pos]);
12136         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12137         if (len + pos > vpd_size)
12138                 goto exit;
12139
12140         strlcpy(bp->board_serialno, &vpd_data[pos], min(len, BNXT_VPD_FLD_LEN));
12141 exit:
12142         kfree(vpd_data);
12143 }
12144
12145 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
12146 {
12147         struct pci_dev *pdev = bp->pdev;
12148         u64 qword;
12149
12150         qword = pci_get_dsn(pdev);
12151         if (!qword) {
12152                 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
12153                 return -EOPNOTSUPP;
12154         }
12155
12156         put_unaligned_le64(qword, dsn);
12157
12158         bp->flags |= BNXT_FLAG_DSN_VALID;
12159         return 0;
12160 }
12161
12162 static int bnxt_map_db_bar(struct bnxt *bp)
12163 {
12164         if (!bp->db_size)
12165                 return -ENODEV;
12166         bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
12167         if (!bp->bar1)
12168                 return -ENOMEM;
12169         return 0;
12170 }
12171
12172 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
12173 {
12174         struct net_device *dev;
12175         struct bnxt *bp;
12176         int rc, max_irqs;
12177
12178         if (pci_is_bridge(pdev))
12179                 return -ENODEV;
12180
12181         /* Clear any pending DMA transactions from crash kernel
12182          * while loading driver in capture kernel.
12183          */
12184         if (is_kdump_kernel()) {
12185                 pci_clear_master(pdev);
12186                 pcie_flr(pdev);
12187         }
12188
12189         max_irqs = bnxt_get_max_irq(pdev);
12190         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
12191         if (!dev)
12192                 return -ENOMEM;
12193
12194         bp = netdev_priv(dev);
12195         bnxt_set_max_func_irqs(bp, max_irqs);
12196
12197         if (bnxt_vf_pciid(ent->driver_data))
12198                 bp->flags |= BNXT_FLAG_VF;
12199
12200         if (pdev->msix_cap)
12201                 bp->flags |= BNXT_FLAG_MSIX_CAP;
12202
12203         rc = bnxt_init_board(pdev, dev);
12204         if (rc < 0)
12205                 goto init_err_free;
12206
12207         dev->netdev_ops = &bnxt_netdev_ops;
12208         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
12209         dev->ethtool_ops = &bnxt_ethtool_ops;
12210         pci_set_drvdata(pdev, dev);
12211
12212         if (BNXT_PF(bp))
12213                 bnxt_vpd_read_info(bp);
12214
12215         rc = bnxt_alloc_hwrm_resources(bp);
12216         if (rc)
12217                 goto init_err_pci_clean;
12218
12219         mutex_init(&bp->hwrm_cmd_lock);
12220         mutex_init(&bp->link_lock);
12221
12222         rc = bnxt_fw_init_one_p1(bp);
12223         if (rc)
12224                 goto init_err_pci_clean;
12225
12226         if (BNXT_CHIP_P5(bp))
12227                 bp->flags |= BNXT_FLAG_CHIP_P5;
12228
12229         rc = bnxt_alloc_rss_indir_tbl(bp);
12230         if (rc)
12231                 goto init_err_pci_clean;
12232
12233         rc = bnxt_fw_init_one_p2(bp);
12234         if (rc)
12235                 goto init_err_pci_clean;
12236
12237         rc = bnxt_map_db_bar(bp);
12238         if (rc) {
12239                 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
12240                         rc);
12241                 goto init_err_pci_clean;
12242         }
12243
12244         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12245                            NETIF_F_TSO | NETIF_F_TSO6 |
12246                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12247                            NETIF_F_GSO_IPXIP4 |
12248                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12249                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
12250                            NETIF_F_RXCSUM | NETIF_F_GRO;
12251
12252         if (BNXT_SUPPORTS_TPA(bp))
12253                 dev->hw_features |= NETIF_F_LRO;
12254
12255         dev->hw_enc_features =
12256                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12257                         NETIF_F_TSO | NETIF_F_TSO6 |
12258                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12259                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12260                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
12261         dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
12262
12263         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
12264                                     NETIF_F_GSO_GRE_CSUM;
12265         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
12266         if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
12267                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12268         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
12269                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
12270         if (BNXT_SUPPORTS_TPA(bp))
12271                 dev->hw_features |= NETIF_F_GRO_HW;
12272         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
12273         if (dev->features & NETIF_F_GRO_HW)
12274                 dev->features &= ~NETIF_F_LRO;
12275         dev->priv_flags |= IFF_UNICAST_FLT;
12276
12277 #ifdef CONFIG_BNXT_SRIOV
12278         init_waitqueue_head(&bp->sriov_cfg_wait);
12279         mutex_init(&bp->sriov_lock);
12280 #endif
12281         if (BNXT_SUPPORTS_TPA(bp)) {
12282                 bp->gro_func = bnxt_gro_func_5730x;
12283                 if (BNXT_CHIP_P4(bp))
12284                         bp->gro_func = bnxt_gro_func_5731x;
12285                 else if (BNXT_CHIP_P5(bp))
12286                         bp->gro_func = bnxt_gro_func_5750x;
12287         }
12288         if (!BNXT_CHIP_P4_PLUS(bp))
12289                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
12290
12291         bp->ulp_probe = bnxt_ulp_probe;
12292
12293         rc = bnxt_init_mac_addr(bp);
12294         if (rc) {
12295                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
12296                 rc = -EADDRNOTAVAIL;
12297                 goto init_err_pci_clean;
12298         }
12299
12300         if (BNXT_PF(bp)) {
12301                 /* Read the adapter's DSN to use as the eswitch switch_id */
12302                 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
12303         }
12304
12305         /* MTU range: 60 - FW defined max */
12306         dev->min_mtu = ETH_ZLEN;
12307         dev->max_mtu = bp->max_mtu;
12308
12309         rc = bnxt_probe_phy(bp, true);
12310         if (rc)
12311                 goto init_err_pci_clean;
12312
12313         bnxt_set_rx_skb_mode(bp, false);
12314         bnxt_set_tpa_flags(bp);
12315         bnxt_set_ring_params(bp);
12316         rc = bnxt_set_dflt_rings(bp, true);
12317         if (rc) {
12318                 netdev_err(bp->dev, "Not enough rings available.\n");
12319                 rc = -ENOMEM;
12320                 goto init_err_pci_clean;
12321         }
12322
12323         bnxt_fw_init_one_p3(bp);
12324
12325         if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12326                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
12327
12328         rc = bnxt_init_int_mode(bp);
12329         if (rc)
12330                 goto init_err_pci_clean;
12331
12332         /* No TC has been set yet and rings may have been trimmed due to
12333          * limited MSIX, so we re-initialize the TX rings per TC.
12334          */
12335         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12336
12337         if (BNXT_PF(bp)) {
12338                 if (!bnxt_pf_wq) {
12339                         bnxt_pf_wq =
12340                                 create_singlethread_workqueue("bnxt_pf_wq");
12341                         if (!bnxt_pf_wq) {
12342                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
12343                                 goto init_err_pci_clean;
12344                         }
12345                 }
12346                 rc = bnxt_init_tc(bp);
12347                 if (rc)
12348                         netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
12349                                    rc);
12350         }
12351
12352         bnxt_dl_register(bp);
12353
12354         rc = register_netdev(dev);
12355         if (rc)
12356                 goto init_err_cleanup;
12357
12358         if (BNXT_PF(bp))
12359                 devlink_port_type_eth_set(&bp->dl_port, bp->dev);
12360         bnxt_dl_fw_reporters_create(bp);
12361
12362         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
12363                     board_info[ent->driver_data].name,
12364                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
12365         pcie_print_link_status(pdev);
12366
12367         pci_save_state(pdev);
12368         return 0;
12369
12370 init_err_cleanup:
12371         bnxt_dl_unregister(bp);
12372         bnxt_shutdown_tc(bp);
12373         bnxt_clear_int_mode(bp);
12374
12375 init_err_pci_clean:
12376         bnxt_hwrm_func_drv_unrgtr(bp);
12377         bnxt_free_hwrm_short_cmd_req(bp);
12378         bnxt_free_hwrm_resources(bp);
12379         kfree(bp->fw_health);
12380         bp->fw_health = NULL;
12381         bnxt_cleanup_pci(bp);
12382         bnxt_free_ctx_mem(bp);
12383         kfree(bp->ctx);
12384         bp->ctx = NULL;
12385         kfree(bp->rss_indir_tbl);
12386         bp->rss_indir_tbl = NULL;
12387
12388 init_err_free:
12389         free_netdev(dev);
12390         return rc;
12391 }
12392
12393 static void bnxt_shutdown(struct pci_dev *pdev)
12394 {
12395         struct net_device *dev = pci_get_drvdata(pdev);
12396         struct bnxt *bp;
12397
12398         if (!dev)
12399                 return;
12400
12401         rtnl_lock();
12402         bp = netdev_priv(dev);
12403         if (!bp)
12404                 goto shutdown_exit;
12405
12406         if (netif_running(dev))
12407                 dev_close(dev);
12408
12409         bnxt_ulp_shutdown(bp);
12410         bnxt_clear_int_mode(bp);
12411         pci_disable_device(pdev);
12412
12413         if (system_state == SYSTEM_POWER_OFF) {
12414                 pci_wake_from_d3(pdev, bp->wol);
12415                 pci_set_power_state(pdev, PCI_D3hot);
12416         }
12417
12418 shutdown_exit:
12419         rtnl_unlock();
12420 }
12421
12422 #ifdef CONFIG_PM_SLEEP
12423 static int bnxt_suspend(struct device *device)
12424 {
12425         struct net_device *dev = dev_get_drvdata(device);
12426         struct bnxt *bp = netdev_priv(dev);
12427         int rc = 0;
12428
12429         rtnl_lock();
12430         bnxt_ulp_stop(bp);
12431         if (netif_running(dev)) {
12432                 netif_device_detach(dev);
12433                 rc = bnxt_close(dev);
12434         }
12435         bnxt_hwrm_func_drv_unrgtr(bp);
12436         pci_disable_device(bp->pdev);
12437         bnxt_free_ctx_mem(bp);
12438         kfree(bp->ctx);
12439         bp->ctx = NULL;
12440         rtnl_unlock();
12441         return rc;
12442 }
12443
12444 static int bnxt_resume(struct device *device)
12445 {
12446         struct net_device *dev = dev_get_drvdata(device);
12447         struct bnxt *bp = netdev_priv(dev);
12448         int rc = 0;
12449
12450         rtnl_lock();
12451         rc = pci_enable_device(bp->pdev);
12452         if (rc) {
12453                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
12454                            rc);
12455                 goto resume_exit;
12456         }
12457         pci_set_master(bp->pdev);
12458         if (bnxt_hwrm_ver_get(bp)) {
12459                 rc = -ENODEV;
12460                 goto resume_exit;
12461         }
12462         rc = bnxt_hwrm_func_reset(bp);
12463         if (rc) {
12464                 rc = -EBUSY;
12465                 goto resume_exit;
12466         }
12467
12468         rc = bnxt_hwrm_func_qcaps(bp);
12469         if (rc)
12470                 goto resume_exit;
12471
12472         if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
12473                 rc = -ENODEV;
12474                 goto resume_exit;
12475         }
12476
12477         bnxt_get_wol_settings(bp);
12478         if (netif_running(dev)) {
12479                 rc = bnxt_open(dev);
12480                 if (!rc)
12481                         netif_device_attach(dev);
12482         }
12483
12484 resume_exit:
12485         bnxt_ulp_start(bp, rc);
12486         if (!rc)
12487                 bnxt_reenable_sriov(bp);
12488         rtnl_unlock();
12489         return rc;
12490 }
12491
12492 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
12493 #define BNXT_PM_OPS (&bnxt_pm_ops)
12494
12495 #else
12496
12497 #define BNXT_PM_OPS NULL
12498
12499 #endif /* CONFIG_PM_SLEEP */
12500
12501 /**
12502  * bnxt_io_error_detected - called when PCI error is detected
12503  * @pdev: Pointer to PCI device
12504  * @state: The current pci connection state
12505  *
12506  * This function is called after a PCI bus error affecting
12507  * this device has been detected.
12508  */
12509 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
12510                                                pci_channel_state_t state)
12511 {
12512         struct net_device *netdev = pci_get_drvdata(pdev);
12513         struct bnxt *bp = netdev_priv(netdev);
12514
12515         netdev_info(netdev, "PCI I/O error detected\n");
12516
12517         rtnl_lock();
12518         netif_device_detach(netdev);
12519
12520         bnxt_ulp_stop(bp);
12521
12522         if (state == pci_channel_io_perm_failure) {
12523                 rtnl_unlock();
12524                 return PCI_ERS_RESULT_DISCONNECT;
12525         }
12526
12527         if (netif_running(netdev))
12528                 bnxt_close(netdev);
12529
12530         pci_disable_device(pdev);
12531         bnxt_free_ctx_mem(bp);
12532         kfree(bp->ctx);
12533         bp->ctx = NULL;
12534         rtnl_unlock();
12535
12536         /* Request a slot slot reset. */
12537         return PCI_ERS_RESULT_NEED_RESET;
12538 }
12539
12540 /**
12541  * bnxt_io_slot_reset - called after the pci bus has been reset.
12542  * @pdev: Pointer to PCI device
12543  *
12544  * Restart the card from scratch, as if from a cold-boot.
12545  * At this point, the card has exprienced a hard reset,
12546  * followed by fixups by BIOS, and has its config space
12547  * set up identically to what it was at cold boot.
12548  */
12549 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
12550 {
12551         struct net_device *netdev = pci_get_drvdata(pdev);
12552         struct bnxt *bp = netdev_priv(netdev);
12553         int err = 0;
12554         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
12555
12556         netdev_info(bp->dev, "PCI Slot Reset\n");
12557
12558         rtnl_lock();
12559
12560         if (pci_enable_device(pdev)) {
12561                 dev_err(&pdev->dev,
12562                         "Cannot re-enable PCI device after reset.\n");
12563         } else {
12564                 pci_set_master(pdev);
12565                 pci_restore_state(pdev);
12566                 pci_save_state(pdev);
12567
12568                 err = bnxt_hwrm_func_reset(bp);
12569                 if (!err) {
12570                         err = bnxt_hwrm_func_qcaps(bp);
12571                         if (!err && netif_running(netdev))
12572                                 err = bnxt_open(netdev);
12573                 }
12574                 bnxt_ulp_start(bp, err);
12575                 if (!err) {
12576                         bnxt_reenable_sriov(bp);
12577                         result = PCI_ERS_RESULT_RECOVERED;
12578                 }
12579         }
12580
12581         if (result != PCI_ERS_RESULT_RECOVERED) {
12582                 if (netif_running(netdev))
12583                         dev_close(netdev);
12584                 pci_disable_device(pdev);
12585         }
12586
12587         rtnl_unlock();
12588
12589         return result;
12590 }
12591
12592 /**
12593  * bnxt_io_resume - called when traffic can start flowing again.
12594  * @pdev: Pointer to PCI device
12595  *
12596  * This callback is called when the error recovery driver tells
12597  * us that its OK to resume normal operation.
12598  */
12599 static void bnxt_io_resume(struct pci_dev *pdev)
12600 {
12601         struct net_device *netdev = pci_get_drvdata(pdev);
12602
12603         rtnl_lock();
12604
12605         netif_device_attach(netdev);
12606
12607         rtnl_unlock();
12608 }
12609
12610 static const struct pci_error_handlers bnxt_err_handler = {
12611         .error_detected = bnxt_io_error_detected,
12612         .slot_reset     = bnxt_io_slot_reset,
12613         .resume         = bnxt_io_resume
12614 };
12615
12616 static struct pci_driver bnxt_pci_driver = {
12617         .name           = DRV_MODULE_NAME,
12618         .id_table       = bnxt_pci_tbl,
12619         .probe          = bnxt_init_one,
12620         .remove         = bnxt_remove_one,
12621         .shutdown       = bnxt_shutdown,
12622         .driver.pm      = BNXT_PM_OPS,
12623         .err_handler    = &bnxt_err_handler,
12624 #if defined(CONFIG_BNXT_SRIOV)
12625         .sriov_configure = bnxt_sriov_configure,
12626 #endif
12627 };
12628
12629 static int __init bnxt_init(void)
12630 {
12631         bnxt_debug_init();
12632         return pci_register_driver(&bnxt_pci_driver);
12633 }
12634
12635 static void __exit bnxt_exit(void)
12636 {
12637         pci_unregister_driver(&bnxt_pci_driver);
12638         if (bnxt_pf_wq)
12639                 destroy_workqueue(bnxt_pf_wq);
12640         bnxt_debug_exit();
12641 }
12642
12643 module_init(bnxt_init);
12644 module_exit(bnxt_exit);