drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/gro.h>
  41 #include <net/ip.h>
  42 #include <net/tcp.h>
  43 #include <net/udp.h>
  44 #include <net/checksum.h>
  45 #include <net/ip6_checksum.h>
  46 #include <net/udp_tunnel.h>
  47 #include <linux/workqueue.h>
  48 #include <linux/prefetch.h>
  49 #include <linux/cache.h>
  50 #include <linux/log2.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/cpu_rmap.h>
  53 #include <linux/cpumask.h>
  54 #include <net/pkt_cls.h>
  55 #include <net/page_pool/helpers.h>
  56 #include <linux/align.h>
  57 #include <net/netdev_queues.h>
  58
  59 #include "bnxt_hsi.h"
  60 #include "bnxt.h"
  61 #include "bnxt_hwrm.h"
  62 #include "bnxt_ulp.h"
  63 #include "bnxt_sriov.h"
  64 #include "bnxt_ethtool.h"
  65 #include "bnxt_dcb.h"
  66 #include "bnxt_xdp.h"
  67 #include "bnxt_ptp.h"
  68 #include "bnxt_vfr.h"
  69 #include "bnxt_tc.h"
  70 #include "bnxt_devlink.h"
  71 #include "bnxt_debugfs.h"
  72 #include "bnxt_hwmon.h"
  73
  74 #define BNXT_TX_TIMEOUT         (5 * HZ)
  75 #define BNXT_DEF_MSG_ENABLE     (NETIF_MSG_DRV | NETIF_MSG_HW | \
  76                                  NETIF_MSG_TX_ERR)
  77
  78 MODULE_LICENSE("GPL");
  79 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  80
  81 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  82 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  83 #define BNXT_RX_COPY_THRESH 256
  84
  85 #define BNXT_TX_PUSH_THRESH 164
  86
  87 /* indexed by enum board_idx */
  88 static const struct {
  89         char *name;
  90 } board_info[] = {
  91         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
  92         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
  93         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
  94         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
  95         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
  96         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
  97         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
  98         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
  99         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 100         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 101         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 102         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 103         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 104         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 105         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 106         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 107         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 108         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 109         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 110         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 111         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 112         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 113         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 114         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 115         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 116         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 117         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 118         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 119         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 120         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 121         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 122         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 123         [BCM57608] = { "Broadcom BCM57608 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" },
 124         [BCM57604] = { "Broadcom BCM57604 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 125         [BCM57602] = { "Broadcom BCM57602 NetXtreme-E 10Gb/25Gb/50Gb/100Gb Ethernet" },
 126         [BCM57601] = { "Broadcom BCM57601 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" },
 127         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 128         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 129         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 130         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 131         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 132         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 133         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 134         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 135         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 136         [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
 137         [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
 138         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 139         [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
 140 };
 141
 142 static const struct pci_device_id bnxt_pci_tbl[] = {
 143         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 144         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 145         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 146         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 147         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 148         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 149         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 150         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 151         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 152         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 153         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 154         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 155         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 156         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 157         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 158         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 159         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 160         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 161         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 162         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 163         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 164         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 165         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 166         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 167         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 168         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 169         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 170         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 171         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 172         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 173         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 174         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 175         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 176         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 177         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 178         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 179         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 180         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 181         { PCI_VDEVICE(BROADCOM, 0x1760), .driver_data = BCM57608 },
 182         { PCI_VDEVICE(BROADCOM, 0x1761), .driver_data = BCM57604 },
 183         { PCI_VDEVICE(BROADCOM, 0x1762), .driver_data = BCM57602 },
 184         { PCI_VDEVICE(BROADCOM, 0x1763), .driver_data = BCM57601 },
 185         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR },
 186         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 187         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR },
 188         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR },
 189         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 190         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR },
 191         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 192         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 193 #ifdef CONFIG_BNXT_SRIOV
 194         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 195         { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
 196         { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
 197         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 198         { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
 199         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 200         { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
 201         { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
 202         { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
 203         { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
 204         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 205         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 206         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 207         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 208         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 209         { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
 210         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 211         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 212         { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
 213         { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
 214         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 215 #endif
 216         { 0 }
 217 };
 218
 219 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 220
 221 static const u16 bnxt_vf_req_snif[] = {
 222         HWRM_FUNC_CFG,
 223         HWRM_FUNC_VF_CFG,
 224         HWRM_PORT_PHY_QCFG,
 225         HWRM_CFA_L2_FILTER_ALLOC,
 226 };
 227
 228 static const u16 bnxt_async_events_arr[] = {
 229         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 230         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
 231         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 232         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 233         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 234         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 235         ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
 236         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 237         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 238         ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
 239         ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE,
 240         ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
 241         ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
 242         ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP,
 243         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT,
 244         ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE,
 245 };
 246
 247 static struct workqueue_struct *bnxt_pf_wq;
 248
 249 static bool bnxt_vf_pciid(enum board_idx idx)
 250 {
 251         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 252                 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
 253                 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
 254                 idx == NETXTREME_E_P5_VF_HV);
 255 }
 256
 257 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 258 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 259 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 260
 261 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 262                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 263
 264 #define BNXT_DB_CQ(db, idx)                                             \
 265         writel(DB_CP_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell)
 266
 267 #define BNXT_DB_NQ_P5(db, idx)                                          \
 268         bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | DB_RING_IDX(db, idx),\
 269                     (db)->doorbell)
 270
 271 #define BNXT_DB_NQ_P7(db, idx)                                          \
 272         bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_MASK |             \
 273                     DB_RING_IDX(db, idx), (db)->doorbell)
 274
 275 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 276         writel(DB_CP_REARM_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell)
 277
 278 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 279         bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM |              \
 280                     DB_RING_IDX(db, idx), (db)->doorbell)
 281
 282 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 283 {
 284         if (bp->flags & BNXT_FLAG_CHIP_P7)
 285                 BNXT_DB_NQ_P7(db, idx);
 286         else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
 287                 BNXT_DB_NQ_P5(db, idx);
 288         else
 289                 BNXT_DB_CQ(db, idx);
 290 }
 291
 292 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 293 {
 294         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
 295                 BNXT_DB_NQ_ARM_P5(db, idx);
 296         else
 297                 BNXT_DB_CQ_ARM(db, idx);
 298 }
 299
 300 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 301 {
 302         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
 303                 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL |
 304                             DB_RING_IDX(db, idx), db->doorbell);
 305         else
 306                 BNXT_DB_CQ(db, idx);
 307 }
 308
 309 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
 310 {
 311         if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
 312                 return;
 313
 314         if (BNXT_PF(bp))
 315                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
 316         else
 317                 schedule_delayed_work(&bp->fw_reset_task, delay);
 318 }
 319
 320 static void __bnxt_queue_sp_work(struct bnxt *bp)
 321 {
 322         if (BNXT_PF(bp))
 323                 queue_work(bnxt_pf_wq, &bp->sp_task);
 324         else
 325                 schedule_work(&bp->sp_task);
 326 }
 327
 328 static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event)
 329 {
 330         set_bit(event, &bp->sp_event);
 331         __bnxt_queue_sp_work(bp);
 332 }
 333
 334 static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
 335 {
 336         if (!rxr->bnapi->in_reset) {
 337                 rxr->bnapi->in_reset = true;
 338                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
 339                         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
 340                 else
 341                         set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
 342                 __bnxt_queue_sp_work(bp);
 343         }
 344         rxr->rx_next_cons = 0xffff;
 345 }
 346
 347 void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 348                           u16 curr)
 349 {
 350         struct bnxt_napi *bnapi = txr->bnapi;
 351
 352         if (bnapi->tx_fault)
 353                 return;
 354
 355         netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_hw_cons:%u cons:%u prod:%u curr:%u)",
 356                    txr->txq_index, txr->tx_hw_cons,
 357                    txr->tx_cons, txr->tx_prod, curr);
 358         WARN_ON_ONCE(1);
 359         bnapi->tx_fault = 1;
 360         bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
 361 }
 362
 363 const u16 bnxt_lhint_arr[] = {
 364         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 365         TX_BD_FLAGS_LHINT_512_TO_1023,
 366         TX_BD_FLAGS_LHINT_1024_TO_2047,
 367         TX_BD_FLAGS_LHINT_1024_TO_2047,
 368         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 369         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 370         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 371         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 372         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 373         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 374         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 375         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 376         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 377         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 378         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 379         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 380         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 381         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 382         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 383 };
 384
 385 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 386 {
 387         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 388
 389         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 390                 return 0;
 391
 392         return md_dst->u.port_info.port_id;
 393 }
 394
 395 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 396                              u16 prod)
 397 {
 398         /* Sync BD data before updating doorbell */
 399         wmb();
 400         bnxt_db_write(bp, &txr->tx_db, prod);
 401         txr->kick_pending = 0;
 402 }
 403
 404 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 405 {
 406         struct bnxt *bp = netdev_priv(dev);
 407         struct tx_bd *txbd, *txbd0;
 408         struct tx_bd_ext *txbd1;
 409         struct netdev_queue *txq;
 410         int i;
 411         dma_addr_t mapping;
 412         unsigned int length, pad = 0;
 413         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 414         u16 prod, last_frag;
 415         struct pci_dev *pdev = bp->pdev;
 416         struct bnxt_tx_ring_info *txr;
 417         struct bnxt_sw_tx_bd *tx_buf;
 418         __le32 lflags = 0;
 419
 420         i = skb_get_queue_mapping(skb);
 421         if (unlikely(i >= bp->tx_nr_rings)) {
 422                 dev_kfree_skb_any(skb);
 423                 dev_core_stats_tx_dropped_inc(dev);
 424                 return NETDEV_TX_OK;
 425         }
 426
 427         txq = netdev_get_tx_queue(dev, i);
 428         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 429         prod = txr->tx_prod;
 430
 431         free_size = bnxt_tx_avail(bp, txr);
 432         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 433                 /* We must have raced with NAPI cleanup */
 434                 if (net_ratelimit() && txr->kick_pending)
 435                         netif_warn(bp, tx_err, dev,
 436                                    "bnxt: ring busy w/ flush pending!\n");
 437                 if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
 438                                         bp->tx_wake_thresh))
 439                         return NETDEV_TX_BUSY;
 440         }
 441
 442         if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
 443                 goto tx_free;
 444
 445         length = skb->len;
 446         len = skb_headlen(skb);
 447         last_frag = skb_shinfo(skb)->nr_frags;
 448
 449         txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
 450
 451         tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
 452         tx_buf->skb = skb;
 453         tx_buf->nr_frags = last_frag;
 454
 455         vlan_tag_flags = 0;
 456         cfa_action = bnxt_xmit_get_cfa_action(skb);
 457         if (skb_vlan_tag_present(skb)) {
 458                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 459                                  skb_vlan_tag_get(skb);
 460                 /* Currently supports 8021Q, 8021AD vlan offloads
 461                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 462                  */
 463                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 464                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 465         }
 466
 467         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
 468                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
 469
 470                 if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) &&
 471                     atomic_dec_if_positive(&ptp->tx_avail) >= 0) {
 472                         if (!bnxt_ptp_parse(skb, &ptp->tx_seqid,
 473                                             &ptp->tx_hdr_off)) {
 474                                 if (vlan_tag_flags)
 475                                         ptp->tx_hdr_off += VLAN_HLEN;
 476                                 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
 477                                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 478                         } else {
 479                                 atomic_inc(&bp->ptp_cfg->tx_avail);
 480                         }
 481                 }
 482         }
 483
 484         if (unlikely(skb->no_fcs))
 485                 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
 486
 487         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
 488             !lflags) {
 489                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 490                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 491                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 492                 void __iomem *db = txr->tx_db.doorbell;
 493                 void *pdata = tx_push_buf->data;
 494                 u64 *end;
 495                 int j, push_len;
 496
 497                 /* Set COAL_NOW to be ready quickly for the next push */
 498                 tx_push->tx_bd_len_flags_type =
 499                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 500                                         TX_BD_TYPE_LONG_TX_BD |
 501                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 502                                         TX_BD_FLAGS_COAL_NOW |
 503                                         TX_BD_FLAGS_PACKET_END |
 504                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 505
 506                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 507                         tx_push1->tx_bd_hsize_lflags =
 508                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 509                 else
 510                         tx_push1->tx_bd_hsize_lflags = 0;
 511
 512                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 513                 tx_push1->tx_bd_cfa_action =
 514                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 515
 516                 end = pdata + length;
 517                 end = PTR_ALIGN(end, 8) - 1;
 518                 *end = 0;
 519
 520                 skb_copy_from_linear_data(skb, pdata, len);
 521                 pdata += len;
 522                 for (j = 0; j < last_frag; j++) {
 523                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 524                         void *fptr;
 525
 526                         fptr = skb_frag_address_safe(frag);
 527                         if (!fptr)
 528                                 goto normal_tx;
 529
 530                         memcpy(pdata, fptr, skb_frag_size(frag));
 531                         pdata += skb_frag_size(frag);
 532                 }
 533
 534                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 535                 txbd->tx_bd_haddr = txr->data_mapping;
 536                 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2);
 537                 prod = NEXT_TX(prod);
 538                 tx_push->tx_bd_opaque = txbd->tx_bd_opaque;
 539                 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
 540                 memcpy(txbd, tx_push1, sizeof(*txbd));
 541                 prod = NEXT_TX(prod);
 542                 tx_push->doorbell =
 543                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH |
 544                                     DB_RING_IDX(&txr->tx_db, prod));
 545                 WRITE_ONCE(txr->tx_prod, prod);
 546
 547                 tx_buf->is_push = 1;
 548                 netdev_tx_sent_queue(txq, skb->len);
 549                 wmb();  /* Sync is_push and byte queue before pushing data */
 550
 551                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 552                 if (push_len > 16) {
 553                         __iowrite64_copy(db, tx_push_buf, 16);
 554                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 555                                          (push_len - 16) << 1);
 556                 } else {
 557                         __iowrite64_copy(db, tx_push_buf, push_len);
 558                 }
 559
 560                 goto tx_done;
 561         }
 562
 563 normal_tx:
 564         if (length < BNXT_MIN_PKT_SIZE) {
 565                 pad = BNXT_MIN_PKT_SIZE - length;
 566                 if (skb_pad(skb, pad))
 567                         /* SKB already freed. */
 568                         goto tx_kick_pending;
 569                 length = BNXT_MIN_PKT_SIZE;
 570         }
 571
 572         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 573
 574         if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 575                 goto tx_free;
 576
 577         dma_unmap_addr_set(tx_buf, mapping, mapping);
 578         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 579                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 580
 581         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 582         txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2 + last_frag);
 583
 584         prod = NEXT_TX(prod);
 585         txbd1 = (struct tx_bd_ext *)
 586                 &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
 587
 588         txbd1->tx_bd_hsize_lflags = lflags;
 589         if (skb_is_gso(skb)) {
 590                 bool udp_gso = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4);
 591                 u32 hdr_len;
 592
 593                 if (skb->encapsulation) {
 594                         if (udp_gso)
 595                                 hdr_len = skb_inner_transport_offset(skb) +
 596                                           sizeof(struct udphdr);
 597                         else
 598                                 hdr_len = skb_inner_tcp_all_headers(skb);
 599                 } else if (udp_gso) {
 600                         hdr_len = skb_transport_offset(skb) +
 601                                   sizeof(struct udphdr);
 602                 } else {
 603                         hdr_len = skb_tcp_all_headers(skb);
 604                 }
 605
 606                 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
 607                                         TX_BD_FLAGS_T_IPID |
 608                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 609                 length = skb_shinfo(skb)->gso_size;
 610                 txbd1->tx_bd_mss = cpu_to_le32(length);
 611                 length += hdr_len;
 612         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 613                 txbd1->tx_bd_hsize_lflags |=
 614                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 615                 txbd1->tx_bd_mss = 0;
 616         }
 617
 618         length >>= 9;
 619         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 620                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 621                                      skb->len);
 622                 i = 0;
 623                 goto tx_dma_error;
 624         }
 625         flags |= bnxt_lhint_arr[length];
 626         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 627
 628         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 629         txbd1->tx_bd_cfa_action =
 630                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 631         txbd0 = txbd;
 632         for (i = 0; i < last_frag; i++) {
 633                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 634
 635                 prod = NEXT_TX(prod);
 636                 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
 637
 638                 len = skb_frag_size(frag);
 639                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 640                                            DMA_TO_DEVICE);
 641
 642                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 643                         goto tx_dma_error;
 644
 645                 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
 646                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 647
 648                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 649
 650                 flags = len << TX_BD_LEN_SHIFT;
 651                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 652         }
 653
 654         flags &= ~TX_BD_LEN;
 655         txbd->tx_bd_len_flags_type =
 656                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 657                             TX_BD_FLAGS_PACKET_END);
 658
 659         netdev_tx_sent_queue(txq, skb->len);
 660
 661         skb_tx_timestamp(skb);
 662
 663         prod = NEXT_TX(prod);
 664         WRITE_ONCE(txr->tx_prod, prod);
 665
 666         if (!netdev_xmit_more() || netif_xmit_stopped(txq)) {
 667                 bnxt_txr_db_kick(bp, txr, prod);
 668         } else {
 669                 if (free_size >= bp->tx_wake_thresh)
 670                         txbd0->tx_bd_len_flags_type |=
 671                                 cpu_to_le32(TX_BD_FLAGS_NO_CMPL);
 672                 txr->kick_pending = 1;
 673         }
 674
 675 tx_done:
 676
 677         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 678                 if (netdev_xmit_more() && !tx_buf->is_push) {
 679                         txbd0->tx_bd_len_flags_type &=
 680                                 cpu_to_le32(~TX_BD_FLAGS_NO_CMPL);
 681                         bnxt_txr_db_kick(bp, txr, prod);
 682                 }
 683
 684                 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
 685                                    bp->tx_wake_thresh);
 686         }
 687         return NETDEV_TX_OK;
 688
 689 tx_dma_error:
 690         if (BNXT_TX_PTP_IS_SET(lflags))
 691                 atomic_inc(&bp->ptp_cfg->tx_avail);
 692
 693         last_frag = i;
 694
 695         /* start back at beginning and unmap skb */
 696         prod = txr->tx_prod;
 697         tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
 698         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 699                          skb_headlen(skb), DMA_TO_DEVICE);
 700         prod = NEXT_TX(prod);
 701
 702         /* unmap remaining mapped pages */
 703         for (i = 0; i < last_frag; i++) {
 704                 prod = NEXT_TX(prod);
 705                 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
 706                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 707                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 708                                DMA_TO_DEVICE);
 709         }
 710
 711 tx_free:
 712         dev_kfree_skb_any(skb);
 713 tx_kick_pending:
 714         if (txr->kick_pending)
 715                 bnxt_txr_db_kick(bp, txr, txr->tx_prod);
 716         txr->tx_buf_ring[txr->tx_prod].skb = NULL;
 717         dev_core_stats_tx_dropped_inc(dev);
 718         return NETDEV_TX_OK;
 719 }
 720
 721 static void __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 722                           int budget)
 723 {
 724         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 725         struct pci_dev *pdev = bp->pdev;
 726         u16 hw_cons = txr->tx_hw_cons;
 727         unsigned int tx_bytes = 0;
 728         u16 cons = txr->tx_cons;
 729         int tx_pkts = 0;
 730
 731         while (RING_TX(bp, cons) != hw_cons) {
 732                 struct bnxt_sw_tx_bd *tx_buf;
 733                 struct sk_buff *skb;
 734                 int j, last;
 735
 736                 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)];
 737                 cons = NEXT_TX(cons);
 738                 skb = tx_buf->skb;
 739                 tx_buf->skb = NULL;
 740
 741                 if (unlikely(!skb)) {
 742                         bnxt_sched_reset_txr(bp, txr, cons);
 743                         return;
 744                 }
 745
 746                 tx_pkts++;
 747                 tx_bytes += skb->len;
 748
 749                 if (tx_buf->is_push) {
 750                         tx_buf->is_push = 0;
 751                         goto next_tx_int;
 752                 }
 753
 754                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 755                                  skb_headlen(skb), DMA_TO_DEVICE);
 756                 last = tx_buf->nr_frags;
 757
 758                 for (j = 0; j < last; j++) {
 759                         cons = NEXT_TX(cons);
 760                         tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)];
 761                         dma_unmap_page(
 762                                 &pdev->dev,
 763                                 dma_unmap_addr(tx_buf, mapping),
 764                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 765                                 DMA_TO_DEVICE);
 766                 }
 767                 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
 768                         if (BNXT_CHIP_P5(bp)) {
 769                                 /* PTP worker takes ownership of the skb */
 770                                 if (!bnxt_get_tx_ts_p5(bp, skb))
 771                                         skb = NULL;
 772                                 else
 773                                         atomic_inc(&bp->ptp_cfg->tx_avail);
 774                         }
 775                 }
 776
 777 next_tx_int:
 778                 cons = NEXT_TX(cons);
 779
 780                 dev_consume_skb_any(skb);
 781         }
 782
 783         WRITE_ONCE(txr->tx_cons, cons);
 784
 785         __netif_txq_completed_wake(txq, tx_pkts, tx_bytes,
 786                                    bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
 787                                    READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING);
 788 }
 789
 790 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
 791 {
 792         struct bnxt_tx_ring_info *txr;
 793         int i;
 794
 795         bnxt_for_each_napi_tx(i, bnapi, txr) {
 796                 if (txr->tx_hw_cons != RING_TX(bp, txr->tx_cons))
 797                         __bnxt_tx_int(bp, txr, budget);
 798         }
 799         bnapi->events &= ~BNXT_TX_CMP_EVENT;
 800 }
 801
 802 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 803                                          struct bnxt_rx_ring_info *rxr,
 804                                          unsigned int *offset,
 805                                          gfp_t gfp)
 806 {
 807         struct page *page;
 808
 809         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 810                 page = page_pool_dev_alloc_frag(rxr->page_pool, offset,
 811                                                 BNXT_RX_PAGE_SIZE);
 812         } else {
 813                 page = page_pool_dev_alloc_pages(rxr->page_pool);
 814                 *offset = 0;
 815         }
 816         if (!page)
 817                 return NULL;
 818
 819         *mapping = page_pool_get_dma_addr(page) + *offset;
 820         return page;
 821 }
 822
 823 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping,
 824                                        gfp_t gfp)
 825 {
 826         u8 *data;
 827         struct pci_dev *pdev = bp->pdev;
 828
 829         if (gfp == GFP_ATOMIC)
 830                 data = napi_alloc_frag(bp->rx_buf_size);
 831         else
 832                 data = netdev_alloc_frag(bp->rx_buf_size);
 833         if (!data)
 834                 return NULL;
 835
 836         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 837                                         bp->rx_buf_use_size, bp->rx_dir,
 838                                         DMA_ATTR_WEAK_ORDERING);
 839
 840         if (dma_mapping_error(&pdev->dev, *mapping)) {
 841                 skb_free_frag(data);
 842                 data = NULL;
 843         }
 844         return data;
 845 }
 846
 847 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 848                        u16 prod, gfp_t gfp)
 849 {
 850         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
 851         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
 852         dma_addr_t mapping;
 853
 854         if (BNXT_RX_PAGE_MODE(bp)) {
 855                 unsigned int offset;
 856                 struct page *page =
 857                         __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
 858
 859                 if (!page)
 860                         return -ENOMEM;
 861
 862                 mapping += bp->rx_dma_offset;
 863                 rx_buf->data = page;
 864                 rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset;
 865         } else {
 866                 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp);
 867
 868                 if (!data)
 869                         return -ENOMEM;
 870
 871                 rx_buf->data = data;
 872                 rx_buf->data_ptr = data + bp->rx_offset;
 873         }
 874         rx_buf->mapping = mapping;
 875
 876         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 877         return 0;
 878 }
 879
 880 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 881 {
 882         u16 prod = rxr->rx_prod;
 883         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 884         struct bnxt *bp = rxr->bnapi->bp;
 885         struct rx_bd *cons_bd, *prod_bd;
 886
 887         prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
 888         cons_rx_buf = &rxr->rx_buf_ring[cons];
 889
 890         prod_rx_buf->data = data;
 891         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 892
 893         prod_rx_buf->mapping = cons_rx_buf->mapping;
 894
 895         prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
 896         cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)];
 897
 898         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 899 }
 900
 901 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 902 {
 903         u16 next, max = rxr->rx_agg_bmap_size;
 904
 905         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 906         if (next >= max)
 907                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 908         return next;
 909 }
 910
 911 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 912                                      struct bnxt_rx_ring_info *rxr,
 913                                      u16 prod, gfp_t gfp)
 914 {
 915         struct rx_bd *rxbd =
 916                 &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)];
 917         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 918         struct page *page;
 919         dma_addr_t mapping;
 920         u16 sw_prod = rxr->rx_sw_agg_prod;
 921         unsigned int offset = 0;
 922
 923         page = __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
 924
 925         if (!page)
 926                 return -ENOMEM;
 927
 928         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 929                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 930
 931         __set_bit(sw_prod, rxr->rx_agg_bmap);
 932         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 933         rxr->rx_sw_agg_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod));
 934
 935         rx_agg_buf->page = page;
 936         rx_agg_buf->offset = offset;
 937         rx_agg_buf->mapping = mapping;
 938         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 939         rxbd->rx_bd_opaque = sw_prod;
 940         return 0;
 941 }
 942
 943 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 944                                        struct bnxt_cp_ring_info *cpr,
 945                                        u16 cp_cons, u16 curr)
 946 {
 947         struct rx_agg_cmp *agg;
 948
 949         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 950         agg = (struct rx_agg_cmp *)
 951                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 952         return agg;
 953 }
 954
 955 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 956                                               struct bnxt_rx_ring_info *rxr,
 957                                               u16 agg_id, u16 curr)
 958 {
 959         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 960
 961         return &tpa_info->agg_arr[curr];
 962 }
 963
 964 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 965                                    u16 start, u32 agg_bufs, bool tpa)
 966 {
 967         struct bnxt_napi *bnapi = cpr->bnapi;
 968         struct bnxt *bp = bnapi->bp;
 969         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 970         u16 prod = rxr->rx_agg_prod;
 971         u16 sw_prod = rxr->rx_sw_agg_prod;
 972         bool p5_tpa = false;
 973         u32 i;
 974
 975         if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa)
 976                 p5_tpa = true;
 977
 978         for (i = 0; i < agg_bufs; i++) {
 979                 u16 cons;
 980                 struct rx_agg_cmp *agg;
 981                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 982                 struct rx_bd *prod_bd;
 983                 struct page *page;
 984
 985                 if (p5_tpa)
 986                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 987                 else
 988                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 989                 cons = agg->rx_agg_cmp_opaque;
 990                 __clear_bit(cons, rxr->rx_agg_bmap);
 991
 992                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 993                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 994
 995                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 996                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 997                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 998
 999                 /* It is possible for sw_prod to be equal to cons, so
1000                  * set cons_rx_buf->page to NULL first.
1001                  */
1002                 page = cons_rx_buf->page;
1003                 cons_rx_buf->page = NULL;
1004                 prod_rx_buf->page = page;
1005                 prod_rx_buf->offset = cons_rx_buf->offset;
1006
1007                 prod_rx_buf->mapping = cons_rx_buf->mapping;
1008
1009                 prod_bd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)];
1010
1011                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
1012                 prod_bd->rx_bd_opaque = sw_prod;
1013
1014                 prod = NEXT_RX_AGG(prod);
1015                 sw_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod));
1016         }
1017         rxr->rx_agg_prod = prod;
1018         rxr->rx_sw_agg_prod = sw_prod;
1019 }
1020
1021 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp,
1022                                               struct bnxt_rx_ring_info *rxr,
1023                                               u16 cons, void *data, u8 *data_ptr,
1024                                               dma_addr_t dma_addr,
1025                                               unsigned int offset_and_len)
1026 {
1027         unsigned int len = offset_and_len & 0xffff;
1028         struct page *page = data;
1029         u16 prod = rxr->rx_prod;
1030         struct sk_buff *skb;
1031         int err;
1032
1033         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1034         if (unlikely(err)) {
1035                 bnxt_reuse_rx_data(rxr, cons, data);
1036                 return NULL;
1037         }
1038         dma_addr -= bp->rx_dma_offset;
1039         dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1040                                 bp->rx_dir);
1041         skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE);
1042         if (!skb) {
1043                 page_pool_recycle_direct(rxr->page_pool, page);
1044                 return NULL;
1045         }
1046         skb_mark_for_recycle(skb);
1047         skb_reserve(skb, bp->rx_offset);
1048         __skb_put(skb, len);
1049
1050         return skb;
1051 }
1052
1053 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
1054                                         struct bnxt_rx_ring_info *rxr,
1055                                         u16 cons, void *data, u8 *data_ptr,
1056                                         dma_addr_t dma_addr,
1057                                         unsigned int offset_and_len)
1058 {
1059         unsigned int payload = offset_and_len >> 16;
1060         unsigned int len = offset_and_len & 0xffff;
1061         skb_frag_t *frag;
1062         struct page *page = data;
1063         u16 prod = rxr->rx_prod;
1064         struct sk_buff *skb;
1065         int off, err;
1066
1067         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1068         if (unlikely(err)) {
1069                 bnxt_reuse_rx_data(rxr, cons, data);
1070                 return NULL;
1071         }
1072         dma_addr -= bp->rx_dma_offset;
1073         dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1074                                 bp->rx_dir);
1075
1076         if (unlikely(!payload))
1077                 payload = eth_get_headlen(bp->dev, data_ptr, len);
1078
1079         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1080         if (!skb) {
1081                 page_pool_recycle_direct(rxr->page_pool, page);
1082                 return NULL;
1083         }
1084
1085         skb_mark_for_recycle(skb);
1086         off = (void *)data_ptr - page_address(page);
1087         skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE);
1088         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1089                payload + NET_IP_ALIGN);
1090
1091         frag = &skb_shinfo(skb)->frags[0];
1092         skb_frag_size_sub(frag, payload);
1093         skb_frag_off_add(frag, payload);
1094         skb->data_len -= payload;
1095         skb->tail += payload;
1096
1097         return skb;
1098 }
1099
1100 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1101                                    struct bnxt_rx_ring_info *rxr, u16 cons,
1102                                    void *data, u8 *data_ptr,
1103                                    dma_addr_t dma_addr,
1104                                    unsigned int offset_and_len)
1105 {
1106         u16 prod = rxr->rx_prod;
1107         struct sk_buff *skb;
1108         int err;
1109
1110         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1111         if (unlikely(err)) {
1112                 bnxt_reuse_rx_data(rxr, cons, data);
1113                 return NULL;
1114         }
1115
1116         skb = napi_build_skb(data, bp->rx_buf_size);
1117         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1118                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1119         if (!skb) {
1120                 skb_free_frag(data);
1121                 return NULL;
1122         }
1123
1124         skb_reserve(skb, bp->rx_offset);
1125         skb_put(skb, offset_and_len & 0xffff);
1126         return skb;
1127 }
1128
1129 static u32 __bnxt_rx_agg_pages(struct bnxt *bp,
1130                                struct bnxt_cp_ring_info *cpr,
1131                                struct skb_shared_info *shinfo,
1132                                u16 idx, u32 agg_bufs, bool tpa,
1133                                struct xdp_buff *xdp)
1134 {
1135         struct bnxt_napi *bnapi = cpr->bnapi;
1136         struct pci_dev *pdev = bp->pdev;
1137         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1138         u16 prod = rxr->rx_agg_prod;
1139         u32 i, total_frag_len = 0;
1140         bool p5_tpa = false;
1141
1142         if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa)
1143                 p5_tpa = true;
1144
1145         for (i = 0; i < agg_bufs; i++) {
1146                 skb_frag_t *frag = &shinfo->frags[i];
1147                 u16 cons, frag_len;
1148                 struct rx_agg_cmp *agg;
1149                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1150                 struct page *page;
1151                 dma_addr_t mapping;
1152
1153                 if (p5_tpa)
1154                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1155                 else
1156                         agg = bnxt_get_agg(bp, cpr, idx, i);
1157                 cons = agg->rx_agg_cmp_opaque;
1158                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1159                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1160
1161                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1162                 skb_frag_fill_page_desc(frag, cons_rx_buf->page,
1163                                         cons_rx_buf->offset, frag_len);
1164                 shinfo->nr_frags = i + 1;
1165                 __clear_bit(cons, rxr->rx_agg_bmap);
1166
1167                 /* It is possible for bnxt_alloc_rx_page() to allocate
1168                  * a sw_prod index that equals the cons index, so we
1169                  * need to clear the cons entry now.
1170                  */
1171                 mapping = cons_rx_buf->mapping;
1172                 page = cons_rx_buf->page;
1173                 cons_rx_buf->page = NULL;
1174
1175                 if (xdp && page_is_pfmemalloc(page))
1176                         xdp_buff_set_frag_pfmemalloc(xdp);
1177
1178                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1179                         --shinfo->nr_frags;
1180                         cons_rx_buf->page = page;
1181
1182                         /* Update prod since possibly some pages have been
1183                          * allocated already.
1184                          */
1185                         rxr->rx_agg_prod = prod;
1186                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1187                         return 0;
1188                 }
1189
1190                 dma_sync_single_for_cpu(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1191                                         bp->rx_dir);
1192
1193                 total_frag_len += frag_len;
1194                 prod = NEXT_RX_AGG(prod);
1195         }
1196         rxr->rx_agg_prod = prod;
1197         return total_frag_len;
1198 }
1199
1200 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,
1201                                              struct bnxt_cp_ring_info *cpr,
1202                                              struct sk_buff *skb, u16 idx,
1203                                              u32 agg_bufs, bool tpa)
1204 {
1205         struct skb_shared_info *shinfo = skb_shinfo(skb);
1206         u32 total_frag_len = 0;
1207
1208         total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx,
1209                                              agg_bufs, tpa, NULL);
1210         if (!total_frag_len) {
1211                 skb_mark_for_recycle(skb);
1212                 dev_kfree_skb(skb);
1213                 return NULL;
1214         }
1215
1216         skb->data_len += total_frag_len;
1217         skb->len += total_frag_len;
1218         skb->truesize += BNXT_RX_PAGE_SIZE * agg_bufs;
1219         return skb;
1220 }
1221
1222 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,
1223                                  struct bnxt_cp_ring_info *cpr,
1224                                  struct xdp_buff *xdp, u16 idx,
1225                                  u32 agg_bufs, bool tpa)
1226 {
1227         struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp);
1228         u32 total_frag_len = 0;
1229
1230         if (!xdp_buff_has_frags(xdp))
1231                 shinfo->nr_frags = 0;
1232
1233         total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo,
1234                                              idx, agg_bufs, tpa, xdp);
1235         if (total_frag_len) {
1236                 xdp_buff_set_frags_flag(xdp);
1237                 shinfo->nr_frags = agg_bufs;
1238                 shinfo->xdp_frags_size = total_frag_len;
1239         }
1240         return total_frag_len;
1241 }
1242
1243 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1244                                u8 agg_bufs, u32 *raw_cons)
1245 {
1246         u16 last;
1247         struct rx_agg_cmp *agg;
1248
1249         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1250         last = RING_CMP(*raw_cons);
1251         agg = (struct rx_agg_cmp *)
1252                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1253         return RX_AGG_CMP_VALID(agg, *raw_cons);
1254 }
1255
1256 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1257                                             unsigned int len,
1258                                             dma_addr_t mapping)
1259 {
1260         struct bnxt *bp = bnapi->bp;
1261         struct pci_dev *pdev = bp->pdev;
1262         struct sk_buff *skb;
1263
1264         skb = napi_alloc_skb(&bnapi->napi, len);
1265         if (!skb)
1266                 return NULL;
1267
1268         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1269                                 bp->rx_dir);
1270
1271         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1272                len + NET_IP_ALIGN);
1273
1274         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1275                                    bp->rx_dir);
1276
1277         skb_put(skb, len);
1278         return skb;
1279 }
1280
1281 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1282                            u32 *raw_cons, void *cmp)
1283 {
1284         struct rx_cmp *rxcmp = cmp;
1285         u32 tmp_raw_cons = *raw_cons;
1286         u8 cmp_type, agg_bufs = 0;
1287
1288         cmp_type = RX_CMP_TYPE(rxcmp);
1289
1290         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1291                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1292                             RX_CMP_AGG_BUFS) >>
1293                            RX_CMP_AGG_BUFS_SHIFT;
1294         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1295                 struct rx_tpa_end_cmp *tpa_end = cmp;
1296
1297                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
1298                         return 0;
1299
1300                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1301         }
1302
1303         if (agg_bufs) {
1304                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1305                         return -EBUSY;
1306         }
1307         *raw_cons = tmp_raw_cons;
1308         return 0;
1309 }
1310
1311 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1312 {
1313         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1314         u16 idx = agg_id & MAX_TPA_P5_MASK;
1315
1316         if (test_bit(idx, map->agg_idx_bmap))
1317                 idx = find_first_zero_bit(map->agg_idx_bmap,
1318                                           BNXT_AGG_IDX_BMAP_SIZE);
1319         __set_bit(idx, map->agg_idx_bmap);
1320         map->agg_id_tbl[agg_id] = idx;
1321         return idx;
1322 }
1323
1324 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1325 {
1326         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1327
1328         __clear_bit(idx, map->agg_idx_bmap);
1329 }
1330
1331 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1332 {
1333         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1334
1335         return map->agg_id_tbl[agg_id];
1336 }
1337
1338 static void bnxt_tpa_metadata(struct bnxt_tpa_info *tpa_info,
1339                               struct rx_tpa_start_cmp *tpa_start,
1340                               struct rx_tpa_start_cmp_ext *tpa_start1)
1341 {
1342         tpa_info->cfa_code_valid = 1;
1343         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1344         tpa_info->vlan_valid = 0;
1345         if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) {
1346                 tpa_info->vlan_valid = 1;
1347                 tpa_info->metadata =
1348                         le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1349         }
1350 }
1351
1352 static void bnxt_tpa_metadata_v2(struct bnxt_tpa_info *tpa_info,
1353                                  struct rx_tpa_start_cmp *tpa_start,
1354                                  struct rx_tpa_start_cmp_ext *tpa_start1)
1355 {
1356         tpa_info->vlan_valid = 0;
1357         if (TPA_START_VLAN_VALID(tpa_start)) {
1358                 u32 tpid_sel = TPA_START_VLAN_TPID_SEL(tpa_start);
1359                 u32 vlan_proto = ETH_P_8021Q;
1360
1361                 tpa_info->vlan_valid = 1;
1362                 if (tpid_sel == RX_TPA_START_METADATA1_TPID_8021AD)
1363                         vlan_proto = ETH_P_8021AD;
1364                 tpa_info->metadata = vlan_proto << 16 |
1365                                      TPA_START_METADATA0_TCI(tpa_start1);
1366         }
1367 }
1368
1369 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1370                            u8 cmp_type, struct rx_tpa_start_cmp *tpa_start,
1371                            struct rx_tpa_start_cmp_ext *tpa_start1)
1372 {
1373         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1374         struct bnxt_tpa_info *tpa_info;
1375         u16 cons, prod, agg_id;
1376         struct rx_bd *prod_bd;
1377         dma_addr_t mapping;
1378
1379         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
1380                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1381                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1382         } else {
1383                 agg_id = TPA_START_AGG_ID(tpa_start);
1384         }
1385         cons = tpa_start->rx_tpa_start_cmp_opaque;
1386         prod = rxr->rx_prod;
1387         cons_rx_buf = &rxr->rx_buf_ring[cons];
1388         prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
1389         tpa_info = &rxr->rx_tpa[agg_id];
1390
1391         if (unlikely(cons != rxr->rx_next_cons ||
1392                      TPA_START_ERROR(tpa_start))) {
1393                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1394                             cons, rxr->rx_next_cons,
1395                             TPA_START_ERROR_CODE(tpa_start1));
1396                 bnxt_sched_reset_rxr(bp, rxr);
1397                 return;
1398         }
1399         prod_rx_buf->data = tpa_info->data;
1400         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1401
1402         mapping = tpa_info->mapping;
1403         prod_rx_buf->mapping = mapping;
1404
1405         prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
1406
1407         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1408
1409         tpa_info->data = cons_rx_buf->data;
1410         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1411         cons_rx_buf->data = NULL;
1412         tpa_info->mapping = cons_rx_buf->mapping;
1413
1414         tpa_info->len =
1415                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1416                                 RX_TPA_START_CMP_LEN_SHIFT;
1417         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1418                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1419                 tpa_info->gso_type = SKB_GSO_TCPV4;
1420                 if (TPA_START_IS_IPV6(tpa_start1))
1421                         tpa_info->gso_type = SKB_GSO_TCPV6;
1422                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1423                 else if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP &&
1424                          TPA_START_HASH_TYPE(tpa_start) == 3)
1425                         tpa_info->gso_type = SKB_GSO_TCPV6;
1426                 tpa_info->rss_hash =
1427                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1428         } else {
1429                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1430                 tpa_info->gso_type = 0;
1431                 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1432         }
1433         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1434         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1435         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP)
1436                 bnxt_tpa_metadata(tpa_info, tpa_start, tpa_start1);
1437         else
1438                 bnxt_tpa_metadata_v2(tpa_info, tpa_start, tpa_start1);
1439         tpa_info->agg_count = 0;
1440
1441         rxr->rx_prod = NEXT_RX(prod);
1442         cons = RING_RX(bp, NEXT_RX(cons));
1443         rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons));
1444         cons_rx_buf = &rxr->rx_buf_ring[cons];
1445
1446         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1447         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1448         cons_rx_buf->data = NULL;
1449 }
1450
1451 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1452 {
1453         if (agg_bufs)
1454                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1455 }
1456
1457 #ifdef CONFIG_INET
1458 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1459 {
1460         struct udphdr *uh = NULL;
1461
1462         if (ip_proto == htons(ETH_P_IP)) {
1463                 struct iphdr *iph = (struct iphdr *)skb->data;
1464
1465                 if (iph->protocol == IPPROTO_UDP)
1466                         uh = (struct udphdr *)(iph + 1);
1467         } else {
1468                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1469
1470                 if (iph->nexthdr == IPPROTO_UDP)
1471                         uh = (struct udphdr *)(iph + 1);
1472         }
1473         if (uh) {
1474                 if (uh->check)
1475                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1476                 else
1477                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1478         }
1479 }
1480 #endif
1481
1482 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1483                                            int payload_off, int tcp_ts,
1484                                            struct sk_buff *skb)
1485 {
1486 #ifdef CONFIG_INET
1487         struct tcphdr *th;
1488         int len, nw_off;
1489         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1490         u32 hdr_info = tpa_info->hdr_info;
1491         bool loopback = false;
1492
1493         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1494         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1495         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1496
1497         /* If the packet is an internal loopback packet, the offsets will
1498          * have an extra 4 bytes.
1499          */
1500         if (inner_mac_off == 4) {
1501                 loopback = true;
1502         } else if (inner_mac_off > 4) {
1503                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1504                                             ETH_HLEN - 2));
1505
1506                 /* We only support inner iPv4/ipv6.  If we don't see the
1507                  * correct protocol ID, it must be a loopback packet where
1508                  * the offsets are off by 4.
1509                  */
1510                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1511                         loopback = true;
1512         }
1513         if (loopback) {
1514                 /* internal loopback packet, subtract all offsets by 4 */
1515                 inner_ip_off -= 4;
1516                 inner_mac_off -= 4;
1517                 outer_ip_off -= 4;
1518         }
1519
1520         nw_off = inner_ip_off - ETH_HLEN;
1521         skb_set_network_header(skb, nw_off);
1522         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1523                 struct ipv6hdr *iph = ipv6_hdr(skb);
1524
1525                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1526                 len = skb->len - skb_transport_offset(skb);
1527                 th = tcp_hdr(skb);
1528                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1529         } else {
1530                 struct iphdr *iph = ip_hdr(skb);
1531
1532                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1533                 len = skb->len - skb_transport_offset(skb);
1534                 th = tcp_hdr(skb);
1535                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1536         }
1537
1538         if (inner_mac_off) { /* tunnel */
1539                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1540                                             ETH_HLEN - 2));
1541
1542                 bnxt_gro_tunnel(skb, proto);
1543         }
1544 #endif
1545         return skb;
1546 }
1547
1548 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1549                                            int payload_off, int tcp_ts,
1550                                            struct sk_buff *skb)
1551 {
1552 #ifdef CONFIG_INET
1553         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1554         u32 hdr_info = tpa_info->hdr_info;
1555         int iphdr_len, nw_off;
1556
1557         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1558         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1559         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1560
1561         nw_off = inner_ip_off - ETH_HLEN;
1562         skb_set_network_header(skb, nw_off);
1563         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1564                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1565         skb_set_transport_header(skb, nw_off + iphdr_len);
1566
1567         if (inner_mac_off) { /* tunnel */
1568                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1569                                             ETH_HLEN - 2));
1570
1571                 bnxt_gro_tunnel(skb, proto);
1572         }
1573 #endif
1574         return skb;
1575 }
1576
1577 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1578 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1579
1580 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1581                                            int payload_off, int tcp_ts,
1582                                            struct sk_buff *skb)
1583 {
1584 #ifdef CONFIG_INET
1585         struct tcphdr *th;
1586         int len, nw_off, tcp_opt_len = 0;
1587
1588         if (tcp_ts)
1589                 tcp_opt_len = 12;
1590
1591         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1592                 struct iphdr *iph;
1593
1594                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1595                          ETH_HLEN;
1596                 skb_set_network_header(skb, nw_off);
1597                 iph = ip_hdr(skb);
1598                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1599                 len = skb->len - skb_transport_offset(skb);
1600                 th = tcp_hdr(skb);
1601                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1602         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1603                 struct ipv6hdr *iph;
1604
1605                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1606                          ETH_HLEN;
1607                 skb_set_network_header(skb, nw_off);
1608                 iph = ipv6_hdr(skb);
1609                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1610                 len = skb->len - skb_transport_offset(skb);
1611                 th = tcp_hdr(skb);
1612                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1613         } else {
1614                 dev_kfree_skb_any(skb);
1615                 return NULL;
1616         }
1617
1618         if (nw_off) /* tunnel */
1619                 bnxt_gro_tunnel(skb, skb->protocol);
1620 #endif
1621         return skb;
1622 }
1623
1624 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1625                                            struct bnxt_tpa_info *tpa_info,
1626                                            struct rx_tpa_end_cmp *tpa_end,
1627                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1628                                            struct sk_buff *skb)
1629 {
1630 #ifdef CONFIG_INET
1631         int payload_off;
1632         u16 segs;
1633
1634         segs = TPA_END_TPA_SEGS(tpa_end);
1635         if (segs == 1)
1636                 return skb;
1637
1638         NAPI_GRO_CB(skb)->count = segs;
1639         skb_shinfo(skb)->gso_size =
1640                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1641         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1642         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
1643                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1644         else
1645                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1646         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1647         if (likely(skb))
1648                 tcp_gro_complete(skb);
1649 #endif
1650         return skb;
1651 }
1652
1653 /* Given the cfa_code of a received packet determine which
1654  * netdev (vf-rep or PF) the packet is destined to.
1655  */
1656 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1657 {
1658         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1659
1660         /* if vf-rep dev is NULL, the must belongs to the PF */
1661         return dev ? dev : bp->dev;
1662 }
1663
1664 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1665                                            struct bnxt_cp_ring_info *cpr,
1666                                            u32 *raw_cons,
1667                                            struct rx_tpa_end_cmp *tpa_end,
1668                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1669                                            u8 *event)
1670 {
1671         struct bnxt_napi *bnapi = cpr->bnapi;
1672         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1673         struct net_device *dev = bp->dev;
1674         u8 *data_ptr, agg_bufs;
1675         unsigned int len;
1676         struct bnxt_tpa_info *tpa_info;
1677         dma_addr_t mapping;
1678         struct sk_buff *skb;
1679         u16 idx = 0, agg_id;
1680         void *data;
1681         bool gro;
1682
1683         if (unlikely(bnapi->in_reset)) {
1684                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1685
1686                 if (rc < 0)
1687                         return ERR_PTR(-EBUSY);
1688                 return NULL;
1689         }
1690
1691         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
1692                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1693                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1694                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1695                 tpa_info = &rxr->rx_tpa[agg_id];
1696                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1697                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1698                                     agg_bufs, tpa_info->agg_count);
1699                         agg_bufs = tpa_info->agg_count;
1700                 }
1701                 tpa_info->agg_count = 0;
1702                 *event |= BNXT_AGG_EVENT;
1703                 bnxt_free_agg_idx(rxr, agg_id);
1704                 idx = agg_id;
1705                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1706         } else {
1707                 agg_id = TPA_END_AGG_ID(tpa_end);
1708                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1709                 tpa_info = &rxr->rx_tpa[agg_id];
1710                 idx = RING_CMP(*raw_cons);
1711                 if (agg_bufs) {
1712                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1713                                 return ERR_PTR(-EBUSY);
1714
1715                         *event |= BNXT_AGG_EVENT;
1716                         idx = NEXT_CMP(idx);
1717                 }
1718                 gro = !!TPA_END_GRO(tpa_end);
1719         }
1720         data = tpa_info->data;
1721         data_ptr = tpa_info->data_ptr;
1722         prefetch(data_ptr);
1723         len = tpa_info->len;
1724         mapping = tpa_info->mapping;
1725
1726         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1727                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1728                 if (agg_bufs > MAX_SKB_FRAGS)
1729                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1730                                     agg_bufs, (int)MAX_SKB_FRAGS);
1731                 return NULL;
1732         }
1733
1734         if (len <= bp->rx_copy_thresh) {
1735                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1736                 if (!skb) {
1737                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1738                         cpr->sw_stats.rx.rx_oom_discards += 1;
1739                         return NULL;
1740                 }
1741         } else {
1742                 u8 *new_data;
1743                 dma_addr_t new_mapping;
1744
1745                 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC);
1746                 if (!new_data) {
1747                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1748                         cpr->sw_stats.rx.rx_oom_discards += 1;
1749                         return NULL;
1750                 }
1751
1752                 tpa_info->data = new_data;
1753                 tpa_info->data_ptr = new_data + bp->rx_offset;
1754                 tpa_info->mapping = new_mapping;
1755
1756                 skb = napi_build_skb(data, bp->rx_buf_size);
1757                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1758                                        bp->rx_buf_use_size, bp->rx_dir,
1759                                        DMA_ATTR_WEAK_ORDERING);
1760
1761                 if (!skb) {
1762                         skb_free_frag(data);
1763                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1764                         cpr->sw_stats.rx.rx_oom_discards += 1;
1765                         return NULL;
1766                 }
1767                 skb_reserve(skb, bp->rx_offset);
1768                 skb_put(skb, len);
1769         }
1770
1771         if (agg_bufs) {
1772                 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true);
1773                 if (!skb) {
1774                         /* Page reuse already handled by bnxt_rx_pages(). */
1775                         cpr->sw_stats.rx.rx_oom_discards += 1;
1776                         return NULL;
1777                 }
1778         }
1779
1780         if (tpa_info->cfa_code_valid)
1781                 dev = bnxt_get_pkt_dev(bp, tpa_info->cfa_code);
1782         skb->protocol = eth_type_trans(skb, dev);
1783
1784         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1785                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1786
1787         if (tpa_info->vlan_valid &&
1788             (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1789                 __be16 vlan_proto = htons(tpa_info->metadata >>
1790                                           RX_CMP_FLAGS2_METADATA_TPID_SFT);
1791                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1792
1793                 if (eth_type_vlan(vlan_proto)) {
1794                         __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1795                 } else {
1796                         dev_kfree_skb(skb);
1797                         return NULL;
1798                 }
1799         }
1800
1801         skb_checksum_none_assert(skb);
1802         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1803                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1804                 skb->csum_level =
1805                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1806         }
1807
1808         if (gro)
1809                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1810
1811         return skb;
1812 }
1813
1814 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1815                          struct rx_agg_cmp *rx_agg)
1816 {
1817         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1818         struct bnxt_tpa_info *tpa_info;
1819
1820         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1821         tpa_info = &rxr->rx_tpa[agg_id];
1822         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1823         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1824 }
1825
1826 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1827                              struct sk_buff *skb)
1828 {
1829         skb_mark_for_recycle(skb);
1830
1831         if (skb->dev != bp->dev) {
1832                 /* this packet belongs to a vf-rep */
1833                 bnxt_vf_rep_rx(bp, skb);
1834                 return;
1835         }
1836         skb_record_rx_queue(skb, bnapi->index);
1837         napi_gro_receive(&bnapi->napi, skb);
1838 }
1839
1840 static bool bnxt_rx_ts_valid(struct bnxt *bp, u32 flags,
1841                              struct rx_cmp_ext *rxcmp1, u32 *cmpl_ts)
1842 {
1843         u32 ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
1844
1845         if (BNXT_PTP_RX_TS_VALID(flags))
1846                 goto ts_valid;
1847         if (!bp->ptp_all_rx_tstamp || !ts || !BNXT_ALL_RX_TS_VALID(flags))
1848                 return false;
1849
1850 ts_valid:
1851         *cmpl_ts = ts;
1852         return true;
1853 }
1854
1855 static struct sk_buff *bnxt_rx_vlan(struct sk_buff *skb, u8 cmp_type,
1856                                     struct rx_cmp *rxcmp,
1857                                     struct rx_cmp_ext *rxcmp1)
1858 {
1859         __be16 vlan_proto;
1860         u16 vtag;
1861
1862         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1863                 __le32 flags2 = rxcmp1->rx_cmp_flags2;
1864                 u32 meta_data;
1865
1866                 if (!(flags2 & cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)))
1867                         return skb;
1868
1869                 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1870                 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1871                 vlan_proto = htons(meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT);
1872                 if (eth_type_vlan(vlan_proto))
1873                         __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1874                 else
1875                         goto vlan_err;
1876         } else if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
1877                 if (RX_CMP_VLAN_VALID(rxcmp)) {
1878                         u32 tpid_sel = RX_CMP_VLAN_TPID_SEL(rxcmp);
1879
1880                         if (tpid_sel == RX_CMP_METADATA1_TPID_8021Q)
1881                                 vlan_proto = htons(ETH_P_8021Q);
1882                         else if (tpid_sel == RX_CMP_METADATA1_TPID_8021AD)
1883                                 vlan_proto = htons(ETH_P_8021AD);
1884                         else
1885                                 goto vlan_err;
1886                         vtag = RX_CMP_METADATA0_TCI(rxcmp1);
1887                         __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1888                 }
1889         }
1890         return skb;
1891 vlan_err:
1892         dev_kfree_skb(skb);
1893         return NULL;
1894 }
1895
1896 static enum pkt_hash_types bnxt_rss_ext_op(struct bnxt *bp,
1897                                            struct rx_cmp *rxcmp)
1898 {
1899         u8 ext_op;
1900
1901         ext_op = RX_CMP_V3_HASH_TYPE(bp, rxcmp);
1902         switch (ext_op) {
1903         case EXT_OP_INNER_4:
1904         case EXT_OP_OUTER_4:
1905         case EXT_OP_INNFL_3:
1906         case EXT_OP_OUTFL_3:
1907                 return PKT_HASH_TYPE_L4;
1908         default:
1909                 return PKT_HASH_TYPE_L3;
1910         }
1911 }
1912
1913 /* returns the following:
1914  * 1       - 1 packet successfully received
1915  * 0       - successful TPA_START, packet not completed yet
1916  * -EBUSY  - completion ring does not have all the agg buffers yet
1917  * -ENOMEM - packet aborted due to out of memory
1918  * -EIO    - packet aborted due to hw error indicated in BD
1919  */
1920 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1921                        u32 *raw_cons, u8 *event)
1922 {
1923         struct bnxt_napi *bnapi = cpr->bnapi;
1924         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1925         struct net_device *dev = bp->dev;
1926         struct rx_cmp *rxcmp;
1927         struct rx_cmp_ext *rxcmp1;
1928         u32 tmp_raw_cons = *raw_cons;
1929         u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1930         struct bnxt_sw_rx_bd *rx_buf;
1931         unsigned int len;
1932         u8 *data_ptr, agg_bufs, cmp_type;
1933         bool xdp_active = false;
1934         dma_addr_t dma_addr;
1935         struct sk_buff *skb;
1936         struct xdp_buff xdp;
1937         u32 flags, misc;
1938         u32 cmpl_ts;
1939         void *data;
1940         int rc = 0;
1941
1942         rxcmp = (struct rx_cmp *)
1943                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1944
1945         cmp_type = RX_CMP_TYPE(rxcmp);
1946
1947         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1948                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1949                 goto next_rx_no_prod_no_len;
1950         }
1951
1952         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1953         cp_cons = RING_CMP(tmp_raw_cons);
1954         rxcmp1 = (struct rx_cmp_ext *)
1955                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1956
1957         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1958                 return -EBUSY;
1959
1960         /* The valid test of the entry must be done first before
1961          * reading any further.
1962          */
1963         dma_rmb();
1964         prod = rxr->rx_prod;
1965
1966         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP ||
1967             cmp_type == CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
1968                 bnxt_tpa_start(bp, rxr, cmp_type,
1969                                (struct rx_tpa_start_cmp *)rxcmp,
1970                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1971
1972                 *event |= BNXT_RX_EVENT;
1973                 goto next_rx_no_prod_no_len;
1974
1975         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1976                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1977                                    (struct rx_tpa_end_cmp *)rxcmp,
1978                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1979
1980                 if (IS_ERR(skb))
1981                         return -EBUSY;
1982
1983                 rc = -ENOMEM;
1984                 if (likely(skb)) {
1985                         bnxt_deliver_skb(bp, bnapi, skb);
1986                         rc = 1;
1987                 }
1988                 *event |= BNXT_RX_EVENT;
1989                 goto next_rx_no_prod_no_len;
1990         }
1991
1992         cons = rxcmp->rx_cmp_opaque;
1993         if (unlikely(cons != rxr->rx_next_cons)) {
1994                 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
1995
1996                 /* 0xffff is forced error, don't print it */
1997                 if (rxr->rx_next_cons != 0xffff)
1998                         netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1999                                     cons, rxr->rx_next_cons);
2000                 bnxt_sched_reset_rxr(bp, rxr);
2001                 if (rc1)
2002                         return rc1;
2003                 goto next_rx_no_prod_no_len;
2004         }
2005         rx_buf = &rxr->rx_buf_ring[cons];
2006         data = rx_buf->data;
2007         data_ptr = rx_buf->data_ptr;
2008         prefetch(data_ptr);
2009
2010         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
2011         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
2012
2013         if (agg_bufs) {
2014                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
2015                         return -EBUSY;
2016
2017                 cp_cons = NEXT_CMP(cp_cons);
2018                 *event |= BNXT_AGG_EVENT;
2019         }
2020         *event |= BNXT_RX_EVENT;
2021
2022         rx_buf->data = NULL;
2023         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
2024                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
2025
2026                 bnxt_reuse_rx_data(rxr, cons, data);
2027                 if (agg_bufs)
2028                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
2029                                                false);
2030
2031                 rc = -EIO;
2032                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
2033                         bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
2034                         if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
2035                             !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
2036                                 netdev_warn_once(bp->dev, "RX buffer error %x\n",
2037                                                  rx_err);
2038                                 bnxt_sched_reset_rxr(bp, rxr);
2039                         }
2040                 }
2041                 goto next_rx_no_len;
2042         }
2043
2044         flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
2045         len = flags >> RX_CMP_LEN_SHIFT;
2046         dma_addr = rx_buf->mapping;
2047
2048         if (bnxt_xdp_attached(bp, rxr)) {
2049                 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp);
2050                 if (agg_bufs) {
2051                         u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp,
2052                                                              cp_cons, agg_bufs,
2053                                                              false);
2054                         if (!frag_len) {
2055                                 cpr->sw_stats.rx.rx_oom_discards += 1;
2056                                 rc = -ENOMEM;
2057                                 goto next_rx;
2058                         }
2059                 }
2060                 xdp_active = true;
2061         }
2062
2063         if (xdp_active) {
2064                 if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &data_ptr, &len, event)) {
2065                         rc = 1;
2066                         goto next_rx;
2067                 }
2068         }
2069
2070         if (len <= bp->rx_copy_thresh) {
2071                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
2072                 bnxt_reuse_rx_data(rxr, cons, data);
2073                 if (!skb) {
2074                         if (agg_bufs) {
2075                                 if (!xdp_active)
2076                                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
2077                                                                agg_bufs, false);
2078                                 else
2079                                         bnxt_xdp_buff_frags_free(rxr, &xdp);
2080                         }
2081                         cpr->sw_stats.rx.rx_oom_discards += 1;
2082                         rc = -ENOMEM;
2083                         goto next_rx;
2084                 }
2085         } else {
2086                 u32 payload;
2087
2088                 if (rx_buf->data_ptr == data_ptr)
2089                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
2090                 else
2091                         payload = 0;
2092                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
2093                                       payload | len);
2094                 if (!skb) {
2095                         cpr->sw_stats.rx.rx_oom_discards += 1;
2096                         rc = -ENOMEM;
2097                         goto next_rx;
2098                 }
2099         }
2100
2101         if (agg_bufs) {
2102                 if (!xdp_active) {
2103                         skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false);
2104                         if (!skb) {
2105                                 cpr->sw_stats.rx.rx_oom_discards += 1;
2106                                 rc = -ENOMEM;
2107                                 goto next_rx;
2108                         }
2109                 } else {
2110                         skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);
2111                         if (!skb) {
2112                                 /* we should be able to free the old skb here */
2113                                 bnxt_xdp_buff_frags_free(rxr, &xdp);
2114                                 cpr->sw_stats.rx.rx_oom_discards += 1;
2115                                 rc = -ENOMEM;
2116                                 goto next_rx;
2117                         }
2118                 }
2119         }
2120
2121         if (RX_CMP_HASH_VALID(rxcmp)) {
2122                 enum pkt_hash_types type;
2123
2124                 if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
2125                         type = bnxt_rss_ext_op(bp, rxcmp);
2126                 } else {
2127                         u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
2128
2129                         /* RSS profiles 1 and 3 with extract code 0 for inner
2130                          * 4-tuple
2131                          */
2132                         if (hash_type != 1 && hash_type != 3)
2133                                 type = PKT_HASH_TYPE_L3;
2134                         else
2135                                 type = PKT_HASH_TYPE_L4;
2136                 }
2137                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
2138         }
2139
2140         if (cmp_type == CMP_TYPE_RX_L2_CMP)
2141                 dev = bnxt_get_pkt_dev(bp, RX_CMP_CFA_CODE(rxcmp1));
2142         skb->protocol = eth_type_trans(skb, dev);
2143
2144         if (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) {
2145                 skb = bnxt_rx_vlan(skb, cmp_type, rxcmp, rxcmp1);
2146                 if (!skb)
2147                         goto next_rx;
2148         }
2149
2150         skb_checksum_none_assert(skb);
2151         if (RX_CMP_L4_CS_OK(rxcmp1)) {
2152                 if (dev->features & NETIF_F_RXCSUM) {
2153                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2154                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
2155                 }
2156         } else {
2157                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
2158                         if (dev->features & NETIF_F_RXCSUM)
2159                                 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
2160                 }
2161         }
2162
2163         if (bnxt_rx_ts_valid(bp, flags, rxcmp1, &cmpl_ts)) {
2164                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
2165                         u64 ns, ts;
2166
2167                         if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
2168                                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2169
2170                                 spin_lock_bh(&ptp->ptp_lock);
2171                                 ns = timecounter_cyc2time(&ptp->tc, ts);
2172                                 spin_unlock_bh(&ptp->ptp_lock);
2173                                 memset(skb_hwtstamps(skb), 0,
2174                                        sizeof(*skb_hwtstamps(skb)));
2175                                 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
2176                         }
2177                 }
2178         }
2179         bnxt_deliver_skb(bp, bnapi, skb);
2180         rc = 1;
2181
2182 next_rx:
2183         cpr->rx_packets += 1;
2184         cpr->rx_bytes += len;
2185
2186 next_rx_no_len:
2187         rxr->rx_prod = NEXT_RX(prod);
2188         rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons));
2189
2190 next_rx_no_prod_no_len:
2191         *raw_cons = tmp_raw_cons;
2192
2193         return rc;
2194 }
2195
2196 /* In netpoll mode, if we are using a combined completion ring, we need to
2197  * discard the rx packets and recycle the buffers.
2198  */
2199 static int bnxt_force_rx_discard(struct bnxt *bp,
2200                                  struct bnxt_cp_ring_info *cpr,
2201                                  u32 *raw_cons, u8 *event)
2202 {
2203         u32 tmp_raw_cons = *raw_cons;
2204         struct rx_cmp_ext *rxcmp1;
2205         struct rx_cmp *rxcmp;
2206         u16 cp_cons;
2207         u8 cmp_type;
2208         int rc;
2209
2210         cp_cons = RING_CMP(tmp_raw_cons);
2211         rxcmp = (struct rx_cmp *)
2212                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2213
2214         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2215         cp_cons = RING_CMP(tmp_raw_cons);
2216         rxcmp1 = (struct rx_cmp_ext *)
2217                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2218
2219         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2220                 return -EBUSY;
2221
2222         /* The valid test of the entry must be done first before
2223          * reading any further.
2224          */
2225         dma_rmb();
2226         cmp_type = RX_CMP_TYPE(rxcmp);
2227         if (cmp_type == CMP_TYPE_RX_L2_CMP ||
2228             cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
2229                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2230                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2231         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2232                 struct rx_tpa_end_cmp_ext *tpa_end1;
2233
2234                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2235                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2236                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2237         }
2238         rc = bnxt_rx_pkt(bp, cpr, raw_cons, event);
2239         if (rc && rc != -EBUSY)
2240                 cpr->sw_stats.rx.rx_netpoll_discards += 1;
2241         return rc;
2242 }
2243
2244 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2245 {
2246         struct bnxt_fw_health *fw_health = bp->fw_health;
2247         u32 reg = fw_health->regs[reg_idx];
2248         u32 reg_type, reg_off, val = 0;
2249
2250         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2251         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2252         switch (reg_type) {
2253         case BNXT_FW_HEALTH_REG_TYPE_CFG:
2254                 pci_read_config_dword(bp->pdev, reg_off, &val);
2255                 break;
2256         case BNXT_FW_HEALTH_REG_TYPE_GRC:
2257                 reg_off = fw_health->mapped_regs[reg_idx];
2258                 fallthrough;
2259         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2260                 val = readl(bp->bar0 + reg_off);
2261                 break;
2262         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2263                 val = readl(bp->bar1 + reg_off);
2264                 break;
2265         }
2266         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2267                 val &= fw_health->fw_reset_inprog_reg_mask;
2268         return val;
2269 }
2270
2271 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2272 {
2273         int i;
2274
2275         for (i = 0; i < bp->rx_nr_rings; i++) {
2276                 u16 grp_idx = bp->rx_ring[i].bnapi->index;
2277                 struct bnxt_ring_grp_info *grp_info;
2278
2279                 grp_info = &bp->grp_info[grp_idx];
2280                 if (grp_info->agg_fw_ring_id == ring_id)
2281                         return grp_idx;
2282         }
2283         return INVALID_HW_RING_ID;
2284 }
2285
2286 static u16 bnxt_get_force_speed(struct bnxt_link_info *link_info)
2287 {
2288         struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2289
2290         if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2)
2291                 return link_info->force_link_speed2;
2292         if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4)
2293                 return link_info->force_pam4_link_speed;
2294         return link_info->force_link_speed;
2295 }
2296
2297 static void bnxt_set_force_speed(struct bnxt_link_info *link_info)
2298 {
2299         struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2300
2301         if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2302                 link_info->req_link_speed = link_info->force_link_speed2;
2303                 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2304                 switch (link_info->req_link_speed) {
2305                 case BNXT_LINK_SPEED_50GB_PAM4:
2306                 case BNXT_LINK_SPEED_100GB_PAM4:
2307                 case BNXT_LINK_SPEED_200GB_PAM4:
2308                 case BNXT_LINK_SPEED_400GB_PAM4:
2309                         link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
2310                         break;
2311                 case BNXT_LINK_SPEED_100GB_PAM4_112:
2312                 case BNXT_LINK_SPEED_200GB_PAM4_112:
2313                 case BNXT_LINK_SPEED_400GB_PAM4_112:
2314                         link_info->req_signal_mode = BNXT_SIG_MODE_PAM4_112;
2315                         break;
2316                 default:
2317                         link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2318                 }
2319                 return;
2320         }
2321         link_info->req_link_speed = link_info->force_link_speed;
2322         link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2323         if (link_info->force_pam4_link_speed) {
2324                 link_info->req_link_speed = link_info->force_pam4_link_speed;
2325                 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
2326         }
2327 }
2328
2329 static void bnxt_set_auto_speed(struct bnxt_link_info *link_info)
2330 {
2331         struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2332
2333         if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2334                 link_info->advertising = link_info->auto_link_speeds2;
2335                 return;
2336         }
2337         link_info->advertising = link_info->auto_link_speeds;
2338         link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
2339 }
2340
2341 static bool bnxt_force_speed_updated(struct bnxt_link_info *link_info)
2342 {
2343         struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2344
2345         if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2346                 if (link_info->req_link_speed != link_info->force_link_speed2)
2347                         return true;
2348                 return false;
2349         }
2350         if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
2351             link_info->req_link_speed != link_info->force_link_speed)
2352                 return true;
2353         if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
2354             link_info->req_link_speed != link_info->force_pam4_link_speed)
2355                 return true;
2356         return false;
2357 }
2358
2359 static bool bnxt_auto_speed_updated(struct bnxt_link_info *link_info)
2360 {
2361         struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2362
2363         if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2364                 if (link_info->advertising != link_info->auto_link_speeds2)
2365                         return true;
2366                 return false;
2367         }
2368         if (link_info->advertising != link_info->auto_link_speeds ||
2369             link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
2370                 return true;
2371         return false;
2372 }
2373
2374 #define BNXT_EVENT_THERMAL_CURRENT_TEMP(data2)                          \
2375         ((data2) &                                                      \
2376           ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK)
2377
2378 #define BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2)                        \
2379         (((data2) &                                                     \
2380           ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_MASK) >>\
2381          ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_SFT)
2382
2383 #define EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1)                       \
2384         ((data1) &                                                      \
2385          ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_MASK)
2386
2387 #define EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1)             \
2388         (((data1) &                                                     \
2389           ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR) ==\
2390          ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING)
2391
2392 /* Return true if the workqueue has to be scheduled */
2393 static bool bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2)
2394 {
2395         u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1);
2396
2397         switch (err_type) {
2398         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL:
2399                 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n",
2400                            BNXT_EVENT_INVALID_SIGNAL_DATA(data2));
2401                 break;
2402         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM:
2403                 netdev_warn(bp->dev, "Pause Storm detected!\n");
2404                 break;
2405         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD:
2406                 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n");
2407                 break;
2408         case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_THERMAL_THRESHOLD: {
2409                 u32 type = EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1);
2410                 char *threshold_type;
2411                 bool notify = false;
2412                 char *dir_str;
2413
2414                 switch (type) {
2415                 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_WARN:
2416                         threshold_type = "warning";
2417                         break;
2418                 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_CRITICAL:
2419                         threshold_type = "critical";
2420                         break;
2421                 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_FATAL:
2422                         threshold_type = "fatal";
2423                         break;
2424                 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SHUTDOWN:
2425                         threshold_type = "shutdown";
2426                         break;
2427                 default:
2428                         netdev_err(bp->dev, "Unknown Thermal threshold type event\n");
2429                         return false;
2430                 }
2431                 if (EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1)) {
2432                         dir_str = "above";
2433                         notify = true;
2434                 } else {
2435                         dir_str = "below";
2436                 }
2437                 netdev_warn(bp->dev, "Chip temperature has gone %s the %s thermal threshold!\n",
2438                             dir_str, threshold_type);
2439                 netdev_warn(bp->dev, "Temperature (In Celsius), Current: %lu, threshold: %lu\n",
2440                             BNXT_EVENT_THERMAL_CURRENT_TEMP(data2),
2441                             BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2));
2442                 if (notify) {
2443                         bp->thermal_threshold_type = type;
2444                         set_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event);
2445                         return true;
2446                 }
2447                 return false;
2448         }
2449         default:
2450                 netdev_err(bp->dev, "FW reported unknown error type %u\n",
2451                            err_type);
2452                 break;
2453         }
2454         return false;
2455 }
2456
2457 #define BNXT_GET_EVENT_PORT(data)       \
2458         ((data) &                       \
2459          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2460
2461 #define BNXT_EVENT_RING_TYPE(data2)     \
2462         ((data2) &                      \
2463          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2464
2465 #define BNXT_EVENT_RING_TYPE_RX(data2)  \
2466         (BNXT_EVENT_RING_TYPE(data2) == \
2467          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2468
2469 #define BNXT_EVENT_PHC_EVENT_TYPE(data1)        \
2470         (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\
2471          ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT)
2472
2473 #define BNXT_EVENT_PHC_RTC_UPDATE(data1)        \
2474         (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\
2475          ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT)
2476
2477 #define BNXT_PHC_BITS   48
2478
2479 static int bnxt_async_event_process(struct bnxt *bp,
2480                                     struct hwrm_async_event_cmpl *cmpl)
2481 {
2482         u16 event_id = le16_to_cpu(cmpl->event_id);
2483         u32 data1 = le32_to_cpu(cmpl->event_data1);
2484         u32 data2 = le32_to_cpu(cmpl->event_data2);
2485
2486         netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n",
2487                    event_id, data1, data2);
2488
2489         /* TODO CHIMP_FW: Define event id's for link change, error etc */
2490         switch (event_id) {
2491         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2492                 struct bnxt_link_info *link_info = &bp->link_info;
2493
2494                 if (BNXT_VF(bp))
2495                         goto async_event_process_exit;
2496
2497                 /* print unsupported speed warning in forced speed mode only */
2498                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2499                     (data1 & 0x20000)) {
2500                         u16 fw_speed = bnxt_get_force_speed(link_info);
2501                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2502
2503                         if (speed != SPEED_UNKNOWN)
2504                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2505                                             speed);
2506                 }
2507                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2508         }
2509                 fallthrough;
2510         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2511         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2512                 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2513                 fallthrough;
2514         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2515                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2516                 break;
2517         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2518                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2519                 break;
2520         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2521                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2522
2523                 if (BNXT_VF(bp))
2524                         break;
2525
2526                 if (bp->pf.port_id != port_id)
2527                         break;
2528
2529                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2530                 break;
2531         }
2532         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2533                 if (BNXT_PF(bp))
2534                         goto async_event_process_exit;
2535                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2536                 break;
2537         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2538                 char *type_str = "Solicited";
2539
2540                 if (!bp->fw_health)
2541                         goto async_event_process_exit;
2542
2543                 bp->fw_reset_timestamp = jiffies;
2544                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2545                 if (!bp->fw_reset_min_dsecs)
2546                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2547                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2548                 if (!bp->fw_reset_max_dsecs)
2549                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2550                 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) {
2551                         set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state);
2552                 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2553                         type_str = "Fatal";
2554                         bp->fw_health->fatalities++;
2555                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2556                 } else if (data2 && BNXT_FW_STATUS_HEALTHY !=
2557                            EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) {
2558                         type_str = "Non-fatal";
2559                         bp->fw_health->survivals++;
2560                         set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
2561                 }
2562                 netif_warn(bp, hw, bp->dev,
2563                            "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2564                            type_str, data1, data2,
2565                            bp->fw_reset_min_dsecs * 100,
2566                            bp->fw_reset_max_dsecs * 100);
2567                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2568                 break;
2569         }
2570         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2571                 struct bnxt_fw_health *fw_health = bp->fw_health;
2572                 char *status_desc = "healthy";
2573                 u32 status;
2574
2575                 if (!fw_health)
2576                         goto async_event_process_exit;
2577
2578                 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) {
2579                         fw_health->enabled = false;
2580                         netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n");
2581                         break;
2582                 }
2583                 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2584                 fw_health->tmr_multiplier =
2585                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2586                                      bp->current_interval * 10);
2587                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2588                 if (!fw_health->enabled)
2589                         fw_health->last_fw_heartbeat =
2590                                 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2591                 fw_health->last_fw_reset_cnt =
2592                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2593                 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
2594                 if (status != BNXT_FW_STATUS_HEALTHY)
2595                         status_desc = "unhealthy";
2596                 netif_info(bp, drv, bp->dev,
2597                            "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n",
2598                            fw_health->primary ? "primary" : "backup", status,
2599                            status_desc, fw_health->last_fw_reset_cnt);
2600                 if (!fw_health->enabled) {
2601                         /* Make sure tmr_counter is set and visible to
2602                          * bnxt_health_check() before setting enabled to true.
2603                          */
2604                         smp_wmb();
2605                         fw_health->enabled = true;
2606                 }
2607                 goto async_event_process_exit;
2608         }
2609         case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2610                 netif_notice(bp, hw, bp->dev,
2611                              "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2612                              data1, data2);
2613                 goto async_event_process_exit;
2614         case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2615                 struct bnxt_rx_ring_info *rxr;
2616                 u16 grp_idx;
2617
2618                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
2619                         goto async_event_process_exit;
2620
2621                 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2622                             BNXT_EVENT_RING_TYPE(data2), data1);
2623                 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2624                         goto async_event_process_exit;
2625
2626                 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2627                 if (grp_idx == INVALID_HW_RING_ID) {
2628                         netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2629                                     data1);
2630                         goto async_event_process_exit;
2631                 }
2632                 rxr = bp->bnapi[grp_idx]->rx_ring;
2633                 bnxt_sched_reset_rxr(bp, rxr);
2634                 goto async_event_process_exit;
2635         }
2636         case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2637                 struct bnxt_fw_health *fw_health = bp->fw_health;
2638
2639                 netif_notice(bp, hw, bp->dev,
2640                              "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2641                              data1, data2);
2642                 if (fw_health) {
2643                         fw_health->echo_req_data1 = data1;
2644                         fw_health->echo_req_data2 = data2;
2645                         set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2646                         break;
2647                 }
2648                 goto async_event_process_exit;
2649         }
2650         case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: {
2651                 bnxt_ptp_pps_event(bp, data1, data2);
2652                 goto async_event_process_exit;
2653         }
2654         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: {
2655                 if (bnxt_event_error_report(bp, data1, data2))
2656                         break;
2657                 goto async_event_process_exit;
2658         }
2659         case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
2660                 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
2661                 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
2662                         if (BNXT_PTP_USE_RTC(bp)) {
2663                                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2664                                 u64 ns;
2665
2666                                 if (!ptp)
2667                                         goto async_event_process_exit;
2668
2669                                 spin_lock_bh(&ptp->ptp_lock);
2670                                 bnxt_ptp_update_current_time(bp);
2671                                 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) <<
2672                                        BNXT_PHC_BITS) | ptp->current_time);
2673                                 bnxt_ptp_rtc_timecounter_init(ptp, ns);
2674                                 spin_unlock_bh(&ptp->ptp_lock);
2675                         }
2676                         break;
2677                 }
2678                 goto async_event_process_exit;
2679         }
2680         case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: {
2681                 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff;
2682
2683                 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED);
2684                 goto async_event_process_exit;
2685         }
2686         default:
2687                 goto async_event_process_exit;
2688         }
2689         __bnxt_queue_sp_work(bp);
2690 async_event_process_exit:
2691         return 0;
2692 }
2693
2694 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2695 {
2696         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2697         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2698         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2699                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2700
2701         switch (cmpl_type) {
2702         case CMPL_BASE_TYPE_HWRM_DONE:
2703                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2704                 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE);
2705                 break;
2706
2707         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2708                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2709
2710                 if ((vf_id < bp->pf.first_vf_id) ||
2711                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2712                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2713                                    vf_id);
2714                         return -EINVAL;
2715                 }
2716
2717                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2718                 bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT);
2719                 break;
2720
2721         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2722                 bnxt_async_event_process(bp,
2723                                          (struct hwrm_async_event_cmpl *)txcmp);
2724                 break;
2725
2726         default:
2727                 break;
2728         }
2729
2730         return 0;
2731 }
2732
2733 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2734 {
2735         struct bnxt_napi *bnapi = dev_instance;
2736         struct bnxt *bp = bnapi->bp;
2737         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2738         u32 cons = RING_CMP(cpr->cp_raw_cons);
2739
2740         cpr->event_ctr++;
2741         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2742         napi_schedule(&bnapi->napi);
2743         return IRQ_HANDLED;
2744 }
2745
2746 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2747 {
2748         u32 raw_cons = cpr->cp_raw_cons;
2749         u16 cons = RING_CMP(raw_cons);
2750         struct tx_cmp *txcmp;
2751
2752         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2753
2754         return TX_CMP_VALID(txcmp, raw_cons);
2755 }
2756
2757 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2758 {
2759         struct bnxt_napi *bnapi = dev_instance;
2760         struct bnxt *bp = bnapi->bp;
2761         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2762         u32 cons = RING_CMP(cpr->cp_raw_cons);
2763         u32 int_status;
2764
2765         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2766
2767         if (!bnxt_has_work(bp, cpr)) {
2768                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2769                 /* return if erroneous interrupt */
2770                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2771                         return IRQ_NONE;
2772         }
2773
2774         /* disable ring IRQ */
2775         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2776
2777         /* Return here if interrupt is shared and is disabled. */
2778         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2779                 return IRQ_HANDLED;
2780
2781         napi_schedule(&bnapi->napi);
2782         return IRQ_HANDLED;
2783 }
2784
2785 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2786                             int budget)
2787 {
2788         struct bnxt_napi *bnapi = cpr->bnapi;
2789         u32 raw_cons = cpr->cp_raw_cons;
2790         u32 cons;
2791         int rx_pkts = 0;
2792         u8 event = 0;
2793         struct tx_cmp *txcmp;
2794
2795         cpr->has_more_work = 0;
2796         cpr->had_work_done = 1;
2797         while (1) {
2798                 u8 cmp_type;
2799                 int rc;
2800
2801                 cons = RING_CMP(raw_cons);
2802                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2803
2804                 if (!TX_CMP_VALID(txcmp, raw_cons))
2805                         break;
2806
2807                 /* The valid test of the entry must be done first before
2808                  * reading any further.
2809                  */
2810                 dma_rmb();
2811                 cmp_type = TX_CMP_TYPE(txcmp);
2812                 if (cmp_type == CMP_TYPE_TX_L2_CMP ||
2813                     cmp_type == CMP_TYPE_TX_L2_COAL_CMP) {
2814                         u32 opaque = txcmp->tx_cmp_opaque;
2815                         struct bnxt_tx_ring_info *txr;
2816                         u16 tx_freed;
2817
2818                         txr = bnapi->tx_ring[TX_OPAQUE_RING(opaque)];
2819                         event |= BNXT_TX_CMP_EVENT;
2820                         if (cmp_type == CMP_TYPE_TX_L2_COAL_CMP)
2821                                 txr->tx_hw_cons = TX_CMP_SQ_CONS_IDX(txcmp);
2822                         else
2823                                 txr->tx_hw_cons = TX_OPAQUE_PROD(bp, opaque);
2824                         tx_freed = (txr->tx_hw_cons - txr->tx_cons) &
2825                                    bp->tx_ring_mask;
2826                         /* return full budget so NAPI will complete. */
2827                         if (unlikely(tx_freed >= bp->tx_wake_thresh)) {
2828                                 rx_pkts = budget;
2829                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2830                                 if (budget)
2831                                         cpr->has_more_work = 1;
2832                                 break;
2833                         }
2834                 } else if (cmp_type >= CMP_TYPE_RX_L2_CMP &&
2835                            cmp_type <= CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
2836                         if (likely(budget))
2837                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2838                         else
2839                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2840                                                            &event);
2841                         if (likely(rc >= 0))
2842                                 rx_pkts += rc;
2843                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2844                          * the NAPI budget.  Otherwise, we may potentially loop
2845                          * here forever if we consistently cannot allocate
2846                          * buffers.
2847                          */
2848                         else if (rc == -ENOMEM && budget)
2849                                 rx_pkts++;
2850                         else if (rc == -EBUSY)  /* partial completion */
2851                                 break;
2852                 } else if (unlikely(cmp_type == CMPL_BASE_TYPE_HWRM_DONE ||
2853                                     cmp_type == CMPL_BASE_TYPE_HWRM_FWD_REQ ||
2854                                     cmp_type == CMPL_BASE_TYPE_HWRM_ASYNC_EVENT)) {
2855                         bnxt_hwrm_handler(bp, txcmp);
2856                 }
2857                 raw_cons = NEXT_RAW_CMP(raw_cons);
2858
2859                 if (rx_pkts && rx_pkts == budget) {
2860                         cpr->has_more_work = 1;
2861                         break;
2862                 }
2863         }
2864
2865         if (event & BNXT_REDIRECT_EVENT)
2866                 xdp_do_flush();
2867
2868         if (event & BNXT_TX_EVENT) {
2869                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0];
2870                 u16 prod = txr->tx_prod;
2871
2872                 /* Sync BD data before updating doorbell */
2873                 wmb();
2874
2875                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2876         }
2877
2878         cpr->cp_raw_cons = raw_cons;
2879         bnapi->events |= event;
2880         return rx_pkts;
2881 }
2882
2883 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2884                                   int budget)
2885 {
2886         if ((bnapi->events & BNXT_TX_CMP_EVENT) && !bnapi->tx_fault)
2887                 bnapi->tx_int(bp, bnapi, budget);
2888
2889         if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2890                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2891
2892                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2893         }
2894         if (bnapi->events & BNXT_AGG_EVENT) {
2895                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2896
2897                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2898         }
2899         bnapi->events &= BNXT_TX_CMP_EVENT;
2900 }
2901
2902 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2903                           int budget)
2904 {
2905         struct bnxt_napi *bnapi = cpr->bnapi;
2906         int rx_pkts;
2907
2908         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2909
2910         /* ACK completion ring before freeing tx ring and producing new
2911          * buffers in rx/agg rings to prevent overflowing the completion
2912          * ring.
2913          */
2914         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2915
2916         __bnxt_poll_work_done(bp, bnapi, budget);
2917         return rx_pkts;
2918 }
2919
2920 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2921 {
2922         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2923         struct bnxt *bp = bnapi->bp;
2924         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2925         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2926         struct tx_cmp *txcmp;
2927         struct rx_cmp_ext *rxcmp1;
2928         u32 cp_cons, tmp_raw_cons;
2929         u32 raw_cons = cpr->cp_raw_cons;
2930         bool flush_xdp = false;
2931         u32 rx_pkts = 0;
2932         u8 event = 0;
2933
2934         while (1) {
2935                 int rc;
2936
2937                 cp_cons = RING_CMP(raw_cons);
2938                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2939
2940                 if (!TX_CMP_VALID(txcmp, raw_cons))
2941                         break;
2942
2943                 /* The valid test of the entry must be done first before
2944                  * reading any further.
2945                  */
2946                 dma_rmb();
2947                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2948                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2949                         cp_cons = RING_CMP(tmp_raw_cons);
2950                         rxcmp1 = (struct rx_cmp_ext *)
2951                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2952
2953                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2954                                 break;
2955
2956                         /* force an error to recycle the buffer */
2957                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2958                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2959
2960                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2961                         if (likely(rc == -EIO) && budget)
2962                                 rx_pkts++;
2963                         else if (rc == -EBUSY)  /* partial completion */
2964                                 break;
2965                         if (event & BNXT_REDIRECT_EVENT)
2966                                 flush_xdp = true;
2967                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2968                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2969                         bnxt_hwrm_handler(bp, txcmp);
2970                 } else {
2971                         netdev_err(bp->dev,
2972                                    "Invalid completion received on special ring\n");
2973                 }
2974                 raw_cons = NEXT_RAW_CMP(raw_cons);
2975
2976                 if (rx_pkts == budget)
2977                         break;
2978         }
2979
2980         cpr->cp_raw_cons = raw_cons;
2981         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2982         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2983
2984         if (event & BNXT_AGG_EVENT)
2985                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2986         if (flush_xdp)
2987                 xdp_do_flush();
2988
2989         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2990                 napi_complete_done(napi, rx_pkts);
2991                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2992         }
2993         return rx_pkts;
2994 }
2995
2996 static int bnxt_poll(struct napi_struct *napi, int budget)
2997 {
2998         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2999         struct bnxt *bp = bnapi->bp;
3000         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3001         int work_done = 0;
3002
3003         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
3004                 napi_complete(napi);
3005                 return 0;
3006         }
3007         while (1) {
3008                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
3009
3010                 if (work_done >= budget) {
3011                         if (!budget)
3012                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3013                         break;
3014                 }
3015
3016                 if (!bnxt_has_work(bp, cpr)) {
3017                         if (napi_complete_done(napi, work_done))
3018                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3019                         break;
3020                 }
3021         }
3022         if (bp->flags & BNXT_FLAG_DIM) {
3023                 struct dim_sample dim_sample = {};
3024
3025                 dim_update_sample(cpr->event_ctr,
3026                                   cpr->rx_packets,
3027                                   cpr->rx_bytes,
3028                                   &dim_sample);
3029                 net_dim(&cpr->dim, dim_sample);
3030         }
3031         return work_done;
3032 }
3033
3034 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
3035 {
3036         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3037         int i, work_done = 0;
3038
3039         for (i = 0; i < cpr->cp_ring_count; i++) {
3040                 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
3041
3042                 if (cpr2->had_nqe_notify) {
3043                         work_done += __bnxt_poll_work(bp, cpr2,
3044                                                       budget - work_done);
3045                         cpr->has_more_work |= cpr2->has_more_work;
3046                 }
3047         }
3048         return work_done;
3049 }
3050
3051 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
3052                                  u64 dbr_type, int budget)
3053 {
3054         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3055         int i;
3056
3057         for (i = 0; i < cpr->cp_ring_count; i++) {
3058                 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
3059                 struct bnxt_db_info *db;
3060
3061                 if (cpr2->had_work_done) {
3062                         u32 tgl = 0;
3063
3064                         if (dbr_type == DBR_TYPE_CQ_ARMALL) {
3065                                 cpr2->had_nqe_notify = 0;
3066                                 tgl = cpr2->toggle;
3067                         }
3068                         db = &cpr2->cp_db;
3069                         bnxt_writeq(bp,
3070                                     db->db_key64 | dbr_type | DB_TOGGLE(tgl) |
3071                                     DB_RING_IDX(db, cpr2->cp_raw_cons),
3072                                     db->doorbell);
3073                         cpr2->had_work_done = 0;
3074                 }
3075         }
3076         __bnxt_poll_work_done(bp, bnapi, budget);
3077 }
3078
3079 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
3080 {
3081         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
3082         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3083         struct bnxt_cp_ring_info *cpr_rx;
3084         u32 raw_cons = cpr->cp_raw_cons;
3085         struct bnxt *bp = bnapi->bp;
3086         struct nqe_cn *nqcmp;
3087         int work_done = 0;
3088         u32 cons;
3089
3090         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
3091                 napi_complete(napi);
3092                 return 0;
3093         }
3094         if (cpr->has_more_work) {
3095                 cpr->has_more_work = 0;
3096                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
3097         }
3098         while (1) {
3099                 u16 type;
3100
3101                 cons = RING_CMP(raw_cons);
3102                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
3103
3104                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
3105                         if (cpr->has_more_work)
3106                                 break;
3107
3108                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
3109                                              budget);
3110                         cpr->cp_raw_cons = raw_cons;
3111                         if (napi_complete_done(napi, work_done))
3112                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
3113                                                   cpr->cp_raw_cons);
3114                         goto poll_done;
3115                 }
3116
3117                 /* The valid test of the entry must be done first before
3118                  * reading any further.
3119                  */
3120                 dma_rmb();
3121
3122                 type = le16_to_cpu(nqcmp->type);
3123                 if (NQE_CN_TYPE(type) == NQ_CN_TYPE_CQ_NOTIFICATION) {
3124                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
3125                         u32 cq_type = BNXT_NQ_HDL_TYPE(idx);
3126                         struct bnxt_cp_ring_info *cpr2;
3127
3128                         /* No more budget for RX work */
3129                         if (budget && work_done >= budget &&
3130                             cq_type == BNXT_NQ_HDL_TYPE_RX)
3131                                 break;
3132
3133                         idx = BNXT_NQ_HDL_IDX(idx);
3134                         cpr2 = &cpr->cp_ring_arr[idx];
3135                         cpr2->had_nqe_notify = 1;
3136                         cpr2->toggle = NQE_CN_TOGGLE(type);
3137                         work_done += __bnxt_poll_work(bp, cpr2,
3138                                                       budget - work_done);
3139                         cpr->has_more_work |= cpr2->has_more_work;
3140                 } else {
3141                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
3142                 }
3143                 raw_cons = NEXT_RAW_CMP(raw_cons);
3144         }
3145         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, budget);
3146         if (raw_cons != cpr->cp_raw_cons) {
3147                 cpr->cp_raw_cons = raw_cons;
3148                 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
3149         }
3150 poll_done:
3151         cpr_rx = &cpr->cp_ring_arr[0];
3152         if (cpr_rx->cp_ring_type == BNXT_NQ_HDL_TYPE_RX &&
3153             (bp->flags & BNXT_FLAG_DIM)) {
3154                 struct dim_sample dim_sample = {};
3155
3156                 dim_update_sample(cpr->event_ctr,
3157                                   cpr_rx->rx_packets,
3158                                   cpr_rx->rx_bytes,
3159                                   &dim_sample);
3160                 net_dim(&cpr->dim, dim_sample);
3161         }
3162         return work_done;
3163 }
3164
3165 static void bnxt_free_tx_skbs(struct bnxt *bp)
3166 {
3167         int i, max_idx;
3168         struct pci_dev *pdev = bp->pdev;
3169
3170         if (!bp->tx_ring)
3171                 return;
3172
3173         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
3174         for (i = 0; i < bp->tx_nr_rings; i++) {
3175                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3176                 int j;
3177
3178                 if (!txr->tx_buf_ring)
3179                         continue;
3180
3181                 for (j = 0; j < max_idx;) {
3182                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
3183                         struct sk_buff *skb;
3184                         int k, last;
3185
3186                         if (i < bp->tx_nr_rings_xdp &&
3187                             tx_buf->action == XDP_REDIRECT) {
3188                                 dma_unmap_single(&pdev->dev,
3189                                         dma_unmap_addr(tx_buf, mapping),
3190                                         dma_unmap_len(tx_buf, len),
3191                                         DMA_TO_DEVICE);
3192                                 xdp_return_frame(tx_buf->xdpf);
3193                                 tx_buf->action = 0;
3194                                 tx_buf->xdpf = NULL;
3195                                 j++;
3196                                 continue;
3197                         }
3198
3199                         skb = tx_buf->skb;
3200                         if (!skb) {
3201                                 j++;
3202                                 continue;
3203                         }
3204
3205                         tx_buf->skb = NULL;
3206
3207                         if (tx_buf->is_push) {
3208                                 dev_kfree_skb(skb);
3209                                 j += 2;
3210                                 continue;
3211                         }
3212
3213                         dma_unmap_single(&pdev->dev,
3214                                          dma_unmap_addr(tx_buf, mapping),
3215                                          skb_headlen(skb),
3216                                          DMA_TO_DEVICE);
3217
3218                         last = tx_buf->nr_frags;
3219                         j += 2;
3220                         for (k = 0; k < last; k++, j++) {
3221                                 int ring_idx = j & bp->tx_ring_mask;
3222                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
3223
3224                                 tx_buf = &txr->tx_buf_ring[ring_idx];
3225                                 dma_unmap_page(
3226                                         &pdev->dev,
3227                                         dma_unmap_addr(tx_buf, mapping),
3228                                         skb_frag_size(frag), DMA_TO_DEVICE);
3229                         }
3230                         dev_kfree_skb(skb);
3231                 }
3232                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
3233         }
3234 }
3235
3236 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
3237 {
3238         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3239         struct pci_dev *pdev = bp->pdev;
3240         struct bnxt_tpa_idx_map *map;
3241         int i, max_idx, max_agg_idx;
3242
3243         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
3244         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
3245         if (!rxr->rx_tpa)
3246                 goto skip_rx_tpa_free;
3247
3248         for (i = 0; i < bp->max_tpa; i++) {
3249                 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
3250                 u8 *data = tpa_info->data;
3251
3252                 if (!data)
3253                         continue;
3254
3255                 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
3256                                        bp->rx_buf_use_size, bp->rx_dir,
3257                                        DMA_ATTR_WEAK_ORDERING);
3258
3259                 tpa_info->data = NULL;
3260
3261                 skb_free_frag(data);
3262         }
3263
3264 skip_rx_tpa_free:
3265         if (!rxr->rx_buf_ring)
3266                 goto skip_rx_buf_free;
3267
3268         for (i = 0; i < max_idx; i++) {
3269                 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
3270                 dma_addr_t mapping = rx_buf->mapping;
3271                 void *data = rx_buf->data;
3272
3273                 if (!data)
3274                         continue;
3275
3276                 rx_buf->data = NULL;
3277                 if (BNXT_RX_PAGE_MODE(bp)) {
3278                         page_pool_recycle_direct(rxr->page_pool, data);
3279                 } else {
3280                         dma_unmap_single_attrs(&pdev->dev, mapping,
3281                                                bp->rx_buf_use_size, bp->rx_dir,
3282                                                DMA_ATTR_WEAK_ORDERING);
3283                         skb_free_frag(data);
3284                 }
3285         }
3286
3287 skip_rx_buf_free:
3288         if (!rxr->rx_agg_ring)
3289                 goto skip_rx_agg_free;
3290
3291         for (i = 0; i < max_agg_idx; i++) {
3292                 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
3293                 struct page *page = rx_agg_buf->page;
3294
3295                 if (!page)
3296                         continue;
3297
3298                 rx_agg_buf->page = NULL;
3299                 __clear_bit(i, rxr->rx_agg_bmap);
3300
3301                 page_pool_recycle_direct(rxr->page_pool, page);
3302         }
3303
3304 skip_rx_agg_free:
3305         map = rxr->rx_tpa_idx_map;
3306         if (map)
3307                 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
3308 }
3309
3310 static void bnxt_free_rx_skbs(struct bnxt *bp)
3311 {
3312         int i;
3313
3314         if (!bp->rx_ring)
3315                 return;
3316
3317         for (i = 0; i < bp->rx_nr_rings; i++)
3318                 bnxt_free_one_rx_ring_skbs(bp, i);
3319 }
3320
3321 static void bnxt_free_skbs(struct bnxt *bp)
3322 {
3323         bnxt_free_tx_skbs(bp);
3324         bnxt_free_rx_skbs(bp);
3325 }
3326
3327 static void bnxt_init_ctx_mem(struct bnxt_ctx_mem_type *ctxm, void *p, int len)
3328 {
3329         u8 init_val = ctxm->init_value;
3330         u16 offset = ctxm->init_offset;
3331         u8 *p2 = p;
3332         int i;
3333
3334         if (!init_val)
3335                 return;
3336         if (offset == BNXT_CTX_INIT_INVALID_OFFSET) {
3337                 memset(p, init_val, len);
3338                 return;
3339         }
3340         for (i = 0; i < len; i += ctxm->entry_size)
3341                 *(p2 + i + offset) = init_val;
3342 }
3343
3344 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3345 {
3346         struct pci_dev *pdev = bp->pdev;
3347         int i;
3348
3349         if (!rmem->pg_arr)
3350                 goto skip_pages;
3351
3352         for (i = 0; i < rmem->nr_pages; i++) {
3353                 if (!rmem->pg_arr[i])
3354                         continue;
3355
3356                 dma_free_coherent(&pdev->dev, rmem->page_size,
3357                                   rmem->pg_arr[i], rmem->dma_arr[i]);
3358
3359                 rmem->pg_arr[i] = NULL;
3360         }
3361 skip_pages:
3362         if (rmem->pg_tbl) {
3363                 size_t pg_tbl_size = rmem->nr_pages * 8;
3364
3365                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3366                         pg_tbl_size = rmem->page_size;
3367                 dma_free_coherent(&pdev->dev, pg_tbl_size,
3368                                   rmem->pg_tbl, rmem->pg_tbl_map);
3369                 rmem->pg_tbl = NULL;
3370         }
3371         if (rmem->vmem_size && *rmem->vmem) {
3372                 vfree(*rmem->vmem);
3373                 *rmem->vmem = NULL;
3374         }
3375 }
3376
3377 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3378 {
3379         struct pci_dev *pdev = bp->pdev;
3380         u64 valid_bit = 0;
3381         int i;
3382
3383         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
3384                 valid_bit = PTU_PTE_VALID;
3385         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
3386                 size_t pg_tbl_size = rmem->nr_pages * 8;
3387
3388                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3389                         pg_tbl_size = rmem->page_size;
3390                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
3391                                                   &rmem->pg_tbl_map,
3392                                                   GFP_KERNEL);
3393                 if (!rmem->pg_tbl)
3394                         return -ENOMEM;
3395         }
3396
3397         for (i = 0; i < rmem->nr_pages; i++) {
3398                 u64 extra_bits = valid_bit;
3399
3400                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
3401                                                      rmem->page_size,
3402                                                      &rmem->dma_arr[i],
3403                                                      GFP_KERNEL);
3404                 if (!rmem->pg_arr[i])
3405                         return -ENOMEM;
3406
3407                 if (rmem->ctx_mem)
3408                         bnxt_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i],
3409                                           rmem->page_size);
3410                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
3411                         if (i == rmem->nr_pages - 2 &&
3412                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3413                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
3414                         else if (i == rmem->nr_pages - 1 &&
3415                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3416                                 extra_bits |= PTU_PTE_LAST;
3417                         rmem->pg_tbl[i] =
3418                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
3419                 }
3420         }
3421
3422         if (rmem->vmem_size) {
3423                 *rmem->vmem = vzalloc(rmem->vmem_size);
3424                 if (!(*rmem->vmem))
3425                         return -ENOMEM;
3426         }
3427         return 0;
3428 }
3429
3430 static void bnxt_free_tpa_info(struct bnxt *bp)
3431 {
3432         int i, j;
3433
3434         for (i = 0; i < bp->rx_nr_rings; i++) {
3435                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3436
3437                 kfree(rxr->rx_tpa_idx_map);
3438                 rxr->rx_tpa_idx_map = NULL;
3439                 if (rxr->rx_tpa) {
3440                         for (j = 0; j < bp->max_tpa; j++) {
3441                                 kfree(rxr->rx_tpa[j].agg_arr);
3442                                 rxr->rx_tpa[j].agg_arr = NULL;
3443                         }
3444                 }
3445                 kfree(rxr->rx_tpa);
3446                 rxr->rx_tpa = NULL;
3447         }
3448 }
3449
3450 static int bnxt_alloc_tpa_info(struct bnxt *bp)
3451 {
3452         int i, j;
3453
3454         bp->max_tpa = MAX_TPA;
3455         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
3456                 if (!bp->max_tpa_v2)
3457                         return 0;
3458                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
3459         }
3460
3461         for (i = 0; i < bp->rx_nr_rings; i++) {
3462                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3463                 struct rx_agg_cmp *agg;
3464
3465                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
3466                                       GFP_KERNEL);
3467                 if (!rxr->rx_tpa)
3468                         return -ENOMEM;
3469
3470                 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
3471                         continue;
3472                 for (j = 0; j < bp->max_tpa; j++) {
3473                         agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL);
3474                         if (!agg)
3475                                 return -ENOMEM;
3476                         rxr->rx_tpa[j].agg_arr = agg;
3477                 }
3478                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
3479                                               GFP_KERNEL);
3480                 if (!rxr->rx_tpa_idx_map)
3481                         return -ENOMEM;
3482         }
3483         return 0;
3484 }
3485
3486 static void bnxt_free_rx_rings(struct bnxt *bp)
3487 {
3488         int i;
3489
3490         if (!bp->rx_ring)
3491                 return;
3492
3493         bnxt_free_tpa_info(bp);
3494         for (i = 0; i < bp->rx_nr_rings; i++) {
3495                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3496                 struct bnxt_ring_struct *ring;
3497
3498                 if (rxr->xdp_prog)
3499                         bpf_prog_put(rxr->xdp_prog);
3500
3501                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3502                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
3503
3504                 page_pool_destroy(rxr->page_pool);
3505                 rxr->page_pool = NULL;
3506
3507                 kfree(rxr->rx_agg_bmap);
3508                 rxr->rx_agg_bmap = NULL;
3509
3510                 ring = &rxr->rx_ring_struct;
3511                 bnxt_free_ring(bp, &ring->ring_mem);
3512
3513                 ring = &rxr->rx_agg_ring_struct;
3514                 bnxt_free_ring(bp, &ring->ring_mem);
3515         }
3516 }
3517
3518 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3519                                    struct bnxt_rx_ring_info *rxr)
3520 {
3521         struct page_pool_params pp = { 0 };
3522
3523         pp.pool_size = bp->rx_agg_ring_size;
3524         if (BNXT_RX_PAGE_MODE(bp))
3525                 pp.pool_size += bp->rx_ring_size;
3526         pp.nid = dev_to_node(&bp->pdev->dev);
3527         pp.napi = &rxr->bnapi->napi;
3528         pp.netdev = bp->dev;
3529         pp.dev = &bp->pdev->dev;
3530         pp.dma_dir = bp->rx_dir;
3531         pp.max_len = PAGE_SIZE;
3532         pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
3533
3534         rxr->page_pool = page_pool_create(&pp);
3535         if (IS_ERR(rxr->page_pool)) {
3536                 int err = PTR_ERR(rxr->page_pool);
3537
3538                 rxr->page_pool = NULL;
3539                 return err;
3540         }
3541         return 0;
3542 }
3543
3544 static int bnxt_alloc_rx_rings(struct bnxt *bp)
3545 {
3546         int i, rc = 0, agg_rings = 0;
3547
3548         if (!bp->rx_ring)
3549                 return -ENOMEM;
3550
3551         if (bp->flags & BNXT_FLAG_AGG_RINGS)
3552                 agg_rings = 1;
3553
3554         for (i = 0; i < bp->rx_nr_rings; i++) {
3555                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3556                 struct bnxt_ring_struct *ring;
3557
3558                 ring = &rxr->rx_ring_struct;
3559
3560                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
3561                 if (rc)
3562                         return rc;
3563
3564                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3565                 if (rc < 0)
3566                         return rc;
3567
3568                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3569                                                 MEM_TYPE_PAGE_POOL,
3570                                                 rxr->page_pool);
3571                 if (rc) {
3572                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
3573                         return rc;
3574                 }
3575
3576                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3577                 if (rc)
3578                         return rc;
3579
3580                 ring->grp_idx = i;
3581                 if (agg_rings) {
3582                         u16 mem_size;
3583
3584                         ring = &rxr->rx_agg_ring_struct;
3585                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3586                         if (rc)
3587                                 return rc;
3588
3589                         ring->grp_idx = i;
3590                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3591                         mem_size = rxr->rx_agg_bmap_size / 8;
3592                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3593                         if (!rxr->rx_agg_bmap)
3594                                 return -ENOMEM;
3595                 }
3596         }
3597         if (bp->flags & BNXT_FLAG_TPA)
3598                 rc = bnxt_alloc_tpa_info(bp);
3599         return rc;
3600 }
3601
3602 static void bnxt_free_tx_rings(struct bnxt *bp)
3603 {
3604         int i;
3605         struct pci_dev *pdev = bp->pdev;
3606
3607         if (!bp->tx_ring)
3608                 return;
3609
3610         for (i = 0; i < bp->tx_nr_rings; i++) {
3611                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3612                 struct bnxt_ring_struct *ring;
3613
3614                 if (txr->tx_push) {
3615                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
3616                                           txr->tx_push, txr->tx_push_mapping);
3617                         txr->tx_push = NULL;
3618                 }
3619
3620                 ring = &txr->tx_ring_struct;
3621
3622                 bnxt_free_ring(bp, &ring->ring_mem);
3623         }
3624 }
3625
3626 #define BNXT_TC_TO_RING_BASE(bp, tc)    \
3627         ((tc) * (bp)->tx_nr_rings_per_tc)
3628
3629 #define BNXT_RING_TO_TC_OFF(bp, tx)     \
3630         ((tx) % (bp)->tx_nr_rings_per_tc)
3631
3632 #define BNXT_RING_TO_TC(bp, tx)         \
3633         ((tx) / (bp)->tx_nr_rings_per_tc)
3634
3635 static int bnxt_alloc_tx_rings(struct bnxt *bp)
3636 {
3637         int i, j, rc;
3638         struct pci_dev *pdev = bp->pdev;
3639
3640         bp->tx_push_size = 0;
3641         if (bp->tx_push_thresh) {
3642                 int push_size;
3643
3644                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3645                                         bp->tx_push_thresh);
3646
3647                 if (push_size > 256) {
3648                         push_size = 0;
3649                         bp->tx_push_thresh = 0;
3650                 }
3651
3652                 bp->tx_push_size = push_size;
3653         }
3654
3655         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3656                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3657                 struct bnxt_ring_struct *ring;
3658                 u8 qidx;
3659
3660                 ring = &txr->tx_ring_struct;
3661
3662                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3663                 if (rc)
3664                         return rc;
3665
3666                 ring->grp_idx = txr->bnapi->index;
3667                 if (bp->tx_push_size) {
3668                         dma_addr_t mapping;
3669
3670                         /* One pre-allocated DMA buffer to backup
3671                          * TX push operation
3672                          */
3673                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
3674                                                 bp->tx_push_size,
3675                                                 &txr->tx_push_mapping,
3676                                                 GFP_KERNEL);
3677
3678                         if (!txr->tx_push)
3679                                 return -ENOMEM;
3680
3681                         mapping = txr->tx_push_mapping +
3682                                 sizeof(struct tx_push_bd);
3683                         txr->data_mapping = cpu_to_le64(mapping);
3684                 }
3685                 qidx = bp->tc_to_qidx[j];
3686                 ring->queue_id = bp->q_info[qidx].queue_id;
3687                 spin_lock_init(&txr->xdp_tx_lock);
3688                 if (i < bp->tx_nr_rings_xdp)
3689                         continue;
3690                 if (BNXT_RING_TO_TC_OFF(bp, i) == (bp->tx_nr_rings_per_tc - 1))
3691                         j++;
3692         }
3693         return 0;
3694 }
3695
3696 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr)
3697 {
3698         struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3699
3700         kfree(cpr->cp_desc_ring);
3701         cpr->cp_desc_ring = NULL;
3702         ring->ring_mem.pg_arr = NULL;
3703         kfree(cpr->cp_desc_mapping);
3704         cpr->cp_desc_mapping = NULL;
3705         ring->ring_mem.dma_arr = NULL;
3706 }
3707
3708 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n)
3709 {
3710         cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL);
3711         if (!cpr->cp_desc_ring)
3712                 return -ENOMEM;
3713         cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping),
3714                                        GFP_KERNEL);
3715         if (!cpr->cp_desc_mapping)
3716                 return -ENOMEM;
3717         return 0;
3718 }
3719
3720 static void bnxt_free_all_cp_arrays(struct bnxt *bp)
3721 {
3722         int i;
3723
3724         if (!bp->bnapi)
3725                 return;
3726         for (i = 0; i < bp->cp_nr_rings; i++) {
3727                 struct bnxt_napi *bnapi = bp->bnapi[i];
3728
3729                 if (!bnapi)
3730                         continue;
3731                 bnxt_free_cp_arrays(&bnapi->cp_ring);
3732         }
3733 }
3734
3735 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp)
3736 {
3737         int i, n = bp->cp_nr_pages;
3738
3739         for (i = 0; i < bp->cp_nr_rings; i++) {
3740                 struct bnxt_napi *bnapi = bp->bnapi[i];
3741                 int rc;
3742
3743                 if (!bnapi)
3744                         continue;
3745                 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n);
3746                 if (rc)
3747                         return rc;
3748         }
3749         return 0;
3750 }
3751
3752 static void bnxt_free_cp_rings(struct bnxt *bp)
3753 {
3754         int i;
3755
3756         if (!bp->bnapi)
3757                 return;
3758
3759         for (i = 0; i < bp->cp_nr_rings; i++) {
3760                 struct bnxt_napi *bnapi = bp->bnapi[i];
3761                 struct bnxt_cp_ring_info *cpr;
3762                 struct bnxt_ring_struct *ring;
3763                 int j;
3764
3765                 if (!bnapi)
3766                         continue;
3767
3768                 cpr = &bnapi->cp_ring;
3769                 ring = &cpr->cp_ring_struct;
3770
3771                 bnxt_free_ring(bp, &ring->ring_mem);
3772
3773                 if (!cpr->cp_ring_arr)
3774                         continue;
3775
3776                 for (j = 0; j < cpr->cp_ring_count; j++) {
3777                         struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
3778
3779                         ring = &cpr2->cp_ring_struct;
3780                         bnxt_free_ring(bp, &ring->ring_mem);
3781                         bnxt_free_cp_arrays(cpr2);
3782                 }
3783                 kfree(cpr->cp_ring_arr);
3784                 cpr->cp_ring_arr = NULL;
3785                 cpr->cp_ring_count = 0;
3786         }
3787 }
3788
3789 static int bnxt_alloc_cp_sub_ring(struct bnxt *bp,
3790                                   struct bnxt_cp_ring_info *cpr)
3791 {
3792         struct bnxt_ring_mem_info *rmem;
3793         struct bnxt_ring_struct *ring;
3794         int rc;
3795
3796         rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
3797         if (rc) {
3798                 bnxt_free_cp_arrays(cpr);
3799                 return -ENOMEM;
3800         }
3801         ring = &cpr->cp_ring_struct;
3802         rmem = &ring->ring_mem;
3803         rmem->nr_pages = bp->cp_nr_pages;
3804         rmem->page_size = HW_CMPD_RING_SIZE;
3805         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3806         rmem->dma_arr = cpr->cp_desc_mapping;
3807         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3808         rc = bnxt_alloc_ring(bp, rmem);
3809         if (rc) {
3810                 bnxt_free_ring(bp, rmem);
3811                 bnxt_free_cp_arrays(cpr);
3812         }
3813         return rc;
3814 }
3815
3816 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3817 {
3818         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3819         int i, j, rc, ulp_base_vec, ulp_msix;
3820         int tcs = bp->num_tc;
3821
3822         if (!tcs)
3823                 tcs = 1;
3824         ulp_msix = bnxt_get_ulp_msix_num(bp);
3825         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3826         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
3827                 struct bnxt_napi *bnapi = bp->bnapi[i];
3828                 struct bnxt_cp_ring_info *cpr, *cpr2;
3829                 struct bnxt_ring_struct *ring;
3830                 int cp_count = 0, k;
3831                 int rx = 0, tx = 0;
3832
3833                 if (!bnapi)
3834                         continue;
3835
3836                 cpr = &bnapi->cp_ring;
3837                 cpr->bnapi = bnapi;
3838                 ring = &cpr->cp_ring_struct;
3839
3840                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3841                 if (rc)
3842                         return rc;
3843
3844                 if (ulp_msix && i >= ulp_base_vec)
3845                         ring->map_idx = i + ulp_msix;
3846                 else
3847                         ring->map_idx = i;
3848
3849                 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
3850                         continue;
3851
3852                 if (i < bp->rx_nr_rings) {
3853                         cp_count++;
3854                         rx = 1;
3855                 }
3856                 if (i < bp->tx_nr_rings_xdp) {
3857                         cp_count++;
3858                         tx = 1;
3859                 } else if ((sh && i < bp->tx_nr_rings) ||
3860                          (!sh && i >= bp->rx_nr_rings)) {
3861                         cp_count += tcs;
3862                         tx = 1;
3863                 }
3864
3865                 cpr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr),
3866                                            GFP_KERNEL);
3867                 if (!cpr->cp_ring_arr)
3868                         return -ENOMEM;
3869                 cpr->cp_ring_count = cp_count;
3870
3871                 for (k = 0; k < cp_count; k++) {
3872                         cpr2 = &cpr->cp_ring_arr[k];
3873                         rc = bnxt_alloc_cp_sub_ring(bp, cpr2);
3874                         if (rc)
3875                                 return rc;
3876                         cpr2->bnapi = bnapi;
3877                         cpr2->cp_idx = k;
3878                         if (!k && rx) {
3879                                 bp->rx_ring[i].rx_cpr = cpr2;
3880                                 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_RX;
3881                         } else {
3882                                 int n, tc = k - rx;
3883
3884                                 n = BNXT_TC_TO_RING_BASE(bp, tc) + j;
3885                                 bp->tx_ring[n].tx_cpr = cpr2;
3886                                 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_TX;
3887                         }
3888                 }
3889                 if (tx)
3890                         j++;
3891         }
3892         return 0;
3893 }
3894
3895 static void bnxt_init_ring_struct(struct bnxt *bp)
3896 {
3897         int i, j;
3898
3899         for (i = 0; i < bp->cp_nr_rings; i++) {
3900                 struct bnxt_napi *bnapi = bp->bnapi[i];
3901                 struct bnxt_ring_mem_info *rmem;
3902                 struct bnxt_cp_ring_info *cpr;
3903                 struct bnxt_rx_ring_info *rxr;
3904                 struct bnxt_tx_ring_info *txr;
3905                 struct bnxt_ring_struct *ring;
3906
3907                 if (!bnapi)
3908                         continue;
3909
3910                 cpr = &bnapi->cp_ring;
3911                 ring = &cpr->cp_ring_struct;
3912                 rmem = &ring->ring_mem;
3913                 rmem->nr_pages = bp->cp_nr_pages;
3914                 rmem->page_size = HW_CMPD_RING_SIZE;
3915                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3916                 rmem->dma_arr = cpr->cp_desc_mapping;
3917                 rmem->vmem_size = 0;
3918
3919                 rxr = bnapi->rx_ring;
3920                 if (!rxr)
3921                         goto skip_rx;
3922
3923                 ring = &rxr->rx_ring_struct;
3924                 rmem = &ring->ring_mem;
3925                 rmem->nr_pages = bp->rx_nr_pages;
3926                 rmem->page_size = HW_RXBD_RING_SIZE;
3927                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3928                 rmem->dma_arr = rxr->rx_desc_mapping;
3929                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3930                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3931
3932                 ring = &rxr->rx_agg_ring_struct;
3933                 rmem = &ring->ring_mem;
3934                 rmem->nr_pages = bp->rx_agg_nr_pages;
3935                 rmem->page_size = HW_RXBD_RING_SIZE;
3936                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3937                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3938                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3939                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3940
3941 skip_rx:
3942                 bnxt_for_each_napi_tx(j, bnapi, txr) {
3943                         ring = &txr->tx_ring_struct;
3944                         rmem = &ring->ring_mem;
3945                         rmem->nr_pages = bp->tx_nr_pages;
3946                         rmem->page_size = HW_TXBD_RING_SIZE;
3947                         rmem->pg_arr = (void **)txr->tx_desc_ring;
3948                         rmem->dma_arr = txr->tx_desc_mapping;
3949                         rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3950                         rmem->vmem = (void **)&txr->tx_buf_ring;
3951                 }
3952         }
3953 }
3954
3955 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3956 {
3957         int i;
3958         u32 prod;
3959         struct rx_bd **rx_buf_ring;
3960
3961         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3962         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3963                 int j;
3964                 struct rx_bd *rxbd;
3965
3966                 rxbd = rx_buf_ring[i];
3967                 if (!rxbd)
3968                         continue;
3969
3970                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3971                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3972                         rxbd->rx_bd_opaque = prod;
3973                 }
3974         }
3975 }
3976
3977 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3978 {
3979         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3980         struct net_device *dev = bp->dev;
3981         u32 prod;
3982         int i;
3983
3984         prod = rxr->rx_prod;
3985         for (i = 0; i < bp->rx_ring_size; i++) {
3986                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3987                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3988                                     ring_nr, i, bp->rx_ring_size);
3989                         break;
3990                 }
3991                 prod = NEXT_RX(prod);
3992         }
3993         rxr->rx_prod = prod;
3994
3995         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3996                 return 0;
3997
3998         prod = rxr->rx_agg_prod;
3999         for (i = 0; i < bp->rx_agg_ring_size; i++) {
4000                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
4001                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
4002                                     ring_nr, i, bp->rx_ring_size);
4003                         break;
4004                 }
4005                 prod = NEXT_RX_AGG(prod);
4006         }
4007         rxr->rx_agg_prod = prod;
4008
4009         if (rxr->rx_tpa) {
4010                 dma_addr_t mapping;
4011                 u8 *data;
4012
4013                 for (i = 0; i < bp->max_tpa; i++) {
4014                         data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL);
4015                         if (!data)
4016                                 return -ENOMEM;
4017
4018                         rxr->rx_tpa[i].data = data;
4019                         rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
4020                         rxr->rx_tpa[i].mapping = mapping;
4021                 }
4022         }
4023         return 0;
4024 }
4025
4026 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
4027 {
4028         struct bnxt_rx_ring_info *rxr;
4029         struct bnxt_ring_struct *ring;
4030         u32 type;
4031
4032         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
4033                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
4034
4035         if (NET_IP_ALIGN == 2)
4036                 type |= RX_BD_FLAGS_SOP;
4037
4038         rxr = &bp->rx_ring[ring_nr];
4039         ring = &rxr->rx_ring_struct;
4040         bnxt_init_rxbd_pages(ring, type);
4041
4042         netif_queue_set_napi(bp->dev, ring_nr, NETDEV_QUEUE_TYPE_RX,
4043                              &rxr->bnapi->napi);
4044
4045         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
4046                 bpf_prog_add(bp->xdp_prog, 1);
4047                 rxr->xdp_prog = bp->xdp_prog;
4048         }
4049         ring->fw_ring_id = INVALID_HW_RING_ID;
4050
4051         ring = &rxr->rx_agg_ring_struct;
4052         ring->fw_ring_id = INVALID_HW_RING_ID;
4053
4054         if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
4055                 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
4056                         RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
4057
4058                 bnxt_init_rxbd_pages(ring, type);
4059         }
4060
4061         return bnxt_alloc_one_rx_ring(bp, ring_nr);
4062 }
4063
4064 static void bnxt_init_cp_rings(struct bnxt *bp)
4065 {
4066         int i, j;
4067
4068         for (i = 0; i < bp->cp_nr_rings; i++) {
4069                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
4070                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4071
4072                 ring->fw_ring_id = INVALID_HW_RING_ID;
4073                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
4074                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
4075                 if (!cpr->cp_ring_arr)
4076                         continue;
4077                 for (j = 0; j < cpr->cp_ring_count; j++) {
4078                         struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
4079
4080                         ring = &cpr2->cp_ring_struct;
4081                         ring->fw_ring_id = INVALID_HW_RING_ID;
4082                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
4083                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
4084                 }
4085         }
4086 }
4087
4088 static int bnxt_init_rx_rings(struct bnxt *bp)
4089 {
4090         int i, rc = 0;
4091
4092         if (BNXT_RX_PAGE_MODE(bp)) {
4093                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
4094                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
4095         } else {
4096                 bp->rx_offset = BNXT_RX_OFFSET;
4097                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
4098         }
4099
4100         for (i = 0; i < bp->rx_nr_rings; i++) {
4101                 rc = bnxt_init_one_rx_ring(bp, i);
4102                 if (rc)
4103                         break;
4104         }
4105
4106         return rc;
4107 }
4108
4109 static int bnxt_init_tx_rings(struct bnxt *bp)
4110 {
4111         u16 i;
4112
4113         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
4114                                    BNXT_MIN_TX_DESC_CNT);
4115
4116         for (i = 0; i < bp->tx_nr_rings; i++) {
4117                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4118                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4119
4120                 ring->fw_ring_id = INVALID_HW_RING_ID;
4121
4122                 if (i >= bp->tx_nr_rings_xdp)
4123                         netif_queue_set_napi(bp->dev, i - bp->tx_nr_rings_xdp,
4124                                              NETDEV_QUEUE_TYPE_TX,
4125                                              &txr->bnapi->napi);
4126         }
4127
4128         return 0;
4129 }
4130
4131 static void bnxt_free_ring_grps(struct bnxt *bp)
4132 {
4133         kfree(bp->grp_info);
4134         bp->grp_info = NULL;
4135 }
4136
4137 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
4138 {
4139         int i;
4140
4141         if (irq_re_init) {
4142                 bp->grp_info = kcalloc(bp->cp_nr_rings,
4143                                        sizeof(struct bnxt_ring_grp_info),
4144                                        GFP_KERNEL);
4145                 if (!bp->grp_info)
4146                         return -ENOMEM;
4147         }
4148         for (i = 0; i < bp->cp_nr_rings; i++) {
4149                 if (irq_re_init)
4150                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
4151                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4152                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
4153                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
4154                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
4155         }
4156         return 0;
4157 }
4158
4159 static void bnxt_free_vnics(struct bnxt *bp)
4160 {
4161         kfree(bp->vnic_info);
4162         bp->vnic_info = NULL;
4163         bp->nr_vnics = 0;
4164 }
4165
4166 static int bnxt_alloc_vnics(struct bnxt *bp)
4167 {
4168         int num_vnics = 1;
4169
4170 #ifdef CONFIG_RFS_ACCEL
4171         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5_PLUS)) == BNXT_FLAG_RFS)
4172                 num_vnics += bp->rx_nr_rings;
4173 #endif
4174
4175         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4176                 num_vnics++;
4177
4178         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
4179                                 GFP_KERNEL);
4180         if (!bp->vnic_info)
4181                 return -ENOMEM;
4182
4183         bp->nr_vnics = num_vnics;
4184         return 0;
4185 }
4186
4187 static void bnxt_init_vnics(struct bnxt *bp)
4188 {
4189         int i;
4190
4191         for (i = 0; i < bp->nr_vnics; i++) {
4192                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4193                 int j;
4194
4195                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
4196                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
4197                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
4198
4199                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
4200
4201                 if (bp->vnic_info[i].rss_hash_key) {
4202                         if (!i) {
4203                                 u8 *key = (void *)vnic->rss_hash_key;
4204                                 int k;
4205
4206                                 bp->toeplitz_prefix = 0;
4207                                 get_random_bytes(vnic->rss_hash_key,
4208                                               HW_HASH_KEY_SIZE);
4209                                 for (k = 0; k < 8; k++) {
4210                                         bp->toeplitz_prefix <<= 8;
4211                                         bp->toeplitz_prefix |= key[k];
4212                                 }
4213                         } else {
4214                                 memcpy(vnic->rss_hash_key,
4215                                        bp->vnic_info[0].rss_hash_key,
4216                                        HW_HASH_KEY_SIZE);
4217                         }
4218                 }
4219         }
4220 }
4221
4222 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
4223 {
4224         int pages;
4225
4226         pages = ring_size / desc_per_pg;
4227
4228         if (!pages)
4229                 return 1;
4230
4231         pages++;
4232
4233         while (pages & (pages - 1))
4234                 pages++;
4235
4236         return pages;
4237 }
4238
4239 void bnxt_set_tpa_flags(struct bnxt *bp)
4240 {
4241         bp->flags &= ~BNXT_FLAG_TPA;
4242         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
4243                 return;
4244         if (bp->dev->features & NETIF_F_LRO)
4245                 bp->flags |= BNXT_FLAG_LRO;
4246         else if (bp->dev->features & NETIF_F_GRO_HW)
4247                 bp->flags |= BNXT_FLAG_GRO;
4248 }
4249
4250 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
4251  * be set on entry.
4252  */
4253 void bnxt_set_ring_params(struct bnxt *bp)
4254 {
4255         u32 ring_size, rx_size, rx_space, max_rx_cmpl;
4256         u32 agg_factor = 0, agg_ring_size = 0;
4257
4258         /* 8 for CRC and VLAN */
4259         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
4260
4261         rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
4262                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4263
4264         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
4265         ring_size = bp->rx_ring_size;
4266         bp->rx_agg_ring_size = 0;
4267         bp->rx_agg_nr_pages = 0;
4268
4269         if (bp->flags & BNXT_FLAG_TPA)
4270                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
4271
4272         bp->flags &= ~BNXT_FLAG_JUMBO;
4273         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
4274                 u32 jumbo_factor;
4275
4276                 bp->flags |= BNXT_FLAG_JUMBO;
4277                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
4278                 if (jumbo_factor > agg_factor)
4279                         agg_factor = jumbo_factor;
4280         }
4281         if (agg_factor) {
4282                 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) {
4283                         ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA;
4284                         netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n",
4285                                     bp->rx_ring_size, ring_size);
4286                         bp->rx_ring_size = ring_size;
4287                 }
4288                 agg_ring_size = ring_size * agg_factor;
4289
4290                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
4291                                                         RX_DESC_CNT);
4292                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
4293                         u32 tmp = agg_ring_size;
4294
4295                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
4296                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
4297                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
4298                                     tmp, agg_ring_size);
4299                 }
4300                 bp->rx_agg_ring_size = agg_ring_size;
4301                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
4302
4303                 if (BNXT_RX_PAGE_MODE(bp)) {
4304                         rx_space = PAGE_SIZE;
4305                         rx_size = PAGE_SIZE -
4306                                   ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) -
4307                                   SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4308                 } else {
4309                         rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
4310                         rx_space = rx_size + NET_SKB_PAD +
4311                                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4312                 }
4313         }
4314
4315         bp->rx_buf_use_size = rx_size;
4316         bp->rx_buf_size = rx_space;
4317
4318         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
4319         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
4320
4321         ring_size = bp->tx_ring_size;
4322         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
4323         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
4324
4325         max_rx_cmpl = bp->rx_ring_size;
4326         /* MAX TPA needs to be added because TPA_START completions are
4327          * immediately recycled, so the TPA completions are not bound by
4328          * the RX ring size.
4329          */
4330         if (bp->flags & BNXT_FLAG_TPA)
4331                 max_rx_cmpl += bp->max_tpa;
4332         /* RX and TPA completions are 32-byte, all others are 16-byte */
4333         ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
4334         bp->cp_ring_size = ring_size;
4335
4336         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
4337         if (bp->cp_nr_pages > MAX_CP_PAGES) {
4338                 bp->cp_nr_pages = MAX_CP_PAGES;
4339                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
4340                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
4341                             ring_size, bp->cp_ring_size);
4342         }
4343         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
4344         bp->cp_ring_mask = bp->cp_bit - 1;
4345 }
4346
4347 /* Changing allocation mode of RX rings.
4348  * TODO: Update when extending xdp_rxq_info to support allocation modes.
4349  */
4350 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
4351 {
4352         struct net_device *dev = bp->dev;
4353
4354         if (page_mode) {
4355                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
4356                 bp->flags |= BNXT_FLAG_RX_PAGE_MODE;
4357
4358                 if (bp->xdp_prog->aux->xdp_has_frags)
4359                         dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_MTU);
4360                 else
4361                         dev->max_mtu =
4362                                 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
4363                 if (dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
4364                         bp->flags |= BNXT_FLAG_JUMBO;
4365                         bp->rx_skb_func = bnxt_rx_multi_page_skb;
4366                 } else {
4367                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4368                         bp->rx_skb_func = bnxt_rx_page_skb;
4369                 }
4370                 bp->rx_dir = DMA_BIDIRECTIONAL;
4371                 /* Disable LRO or GRO_HW */
4372                 netdev_update_features(dev);
4373         } else {
4374                 dev->max_mtu = bp->max_mtu;
4375                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
4376                 bp->rx_dir = DMA_FROM_DEVICE;
4377                 bp->rx_skb_func = bnxt_rx_skb;
4378         }
4379         return 0;
4380 }
4381
4382 static void bnxt_free_vnic_attributes(struct bnxt *bp)
4383 {
4384         int i;
4385         struct bnxt_vnic_info *vnic;
4386         struct pci_dev *pdev = bp->pdev;
4387
4388         if (!bp->vnic_info)
4389                 return;
4390
4391         for (i = 0; i < bp->nr_vnics; i++) {
4392                 vnic = &bp->vnic_info[i];
4393
4394                 kfree(vnic->fw_grp_ids);
4395                 vnic->fw_grp_ids = NULL;
4396
4397                 kfree(vnic->uc_list);
4398                 vnic->uc_list = NULL;
4399
4400                 if (vnic->mc_list) {
4401                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
4402                                           vnic->mc_list, vnic->mc_list_mapping);
4403                         vnic->mc_list = NULL;
4404                 }
4405
4406                 if (vnic->rss_table) {
4407                         dma_free_coherent(&pdev->dev, vnic->rss_table_size,
4408                                           vnic->rss_table,
4409                                           vnic->rss_table_dma_addr);
4410                         vnic->rss_table = NULL;
4411                 }
4412
4413                 vnic->rss_hash_key = NULL;
4414                 vnic->flags = 0;
4415         }
4416 }
4417
4418 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
4419 {
4420         int i, rc = 0, size;
4421         struct bnxt_vnic_info *vnic;
4422         struct pci_dev *pdev = bp->pdev;
4423         int max_rings;
4424
4425         for (i = 0; i < bp->nr_vnics; i++) {
4426                 vnic = &bp->vnic_info[i];
4427
4428                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
4429                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
4430
4431                         if (mem_size > 0) {
4432                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
4433                                 if (!vnic->uc_list) {
4434                                         rc = -ENOMEM;
4435                                         goto out;
4436                                 }
4437                         }
4438                 }
4439
4440                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
4441                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
4442                         vnic->mc_list =
4443                                 dma_alloc_coherent(&pdev->dev,
4444                                                    vnic->mc_list_size,
4445                                                    &vnic->mc_list_mapping,
4446                                                    GFP_KERNEL);
4447                         if (!vnic->mc_list) {
4448                                 rc = -ENOMEM;
4449                                 goto out;
4450                         }
4451                 }
4452
4453                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4454                         goto vnic_skip_grps;
4455
4456                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4457                         max_rings = bp->rx_nr_rings;
4458                 else
4459                         max_rings = 1;
4460
4461                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
4462                 if (!vnic->fw_grp_ids) {
4463                         rc = -ENOMEM;
4464                         goto out;
4465                 }
4466 vnic_skip_grps:
4467                 if ((bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) &&
4468                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
4469                         continue;
4470
4471                 /* Allocate rss table and hash key */
4472                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
4473                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4474                         size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
4475
4476                 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
4477                 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
4478                                                      vnic->rss_table_size,
4479                                                      &vnic->rss_table_dma_addr,
4480                                                      GFP_KERNEL);
4481                 if (!vnic->rss_table) {
4482                         rc = -ENOMEM;
4483                         goto out;
4484                 }
4485
4486                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
4487                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
4488         }
4489         return 0;
4490
4491 out:
4492         return rc;
4493 }
4494
4495 static void bnxt_free_hwrm_resources(struct bnxt *bp)
4496 {
4497         struct bnxt_hwrm_wait_token *token;
4498
4499         dma_pool_destroy(bp->hwrm_dma_pool);
4500         bp->hwrm_dma_pool = NULL;
4501
4502         rcu_read_lock();
4503         hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node)
4504                 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED);
4505         rcu_read_unlock();
4506 }
4507
4508 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
4509 {
4510         bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev,
4511                                             BNXT_HWRM_DMA_SIZE,
4512                                             BNXT_HWRM_DMA_ALIGN, 0);
4513         if (!bp->hwrm_dma_pool)
4514                 return -ENOMEM;
4515
4516         INIT_HLIST_HEAD(&bp->hwrm_pending_list);
4517
4518         return 0;
4519 }
4520
4521 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
4522 {
4523         kfree(stats->hw_masks);
4524         stats->hw_masks = NULL;
4525         kfree(stats->sw_stats);
4526         stats->sw_stats = NULL;
4527         if (stats->hw_stats) {
4528                 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
4529                                   stats->hw_stats_map);
4530                 stats->hw_stats = NULL;
4531         }
4532 }
4533
4534 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
4535                                 bool alloc_masks)
4536 {
4537         stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
4538                                              &stats->hw_stats_map, GFP_KERNEL);
4539         if (!stats->hw_stats)
4540                 return -ENOMEM;
4541
4542         stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
4543         if (!stats->sw_stats)
4544                 goto stats_mem_err;
4545
4546         if (alloc_masks) {
4547                 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
4548                 if (!stats->hw_masks)
4549                         goto stats_mem_err;
4550         }
4551         return 0;
4552
4553 stats_mem_err:
4554         bnxt_free_stats_mem(bp, stats);
4555         return -ENOMEM;
4556 }
4557
4558 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
4559 {
4560         int i;
4561
4562         for (i = 0; i < count; i++)
4563                 mask_arr[i] = mask;
4564 }
4565
4566 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
4567 {
4568         int i;
4569
4570         for (i = 0; i < count; i++)
4571                 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
4572 }
4573
4574 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
4575                                     struct bnxt_stats_mem *stats)
4576 {
4577         struct hwrm_func_qstats_ext_output *resp;
4578         struct hwrm_func_qstats_ext_input *req;
4579         __le64 *hw_masks;
4580         int rc;
4581
4582         if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
4583             !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
4584                 return -EOPNOTSUPP;
4585
4586         rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT);
4587         if (rc)
4588                 return rc;
4589
4590         req->fid = cpu_to_le16(0xffff);
4591         req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4592
4593         resp = hwrm_req_hold(bp, req);
4594         rc = hwrm_req_send(bp, req);
4595         if (!rc) {
4596                 hw_masks = &resp->rx_ucast_pkts;
4597                 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
4598         }
4599         hwrm_req_drop(bp, req);
4600         return rc;
4601 }
4602
4603 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
4604 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
4605
4606 static void bnxt_init_stats(struct bnxt *bp)
4607 {
4608         struct bnxt_napi *bnapi = bp->bnapi[0];
4609         struct bnxt_cp_ring_info *cpr;
4610         struct bnxt_stats_mem *stats;
4611         __le64 *rx_stats, *tx_stats;
4612         int rc, rx_count, tx_count;
4613         u64 *rx_masks, *tx_masks;
4614         u64 mask;
4615         u8 flags;
4616
4617         cpr = &bnapi->cp_ring;
4618         stats = &cpr->stats;
4619         rc = bnxt_hwrm_func_qstat_ext(bp, stats);
4620         if (rc) {
4621                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4622                         mask = (1ULL << 48) - 1;
4623                 else
4624                         mask = -1ULL;
4625                 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
4626         }
4627         if (bp->flags & BNXT_FLAG_PORT_STATS) {
4628                 stats = &bp->port_stats;
4629                 rx_stats = stats->hw_stats;
4630                 rx_masks = stats->hw_masks;
4631                 rx_count = sizeof(struct rx_port_stats) / 8;
4632                 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4633                 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4634                 tx_count = sizeof(struct tx_port_stats) / 8;
4635
4636                 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
4637                 rc = bnxt_hwrm_port_qstats(bp, flags);
4638                 if (rc) {
4639                         mask = (1ULL << 40) - 1;
4640
4641                         bnxt_fill_masks(rx_masks, mask, rx_count);
4642                         bnxt_fill_masks(tx_masks, mask, tx_count);
4643                 } else {
4644                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4645                         bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
4646                         bnxt_hwrm_port_qstats(bp, 0);
4647                 }
4648         }
4649         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
4650                 stats = &bp->rx_port_stats_ext;
4651                 rx_stats = stats->hw_stats;
4652                 rx_masks = stats->hw_masks;
4653                 rx_count = sizeof(struct rx_port_stats_ext) / 8;
4654                 stats = &bp->tx_port_stats_ext;
4655                 tx_stats = stats->hw_stats;
4656                 tx_masks = stats->hw_masks;
4657                 tx_count = sizeof(struct tx_port_stats_ext) / 8;
4658
4659                 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4660                 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
4661                 if (rc) {
4662                         mask = (1ULL << 40) - 1;
4663
4664                         bnxt_fill_masks(rx_masks, mask, rx_count);
4665                         if (tx_stats)
4666                                 bnxt_fill_masks(tx_masks, mask, tx_count);
4667                 } else {
4668                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4669                         if (tx_stats)
4670                                 bnxt_copy_hw_masks(tx_masks, tx_stats,
4671                                                    tx_count);
4672                         bnxt_hwrm_port_qstats_ext(bp, 0);
4673                 }
4674         }
4675 }
4676
4677 static void bnxt_free_port_stats(struct bnxt *bp)
4678 {
4679         bp->flags &= ~BNXT_FLAG_PORT_STATS;
4680         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
4681
4682         bnxt_free_stats_mem(bp, &bp->port_stats);
4683         bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4684         bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4685 }
4686
4687 static void bnxt_free_ring_stats(struct bnxt *bp)
4688 {
4689         int i;
4690
4691         if (!bp->bnapi)
4692                 return;
4693
4694         for (i = 0; i < bp->cp_nr_rings; i++) {
4695                 struct bnxt_napi *bnapi = bp->bnapi[i];
4696                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4697
4698                 bnxt_free_stats_mem(bp, &cpr->stats);
4699         }
4700 }
4701
4702 static int bnxt_alloc_stats(struct bnxt *bp)
4703 {
4704         u32 size, i;
4705         int rc;
4706
4707         size = bp->hw_ring_stats_size;
4708
4709         for (i = 0; i < bp->cp_nr_rings; i++) {
4710                 struct bnxt_napi *bnapi = bp->bnapi[i];
4711                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4712
4713                 cpr->stats.len = size;
4714                 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4715                 if (rc)
4716                         return rc;
4717
4718                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4719         }
4720
4721         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4722                 return 0;
4723
4724         if (bp->port_stats.hw_stats)
4725                 goto alloc_ext_stats;
4726
4727         bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4728         rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4729         if (rc)
4730                 return rc;
4731
4732         bp->flags |= BNXT_FLAG_PORT_STATS;
4733
4734 alloc_ext_stats:
4735         /* Display extended statistics only if FW supports it */
4736         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4737                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4738                         return 0;
4739
4740         if (bp->rx_port_stats_ext.hw_stats)
4741                 goto alloc_tx_ext_stats;
4742
4743         bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4744         rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4745         /* Extended stats are optional */
4746         if (rc)
4747                 return 0;
4748
4749 alloc_tx_ext_stats:
4750         if (bp->tx_port_stats_ext.hw_stats)
4751                 return 0;
4752
4753         if (bp->hwrm_spec_code >= 0x10902 ||
4754             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4755                 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4756                 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4757                 /* Extended stats are optional */
4758                 if (rc)
4759                         return 0;
4760         }
4761         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4762         return 0;
4763 }
4764
4765 static void bnxt_clear_ring_indices(struct bnxt *bp)
4766 {
4767         int i, j;
4768
4769         if (!bp->bnapi)
4770                 return;
4771
4772         for (i = 0; i < bp->cp_nr_rings; i++) {
4773                 struct bnxt_napi *bnapi = bp->bnapi[i];
4774                 struct bnxt_cp_ring_info *cpr;
4775                 struct bnxt_rx_ring_info *rxr;
4776                 struct bnxt_tx_ring_info *txr;
4777
4778                 if (!bnapi)
4779                         continue;
4780
4781                 cpr = &bnapi->cp_ring;
4782                 cpr->cp_raw_cons = 0;
4783
4784                 bnxt_for_each_napi_tx(j, bnapi, txr) {
4785                         txr->tx_prod = 0;
4786                         txr->tx_cons = 0;
4787                         txr->tx_hw_cons = 0;
4788                 }
4789
4790                 rxr = bnapi->rx_ring;
4791                 if (rxr) {
4792                         rxr->rx_prod = 0;
4793                         rxr->rx_agg_prod = 0;
4794                         rxr->rx_sw_agg_prod = 0;
4795                         rxr->rx_next_cons = 0;
4796                 }
4797                 bnapi->events = 0;
4798         }
4799 }
4800
4801 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool all)
4802 {
4803         int i;
4804
4805         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
4806          * safe to delete the hash table.
4807          */
4808         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4809                 struct hlist_head *head;
4810                 struct hlist_node *tmp;
4811                 struct bnxt_ntuple_filter *fltr;
4812
4813                 head = &bp->ntp_fltr_hash_tbl[i];
4814                 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
4815                         bnxt_del_l2_filter(bp, fltr->l2_fltr);
4816                         if (!all && (fltr->base.flags & BNXT_ACT_FUNC_DST))
4817                                 continue;
4818                         hlist_del(&fltr->base.hash);
4819                         clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
4820                         bp->ntp_fltr_count--;
4821                         kfree(fltr);
4822                 }
4823         }
4824         if (!all)
4825                 return;
4826
4827         bitmap_free(bp->ntp_fltr_bmap);
4828         bp->ntp_fltr_bmap = NULL;
4829         bp->ntp_fltr_count = 0;
4830 }
4831
4832 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4833 {
4834         int i, rc = 0;
4835
4836         if (!(bp->flags & BNXT_FLAG_RFS) || bp->ntp_fltr_bmap)
4837                 return 0;
4838
4839         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4840                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4841
4842         bp->ntp_fltr_count = 0;
4843         bp->ntp_fltr_bmap = bitmap_zalloc(BNXT_MAX_FLTR, GFP_KERNEL);
4844
4845         if (!bp->ntp_fltr_bmap)
4846                 rc = -ENOMEM;
4847
4848         return rc;
4849 }
4850
4851 static void bnxt_free_l2_filters(struct bnxt *bp, bool all)
4852 {
4853         int i;
4854
4855         for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++) {
4856                 struct hlist_head *head;
4857                 struct hlist_node *tmp;
4858                 struct bnxt_l2_filter *fltr;
4859
4860                 head = &bp->l2_fltr_hash_tbl[i];
4861                 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
4862                         if (!all && (fltr->base.flags & BNXT_ACT_FUNC_DST))
4863                                 continue;
4864                         hlist_del(&fltr->base.hash);
4865                         if (fltr->base.flags) {
4866                                 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
4867                                 bp->ntp_fltr_count--;
4868                         }
4869                         kfree(fltr);
4870                 }
4871         }
4872 }
4873
4874 static void bnxt_init_l2_fltr_tbl(struct bnxt *bp)
4875 {
4876         int i;
4877
4878         for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++)
4879                 INIT_HLIST_HEAD(&bp->l2_fltr_hash_tbl[i]);
4880         get_random_bytes(&bp->hash_seed, sizeof(bp->hash_seed));
4881 }
4882
4883 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4884 {
4885         bnxt_free_vnic_attributes(bp);
4886         bnxt_free_tx_rings(bp);
4887         bnxt_free_rx_rings(bp);
4888         bnxt_free_cp_rings(bp);
4889         bnxt_free_all_cp_arrays(bp);
4890         bnxt_free_ntp_fltrs(bp, false);
4891         bnxt_free_l2_filters(bp, false);
4892         if (irq_re_init) {
4893                 bnxt_free_ring_stats(bp);
4894                 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
4895                     test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4896                         bnxt_free_port_stats(bp);
4897                 bnxt_free_ring_grps(bp);
4898                 bnxt_free_vnics(bp);
4899                 kfree(bp->tx_ring_map);
4900                 bp->tx_ring_map = NULL;
4901                 kfree(bp->tx_ring);
4902                 bp->tx_ring = NULL;
4903                 kfree(bp->rx_ring);
4904                 bp->rx_ring = NULL;
4905                 kfree(bp->bnapi);
4906                 bp->bnapi = NULL;
4907         } else {
4908                 bnxt_clear_ring_indices(bp);
4909         }
4910 }
4911
4912 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4913 {
4914         int i, j, rc, size, arr_size;
4915         void *bnapi;
4916
4917         if (irq_re_init) {
4918                 /* Allocate bnapi mem pointer array and mem block for
4919                  * all queues
4920                  */
4921                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4922                                 bp->cp_nr_rings);
4923                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4924                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4925                 if (!bnapi)
4926                         return -ENOMEM;
4927
4928                 bp->bnapi = bnapi;
4929                 bnapi += arr_size;
4930                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4931                         bp->bnapi[i] = bnapi;
4932                         bp->bnapi[i]->index = i;
4933                         bp->bnapi[i]->bp = bp;
4934                         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
4935                                 struct bnxt_cp_ring_info *cpr =
4936                                         &bp->bnapi[i]->cp_ring;
4937
4938                                 cpr->cp_ring_struct.ring_mem.flags =
4939                                         BNXT_RMEM_RING_PTE_FLAG;
4940                         }
4941                 }
4942
4943                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4944                                       sizeof(struct bnxt_rx_ring_info),
4945                                       GFP_KERNEL);
4946                 if (!bp->rx_ring)
4947                         return -ENOMEM;
4948
4949                 for (i = 0; i < bp->rx_nr_rings; i++) {
4950                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4951
4952                         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
4953                                 rxr->rx_ring_struct.ring_mem.flags =
4954                                         BNXT_RMEM_RING_PTE_FLAG;
4955                                 rxr->rx_agg_ring_struct.ring_mem.flags =
4956                                         BNXT_RMEM_RING_PTE_FLAG;
4957                         } else {
4958                                 rxr->rx_cpr =  &bp->bnapi[i]->cp_ring;
4959                         }
4960                         rxr->bnapi = bp->bnapi[i];
4961                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4962                 }
4963
4964                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4965                                       sizeof(struct bnxt_tx_ring_info),
4966                                       GFP_KERNEL);
4967                 if (!bp->tx_ring)
4968                         return -ENOMEM;
4969
4970                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4971                                           GFP_KERNEL);
4972
4973                 if (!bp->tx_ring_map)
4974                         return -ENOMEM;
4975
4976                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4977                         j = 0;
4978                 else
4979                         j = bp->rx_nr_rings;
4980
4981                 for (i = 0; i < bp->tx_nr_rings; i++) {
4982                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4983                         struct bnxt_napi *bnapi2;
4984
4985                         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4986                                 txr->tx_ring_struct.ring_mem.flags =
4987                                         BNXT_RMEM_RING_PTE_FLAG;
4988                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4989                         if (i >= bp->tx_nr_rings_xdp) {
4990                                 int k = j + BNXT_RING_TO_TC_OFF(bp, i);
4991
4992                                 bnapi2 = bp->bnapi[k];
4993                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4994                                 txr->tx_napi_idx =
4995                                         BNXT_RING_TO_TC(bp, txr->txq_index);
4996                                 bnapi2->tx_ring[txr->tx_napi_idx] = txr;
4997                                 bnapi2->tx_int = bnxt_tx_int;
4998                         } else {
4999                                 bnapi2 = bp->bnapi[j];
5000                                 bnapi2->flags |= BNXT_NAPI_FLAG_XDP;
5001                                 bnapi2->tx_ring[0] = txr;
5002                                 bnapi2->tx_int = bnxt_tx_int_xdp;
5003                                 j++;
5004                         }
5005                         txr->bnapi = bnapi2;
5006                         if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
5007                                 txr->tx_cpr = &bnapi2->cp_ring;
5008                 }
5009
5010                 rc = bnxt_alloc_stats(bp);
5011                 if (rc)
5012                         goto alloc_mem_err;
5013                 bnxt_init_stats(bp);
5014
5015                 rc = bnxt_alloc_ntp_fltrs(bp);
5016                 if (rc)
5017                         goto alloc_mem_err;
5018
5019                 rc = bnxt_alloc_vnics(bp);
5020                 if (rc)
5021                         goto alloc_mem_err;
5022         }
5023
5024         rc = bnxt_alloc_all_cp_arrays(bp);
5025         if (rc)
5026                 goto alloc_mem_err;
5027
5028         bnxt_init_ring_struct(bp);
5029
5030         rc = bnxt_alloc_rx_rings(bp);
5031         if (rc)
5032                 goto alloc_mem_err;
5033
5034         rc = bnxt_alloc_tx_rings(bp);
5035         if (rc)
5036                 goto alloc_mem_err;
5037
5038         rc = bnxt_alloc_cp_rings(bp);
5039         if (rc)
5040                 goto alloc_mem_err;
5041
5042         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
5043                                   BNXT_VNIC_UCAST_FLAG;
5044         rc = bnxt_alloc_vnic_attributes(bp);
5045         if (rc)
5046                 goto alloc_mem_err;
5047         return 0;
5048
5049 alloc_mem_err:
5050         bnxt_free_mem(bp, true);
5051         return rc;
5052 }
5053
5054 static void bnxt_disable_int(struct bnxt *bp)
5055 {
5056         int i;
5057
5058         if (!bp->bnapi)
5059                 return;
5060
5061         for (i = 0; i < bp->cp_nr_rings; i++) {
5062                 struct bnxt_napi *bnapi = bp->bnapi[i];
5063                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5064                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5065
5066                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
5067                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5068         }
5069 }
5070
5071 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
5072 {
5073         struct bnxt_napi *bnapi = bp->bnapi[n];
5074         struct bnxt_cp_ring_info *cpr;
5075
5076         cpr = &bnapi->cp_ring;
5077         return cpr->cp_ring_struct.map_idx;
5078 }
5079
5080 static void bnxt_disable_int_sync(struct bnxt *bp)
5081 {
5082         int i;
5083
5084         if (!bp->irq_tbl)
5085                 return;
5086
5087         atomic_inc(&bp->intr_sem);
5088
5089         bnxt_disable_int(bp);
5090         for (i = 0; i < bp->cp_nr_rings; i++) {
5091                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
5092
5093                 synchronize_irq(bp->irq_tbl[map_idx].vector);
5094         }
5095 }
5096
5097 static void bnxt_enable_int(struct bnxt *bp)
5098 {
5099         int i;
5100
5101         atomic_set(&bp->intr_sem, 0);
5102         for (i = 0; i < bp->cp_nr_rings; i++) {
5103                 struct bnxt_napi *bnapi = bp->bnapi[i];
5104                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5105
5106                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
5107         }
5108 }
5109
5110 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
5111                             bool async_only)
5112 {
5113         DECLARE_BITMAP(async_events_bmap, 256);
5114         u32 *events = (u32 *)async_events_bmap;
5115         struct hwrm_func_drv_rgtr_output *resp;
5116         struct hwrm_func_drv_rgtr_input *req;
5117         u32 flags;
5118         int rc, i;
5119
5120         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR);
5121         if (rc)
5122                 return rc;
5123
5124         req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
5125                                    FUNC_DRV_RGTR_REQ_ENABLES_VER |
5126                                    FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
5127
5128         req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
5129         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
5130         if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
5131                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
5132         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
5133                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
5134                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
5135         req->flags = cpu_to_le32(flags);
5136         req->ver_maj_8b = DRV_VER_MAJ;
5137         req->ver_min_8b = DRV_VER_MIN;
5138         req->ver_upd_8b = DRV_VER_UPD;
5139         req->ver_maj = cpu_to_le16(DRV_VER_MAJ);
5140         req->ver_min = cpu_to_le16(DRV_VER_MIN);
5141         req->ver_upd = cpu_to_le16(DRV_VER_UPD);
5142
5143         if (BNXT_PF(bp)) {
5144                 u32 data[8];
5145                 int i;
5146
5147                 memset(data, 0, sizeof(data));
5148                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
5149                         u16 cmd = bnxt_vf_req_snif[i];
5150                         unsigned int bit, idx;
5151
5152                         idx = cmd / 32;
5153                         bit = cmd % 32;
5154                         data[idx] |= 1 << bit;
5155                 }
5156
5157                 for (i = 0; i < 8; i++)
5158                         req->vf_req_fwd[i] = cpu_to_le32(data[i]);
5159
5160                 req->enables |=
5161                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
5162         }
5163
5164         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
5165                 req->flags |= cpu_to_le32(
5166                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
5167
5168         memset(async_events_bmap, 0, sizeof(async_events_bmap));
5169         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
5170                 u16 event_id = bnxt_async_events_arr[i];
5171
5172                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
5173                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
5174                         continue;
5175                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE &&
5176                     !bp->ptp_cfg)
5177                         continue;
5178                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
5179         }
5180         if (bmap && bmap_size) {
5181                 for (i = 0; i < bmap_size; i++) {
5182                         if (test_bit(i, bmap))
5183                                 __set_bit(i, async_events_bmap);
5184                 }
5185         }
5186         for (i = 0; i < 8; i++)
5187                 req->async_event_fwd[i] |= cpu_to_le32(events[i]);
5188
5189         if (async_only)
5190                 req->enables =
5191                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
5192
5193         resp = hwrm_req_hold(bp, req);
5194         rc = hwrm_req_send(bp, req);
5195         if (!rc) {
5196                 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
5197                 if (resp->flags &
5198                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
5199                         bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
5200         }
5201         hwrm_req_drop(bp, req);
5202         return rc;
5203 }
5204
5205 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
5206 {
5207         struct hwrm_func_drv_unrgtr_input *req;
5208         int rc;
5209
5210         if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
5211                 return 0;
5212
5213         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR);
5214         if (rc)
5215                 return rc;
5216         return hwrm_req_send(bp, req);
5217 }
5218
5219 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa);
5220
5221 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
5222 {
5223         struct hwrm_tunnel_dst_port_free_input *req;
5224         int rc;
5225
5226         if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN &&
5227             bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID)
5228                 return 0;
5229         if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE &&
5230             bp->nge_fw_dst_port_id == INVALID_HW_RING_ID)
5231                 return 0;
5232
5233         rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE);
5234         if (rc)
5235                 return rc;
5236
5237         req->tunnel_type = tunnel_type;
5238
5239         switch (tunnel_type) {
5240         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
5241                 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
5242                 bp->vxlan_port = 0;
5243                 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
5244                 break;
5245         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
5246                 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
5247                 bp->nge_port = 0;
5248                 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
5249                 break;
5250         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE:
5251                 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_gpe_fw_dst_port_id);
5252                 bp->vxlan_gpe_port = 0;
5253                 bp->vxlan_gpe_fw_dst_port_id = INVALID_HW_RING_ID;
5254                 break;
5255         default:
5256                 break;
5257         }
5258
5259         rc = hwrm_req_send(bp, req);
5260         if (rc)
5261                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
5262                            rc);
5263         if (bp->flags & BNXT_FLAG_TPA)
5264                 bnxt_set_tpa(bp, true);
5265         return rc;
5266 }
5267
5268 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
5269                                            u8 tunnel_type)
5270 {
5271         struct hwrm_tunnel_dst_port_alloc_output *resp;
5272         struct hwrm_tunnel_dst_port_alloc_input *req;
5273         int rc;
5274
5275         rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC);
5276         if (rc)
5277                 return rc;
5278
5279         req->tunnel_type = tunnel_type;
5280         req->tunnel_dst_port_val = port;
5281
5282         resp = hwrm_req_hold(bp, req);
5283         rc = hwrm_req_send(bp, req);
5284         if (rc) {
5285                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
5286                            rc);
5287                 goto err_out;
5288         }
5289
5290         switch (tunnel_type) {
5291         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
5292                 bp->vxlan_port = port;
5293                 bp->vxlan_fw_dst_port_id =
5294                         le16_to_cpu(resp->tunnel_dst_port_id);
5295                 break;
5296         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
5297                 bp->nge_port = port;
5298                 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
5299                 break;
5300         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE:
5301                 bp->vxlan_gpe_port = port;
5302                 bp->vxlan_gpe_fw_dst_port_id =
5303                         le16_to_cpu(resp->tunnel_dst_port_id);
5304                 break;
5305         default:
5306                 break;
5307         }
5308         if (bp->flags & BNXT_FLAG_TPA)
5309                 bnxt_set_tpa(bp, true);
5310
5311 err_out:
5312         hwrm_req_drop(bp, req);
5313         return rc;
5314 }
5315
5316 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
5317 {
5318         struct hwrm_cfa_l2_set_rx_mask_input *req;
5319         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5320         int rc;
5321
5322         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK);
5323         if (rc)
5324                 return rc;
5325
5326         req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5327         if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
5328                 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
5329                 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
5330         }
5331         req->mask = cpu_to_le32(vnic->rx_mask);
5332         return hwrm_req_send_silent(bp, req);
5333 }
5334
5335 void bnxt_del_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5336 {
5337         if (!atomic_dec_and_test(&fltr->refcnt))
5338                 return;
5339         spin_lock_bh(&bp->ntp_fltr_lock);
5340         if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) {
5341                 spin_unlock_bh(&bp->ntp_fltr_lock);
5342                 return;
5343         }
5344         hlist_del_rcu(&fltr->base.hash);
5345         if (fltr->base.flags) {
5346                 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
5347                 bp->ntp_fltr_count--;
5348         }
5349         spin_unlock_bh(&bp->ntp_fltr_lock);
5350         kfree_rcu(fltr, base.rcu);
5351 }
5352
5353 static struct bnxt_l2_filter *__bnxt_lookup_l2_filter(struct bnxt *bp,
5354                                                       struct bnxt_l2_key *key,
5355                                                       u32 idx)
5356 {
5357         struct hlist_head *head = &bp->l2_fltr_hash_tbl[idx];
5358         struct bnxt_l2_filter *fltr;
5359
5360         hlist_for_each_entry_rcu(fltr, head, base.hash) {
5361                 struct bnxt_l2_key *l2_key = &fltr->l2_key;
5362
5363                 if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) &&
5364                     l2_key->vlan == key->vlan)
5365                         return fltr;
5366         }
5367         return NULL;
5368 }
5369
5370 static struct bnxt_l2_filter *bnxt_lookup_l2_filter(struct bnxt *bp,
5371                                                     struct bnxt_l2_key *key,
5372                                                     u32 idx)
5373 {
5374         struct bnxt_l2_filter *fltr = NULL;
5375
5376         rcu_read_lock();
5377         fltr = __bnxt_lookup_l2_filter(bp, key, idx);
5378         if (fltr)
5379                 atomic_inc(&fltr->refcnt);
5380         rcu_read_unlock();
5381         return fltr;
5382 }
5383
5384 #define BNXT_IPV4_4TUPLE(bp, fkeys)                                     \
5385         (((fkeys)->basic.ip_proto == IPPROTO_TCP &&                     \
5386           (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4) ||  \
5387          ((fkeys)->basic.ip_proto == IPPROTO_UDP &&                     \
5388           (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4))
5389
5390 #define BNXT_IPV6_4TUPLE(bp, fkeys)                                     \
5391         (((fkeys)->basic.ip_proto == IPPROTO_TCP &&                     \
5392           (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6) ||  \
5393          ((fkeys)->basic.ip_proto == IPPROTO_UDP &&                     \
5394           (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6))
5395
5396 static u32 bnxt_get_rss_flow_tuple_len(struct bnxt *bp, struct flow_keys *fkeys)
5397 {
5398         if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
5399                 if (BNXT_IPV4_4TUPLE(bp, fkeys))
5400                         return sizeof(fkeys->addrs.v4addrs) +
5401                                sizeof(fkeys->ports);
5402
5403                 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4)
5404                         return sizeof(fkeys->addrs.v4addrs);
5405         }
5406
5407         if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) {
5408                 if (BNXT_IPV6_4TUPLE(bp, fkeys))
5409                         return sizeof(fkeys->addrs.v6addrs) +
5410                                sizeof(fkeys->ports);
5411
5412                 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6)
5413                         return sizeof(fkeys->addrs.v6addrs);
5414         }
5415
5416         return 0;
5417 }
5418
5419 static u32 bnxt_toeplitz(struct bnxt *bp, struct flow_keys *fkeys,
5420                          const unsigned char *key)
5421 {
5422         u64 prefix = bp->toeplitz_prefix, hash = 0;
5423         struct bnxt_ipv4_tuple tuple4;
5424         struct bnxt_ipv6_tuple tuple6;
5425         int i, j, len = 0;
5426         u8 *four_tuple;
5427
5428         len = bnxt_get_rss_flow_tuple_len(bp, fkeys);
5429         if (!len)
5430                 return 0;
5431
5432         if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
5433                 tuple4.v4addrs = fkeys->addrs.v4addrs;
5434                 tuple4.ports = fkeys->ports;
5435                 four_tuple = (unsigned char *)&tuple4;
5436         } else {
5437                 tuple6.v6addrs = fkeys->addrs.v6addrs;
5438                 tuple6.ports = fkeys->ports;
5439                 four_tuple = (unsigned char *)&tuple6;
5440         }
5441
5442         for (i = 0, j = 8; i < len; i++, j++) {
5443                 u8 byte = four_tuple[i];
5444                 int bit;
5445
5446                 for (bit = 0; bit < 8; bit++, prefix <<= 1, byte <<= 1) {
5447                         if (byte & 0x80)
5448                                 hash ^= prefix;
5449                 }
5450                 prefix |= (j < HW_HASH_KEY_SIZE) ? key[j] : 0;
5451         }
5452
5453         /* The valid part of the hash is in the upper 32 bits. */
5454         return (hash >> 32) & BNXT_NTP_FLTR_HASH_MASK;
5455 }
5456
5457 #ifdef CONFIG_RFS_ACCEL
5458 static struct bnxt_l2_filter *
5459 bnxt_lookup_l2_filter_from_key(struct bnxt *bp, struct bnxt_l2_key *key)
5460 {
5461         struct bnxt_l2_filter *fltr;
5462         u32 idx;
5463
5464         idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5465               BNXT_L2_FLTR_HASH_MASK;
5466         fltr = bnxt_lookup_l2_filter(bp, key, idx);
5467         return fltr;
5468 }
5469 #endif
5470
5471 static int bnxt_init_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr,
5472                                struct bnxt_l2_key *key, u32 idx)
5473 {
5474         struct hlist_head *head;
5475
5476         ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr);
5477         fltr->l2_key.vlan = key->vlan;
5478         fltr->base.type = BNXT_FLTR_TYPE_L2;
5479         if (fltr->base.flags) {
5480                 int bit_id;
5481
5482                 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
5483                                                  BNXT_MAX_FLTR, 0);
5484                 if (bit_id < 0)
5485                         return -ENOMEM;
5486                 fltr->base.sw_id = (u16)bit_id;
5487         }
5488         head = &bp->l2_fltr_hash_tbl[idx];
5489         hlist_add_head_rcu(&fltr->base.hash, head);
5490         set_bit(BNXT_FLTR_INSERTED, &fltr->base.state);
5491         atomic_set(&fltr->refcnt, 1);
5492         return 0;
5493 }
5494
5495 static struct bnxt_l2_filter *bnxt_alloc_l2_filter(struct bnxt *bp,
5496                                                    struct bnxt_l2_key *key,
5497                                                    gfp_t gfp)
5498 {
5499         struct bnxt_l2_filter *fltr;
5500         u32 idx;
5501         int rc;
5502
5503         idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5504               BNXT_L2_FLTR_HASH_MASK;
5505         fltr = bnxt_lookup_l2_filter(bp, key, idx);
5506         if (fltr)
5507                 return fltr;
5508
5509         fltr = kzalloc(sizeof(*fltr), gfp);
5510         if (!fltr)
5511                 return ERR_PTR(-ENOMEM);
5512         spin_lock_bh(&bp->ntp_fltr_lock);
5513         rc = bnxt_init_l2_filter(bp, fltr, key, idx);
5514         spin_unlock_bh(&bp->ntp_fltr_lock);
5515         if (rc) {
5516                 bnxt_del_l2_filter(bp, fltr);
5517                 fltr = ERR_PTR(rc);
5518         }
5519         return fltr;
5520 }
5521
5522 static u16 bnxt_vf_target_id(struct bnxt_pf_info *pf, u16 vf_idx)
5523 {
5524 #ifdef CONFIG_BNXT_SRIOV
5525         struct bnxt_vf_info *vf = &pf->vf[vf_idx];
5526
5527         return vf->fw_fid;
5528 #else
5529         return INVALID_HW_RING_ID;
5530 #endif
5531 }
5532
5533 int bnxt_hwrm_l2_filter_free(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5534 {
5535         struct hwrm_cfa_l2_filter_free_input *req;
5536         u16 target_id = 0xffff;
5537         int rc;
5538
5539         if (fltr->base.flags & BNXT_ACT_FUNC_DST) {
5540                 struct bnxt_pf_info *pf = &bp->pf;
5541
5542                 if (fltr->base.vf_idx >= pf->active_vfs)
5543                         return -EINVAL;
5544
5545                 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx);
5546                 if (target_id == INVALID_HW_RING_ID)
5547                         return -EINVAL;
5548         }
5549
5550         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
5551         if (rc)
5552                 return rc;
5553
5554         req->target_id = cpu_to_le16(target_id);
5555         req->l2_filter_id = fltr->base.filter_id;
5556         return hwrm_req_send(bp, req);
5557 }
5558
5559 int bnxt_hwrm_l2_filter_alloc(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5560 {
5561         struct hwrm_cfa_l2_filter_alloc_output *resp;
5562         struct hwrm_cfa_l2_filter_alloc_input *req;
5563         u16 target_id = 0xffff;
5564         int rc;
5565
5566         if (fltr->base.flags & BNXT_ACT_FUNC_DST) {
5567                 struct bnxt_pf_info *pf = &bp->pf;
5568
5569                 if (fltr->base.vf_idx >= pf->active_vfs)
5570                         return -EINVAL;
5571
5572                 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx);
5573         }
5574         rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC);
5575         if (rc)
5576                 return rc;
5577
5578         req->target_id = cpu_to_le16(target_id);
5579         req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
5580
5581         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
5582                 req->flags |=
5583                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
5584         req->dst_id = cpu_to_le16(fltr->base.fw_vnic_id);
5585         req->enables =
5586                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
5587                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
5588                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
5589         ether_addr_copy(req->l2_addr, fltr->l2_key.dst_mac_addr);
5590         eth_broadcast_addr(req->l2_addr_mask);
5591
5592         if (fltr->l2_key.vlan) {
5593                 req->enables |=
5594                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN |
5595                                 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK |
5596                                 CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS);
5597                 req->num_vlans = 1;
5598                 req->l2_ivlan = cpu_to_le16(fltr->l2_key.vlan);
5599                 req->l2_ivlan_mask = cpu_to_le16(0xfff);
5600         }
5601
5602         resp = hwrm_req_hold(bp, req);
5603         rc = hwrm_req_send(bp, req);
5604         if (!rc) {
5605                 fltr->base.filter_id = resp->l2_filter_id;
5606                 set_bit(BNXT_FLTR_VALID, &fltr->base.state);
5607         }
5608         hwrm_req_drop(bp, req);
5609         return rc;
5610 }
5611
5612 int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
5613                                      struct bnxt_ntuple_filter *fltr)
5614 {
5615         struct hwrm_cfa_ntuple_filter_free_input *req;
5616         int rc;
5617
5618         set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state);
5619         rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE);
5620         if (rc)
5621                 return rc;
5622
5623         req->ntuple_filter_id = fltr->base.filter_id;
5624         return hwrm_req_send(bp, req);
5625 }
5626
5627 #define BNXT_NTP_FLTR_FLAGS                                     \
5628         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
5629          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
5630          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
5631          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
5632          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
5633          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
5634          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
5635          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
5636          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
5637          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
5638          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
5639          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
5640          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
5641
5642 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
5643                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
5644
5645 void bnxt_fill_ipv6_mask(__be32 mask[4])
5646 {
5647         int i;
5648
5649         for (i = 0; i < 4; i++)
5650                 mask[i] = cpu_to_be32(~0);
5651 }
5652
5653 int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
5654                                       struct bnxt_ntuple_filter *fltr)
5655 {
5656         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
5657         struct hwrm_cfa_ntuple_filter_alloc_input *req;
5658         struct flow_keys *keys = &fltr->fkeys;
5659         struct bnxt_l2_filter *l2_fltr;
5660         struct bnxt_vnic_info *vnic;
5661         u32 flags = 0;
5662         int rc;
5663
5664         rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC);
5665         if (rc)
5666                 return rc;
5667
5668         l2_fltr = fltr->l2_fltr;
5669         req->l2_filter_id = l2_fltr->base.filter_id;
5670
5671
5672         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
5673                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
5674                 req->dst_id = cpu_to_le16(fltr->base.rxq);
5675         } else {
5676                 vnic = &bp->vnic_info[fltr->base.rxq + 1];
5677                 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
5678         }
5679         req->flags = cpu_to_le32(flags);
5680         req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
5681
5682         req->ethertype = htons(ETH_P_IP);
5683         req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
5684         req->ip_protocol = keys->basic.ip_proto;
5685
5686         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
5687                 req->ethertype = htons(ETH_P_IPV6);
5688                 req->ip_addr_type =
5689                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
5690                 if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_IP) {
5691                         *(struct in6_addr *)&req->src_ipaddr[0] =
5692                                 keys->addrs.v6addrs.src;
5693                         bnxt_fill_ipv6_mask(req->src_ipaddr_mask);
5694                 }
5695                 if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_DST_IP) {
5696                         *(struct in6_addr *)&req->dst_ipaddr[0] =
5697                                 keys->addrs.v6addrs.dst;
5698                         bnxt_fill_ipv6_mask(req->dst_ipaddr_mask);
5699                 }
5700         } else {
5701                 if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_IP) {
5702                         req->src_ipaddr[0] = keys->addrs.v4addrs.src;
5703                         req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
5704                 }
5705                 if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_DST_IP) {
5706                         req->dst_ipaddr[0] = keys->addrs.v4addrs.dst;
5707                         req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
5708                 }
5709         }
5710         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
5711                 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
5712                 req->tunnel_type =
5713                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
5714         }
5715
5716         if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_PORT) {
5717                 req->src_port = keys->ports.src;
5718                 req->src_port_mask = cpu_to_be16(0xffff);
5719         }
5720         if (fltr->ntuple_flags & BNXT_NTUPLE_MATCH_DST_PORT) {
5721                 req->dst_port = keys->ports.dst;
5722                 req->dst_port_mask = cpu_to_be16(0xffff);
5723         }
5724
5725         resp = hwrm_req_hold(bp, req);
5726         rc = hwrm_req_send(bp, req);
5727         if (!rc)
5728                 fltr->base.filter_id = resp->ntuple_filter_id;
5729         hwrm_req_drop(bp, req);
5730         return rc;
5731 }
5732
5733 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
5734                                      const u8 *mac_addr)
5735 {
5736         struct bnxt_l2_filter *fltr;
5737         struct bnxt_l2_key key;
5738         int rc;
5739
5740         ether_addr_copy(key.dst_mac_addr, mac_addr);
5741         key.vlan = 0;
5742         fltr = bnxt_alloc_l2_filter(bp, &key, GFP_KERNEL);
5743         if (IS_ERR(fltr))
5744                 return PTR_ERR(fltr);
5745
5746         fltr->base.fw_vnic_id = bp->vnic_info[vnic_id].fw_vnic_id;
5747         rc = bnxt_hwrm_l2_filter_alloc(bp, fltr);
5748         if (rc)
5749                 bnxt_del_l2_filter(bp, fltr);
5750         else
5751                 bp->vnic_info[vnic_id].l2_filters[idx] = fltr;
5752         return rc;
5753 }
5754
5755 static void bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
5756 {
5757         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
5758
5759         /* Any associated ntuple filters will also be cleared by firmware. */
5760         for (i = 0; i < num_of_vnics; i++) {
5761                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5762
5763                 for (j = 0; j < vnic->uc_filter_count; j++) {
5764                         struct bnxt_l2_filter *fltr = vnic->l2_filters[j];
5765
5766                         bnxt_hwrm_l2_filter_free(bp, fltr);
5767                         bnxt_del_l2_filter(bp, fltr);
5768                 }
5769                 vnic->uc_filter_count = 0;
5770         }
5771 }
5772
5773 #define BNXT_DFLT_TUNL_TPA_BMAP                         \
5774         (VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE |       \
5775          VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV4 |      \
5776          VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV6)
5777
5778 static void bnxt_hwrm_vnic_update_tunl_tpa(struct bnxt *bp,
5779                                            struct hwrm_vnic_tpa_cfg_input *req)
5780 {
5781         u32 tunl_tpa_bmap = BNXT_DFLT_TUNL_TPA_BMAP;
5782
5783         if (!(bp->fw_cap & BNXT_FW_CAP_VNIC_TUNNEL_TPA))
5784                 return;
5785
5786         if (bp->vxlan_port)
5787                 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN;
5788         if (bp->vxlan_gpe_port)
5789                 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_GPE;
5790         if (bp->nge_port)
5791                 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GENEVE;
5792
5793         req->enables |= cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_TNL_TPA_EN);
5794         req->tnl_tpa_en_bitmap = cpu_to_le32(tunl_tpa_bmap);
5795 }
5796
5797 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
5798 {
5799         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5800         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
5801         struct hwrm_vnic_tpa_cfg_input *req;
5802         int rc;
5803
5804         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
5805                 return 0;
5806
5807         rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG);
5808         if (rc)
5809                 return rc;
5810
5811         if (tpa_flags) {
5812                 u16 mss = bp->dev->mtu - 40;
5813                 u32 nsegs, n, segs = 0, flags;
5814
5815                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
5816                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
5817                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
5818                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
5819                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
5820                 if (tpa_flags & BNXT_FLAG_GRO)
5821                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
5822
5823                 req->flags = cpu_to_le32(flags);
5824
5825                 req->enables =
5826                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
5827                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
5828                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
5829
5830                 /* Number of segs are log2 units, and first packet is not
5831                  * included as part of this units.
5832                  */
5833                 if (mss <= BNXT_RX_PAGE_SIZE) {
5834                         n = BNXT_RX_PAGE_SIZE / mss;
5835                         nsegs = (MAX_SKB_FRAGS - 1) * n;
5836                 } else {
5837                         n = mss / BNXT_RX_PAGE_SIZE;
5838                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
5839                                 n++;
5840                         nsegs = (MAX_SKB_FRAGS - n) / n;
5841                 }
5842
5843                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
5844                         segs = MAX_TPA_SEGS_P5;
5845                         max_aggs = bp->max_tpa;
5846                 } else {
5847                         segs = ilog2(nsegs);
5848                 }
5849                 req->max_agg_segs = cpu_to_le16(segs);
5850                 req->max_aggs = cpu_to_le16(max_aggs);
5851
5852                 req->min_agg_len = cpu_to_le32(512);
5853                 bnxt_hwrm_vnic_update_tunl_tpa(bp, req);
5854         }
5855         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5856
5857         return hwrm_req_send(bp, req);
5858 }
5859
5860 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
5861 {
5862         struct bnxt_ring_grp_info *grp_info;
5863
5864         grp_info = &bp->grp_info[ring->grp_idx];
5865         return grp_info->cp_fw_ring_id;
5866 }
5867
5868 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
5869 {
5870         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5871                 return rxr->rx_cpr->cp_ring_struct.fw_ring_id;
5872         else
5873                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
5874 }
5875
5876 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
5877 {
5878         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5879                 return txr->tx_cpr->cp_ring_struct.fw_ring_id;
5880         else
5881                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
5882 }
5883
5884 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5885 {
5886         int entries;
5887
5888         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5889                 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5890         else
5891                 entries = HW_HASH_INDEX_SIZE;
5892
5893         bp->rss_indir_tbl_entries = entries;
5894         bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5895                                           GFP_KERNEL);
5896         if (!bp->rss_indir_tbl)
5897                 return -ENOMEM;
5898         return 0;
5899 }
5900
5901 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5902 {
5903         u16 max_rings, max_entries, pad, i;
5904
5905         if (!bp->rx_nr_rings)
5906                 return;
5907
5908         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5909                 max_rings = bp->rx_nr_rings - 1;
5910         else
5911                 max_rings = bp->rx_nr_rings;
5912
5913         max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5914
5915         for (i = 0; i < max_entries; i++)
5916                 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5917
5918         pad = bp->rss_indir_tbl_entries - max_entries;
5919         if (pad)
5920                 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5921 }
5922
5923 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5924 {
5925         u16 i, tbl_size, max_ring = 0;
5926
5927         if (!bp->rss_indir_tbl)
5928                 return 0;
5929
5930         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5931         for (i = 0; i < tbl_size; i++)
5932                 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5933         return max_ring;
5934 }
5935
5936 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5937 {
5938         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
5939                 if (!rx_rings)
5940                         return 0;
5941                 return bnxt_calc_nr_ring_pages(rx_rings - 1,
5942                                                BNXT_RSS_TABLE_ENTRIES_P5);
5943         }
5944         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5945                 return 2;
5946         return 1;
5947 }
5948
5949 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5950 {
5951         bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5952         u16 i, j;
5953
5954         /* Fill the RSS indirection table with ring group ids */
5955         for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5956                 if (!no_rss)
5957                         j = bp->rss_indir_tbl[i];
5958                 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5959         }
5960 }
5961
5962 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5963                                     struct bnxt_vnic_info *vnic)
5964 {
5965         __le16 *ring_tbl = vnic->rss_table;
5966         struct bnxt_rx_ring_info *rxr;
5967         u16 tbl_size, i;
5968
5969         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5970
5971         for (i = 0; i < tbl_size; i++) {
5972                 u16 ring_id, j;
5973
5974                 j = bp->rss_indir_tbl[i];
5975                 rxr = &bp->rx_ring[j];
5976
5977                 ring_id = rxr->rx_ring_struct.fw_ring_id;
5978                 *ring_tbl++ = cpu_to_le16(ring_id);
5979                 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5980                 *ring_tbl++ = cpu_to_le16(ring_id);
5981         }
5982 }
5983
5984 static void
5985 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req,
5986                          struct bnxt_vnic_info *vnic)
5987 {
5988         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5989                 bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5990         else
5991                 bnxt_fill_hw_rss_tbl(bp, vnic);
5992
5993         if (bp->rss_hash_delta) {
5994                 req->hash_type = cpu_to_le32(bp->rss_hash_delta);
5995                 if (bp->rss_hash_cfg & bp->rss_hash_delta)
5996                         req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE;
5997                 else
5998                         req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE;
5999         } else {
6000                 req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
6001         }
6002         req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
6003         req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
6004         req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
6005 }
6006
6007 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
6008 {
6009         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6010         struct hwrm_vnic_rss_cfg_input *req;
6011         int rc;
6012
6013         if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) ||
6014             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
6015                 return 0;
6016
6017         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
6018         if (rc)
6019                 return rc;
6020
6021         if (set_rss)
6022                 __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
6023         req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6024         return hwrm_req_send(bp, req);
6025 }
6026
6027 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
6028 {
6029         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6030         struct hwrm_vnic_rss_cfg_input *req;
6031         dma_addr_t ring_tbl_map;
6032         u32 i, nr_ctxs;
6033         int rc;
6034
6035         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
6036         if (rc)
6037                 return rc;
6038
6039         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6040         if (!set_rss)
6041                 return hwrm_req_send(bp, req);
6042
6043         __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
6044         ring_tbl_map = vnic->rss_table_dma_addr;
6045         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
6046
6047         hwrm_req_hold(bp, req);
6048         for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
6049                 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
6050                 req->ring_table_pair_index = i;
6051                 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
6052                 rc = hwrm_req_send(bp, req);
6053                 if (rc)
6054                         goto exit;
6055         }
6056
6057 exit:
6058         hwrm_req_drop(bp, req);
6059         return rc;
6060 }
6061
6062 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp)
6063 {
6064         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6065         struct hwrm_vnic_rss_qcfg_output *resp;
6066         struct hwrm_vnic_rss_qcfg_input *req;
6067
6068         if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG))
6069                 return;
6070
6071         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6072         /* all contexts configured to same hash_type, zero always exists */
6073         req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6074         resp = hwrm_req_hold(bp, req);
6075         if (!hwrm_req_send(bp, req)) {
6076                 bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg;
6077                 bp->rss_hash_delta = 0;
6078         }
6079         hwrm_req_drop(bp, req);
6080 }
6081
6082 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
6083 {
6084         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6085         struct hwrm_vnic_plcmodes_cfg_input *req;
6086         int rc;
6087
6088         rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG);
6089         if (rc)
6090                 return rc;
6091
6092         req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
6093         req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
6094
6095         if (BNXT_RX_PAGE_MODE(bp)) {
6096                 req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
6097         } else {
6098                 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
6099                                           VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
6100                 req->enables |=
6101                         cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
6102                 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
6103                 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
6104         }
6105         req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
6106         return hwrm_req_send(bp, req);
6107 }
6108
6109 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
6110                                         u16 ctx_idx)
6111 {
6112         struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
6113
6114         if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
6115                 return;
6116
6117         req->rss_cos_lb_ctx_id =
6118                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
6119
6120         hwrm_req_send(bp, req);
6121         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
6122 }
6123
6124 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
6125 {
6126         int i, j;
6127
6128         for (i = 0; i < bp->nr_vnics; i++) {
6129                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
6130
6131                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
6132                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
6133                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
6134                 }
6135         }
6136         bp->rsscos_nr_ctxs = 0;
6137 }
6138
6139 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
6140 {
6141         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
6142         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
6143         int rc;
6144
6145         rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
6146         if (rc)
6147                 return rc;
6148
6149         resp = hwrm_req_hold(bp, req);
6150         rc = hwrm_req_send(bp, req);
6151         if (!rc)
6152                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
6153                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
6154         hwrm_req_drop(bp, req);
6155
6156         return rc;
6157 }
6158
6159 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
6160 {
6161         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
6162                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
6163         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
6164 }
6165
6166 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
6167 {
6168         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6169         struct hwrm_vnic_cfg_input *req;
6170         unsigned int ring = 0, grp_idx;
6171         u16 def_vlan = 0;
6172         int rc;
6173
6174         rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG);
6175         if (rc)
6176                 return rc;
6177
6178         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6179                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
6180
6181                 req->default_rx_ring_id =
6182                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
6183                 req->default_cmpl_ring_id =
6184                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
6185                 req->enables =
6186                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
6187                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
6188                 goto vnic_mru;
6189         }
6190         req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
6191         /* Only RSS support for now TBD: COS & LB */
6192         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
6193                 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6194                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
6195                                            VNIC_CFG_REQ_ENABLES_MRU);
6196         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
6197                 req->rss_rule =
6198                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
6199                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
6200                                            VNIC_CFG_REQ_ENABLES_MRU);
6201                 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
6202         } else {
6203                 req->rss_rule = cpu_to_le16(0xffff);
6204         }
6205
6206         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
6207             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
6208                 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
6209                 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
6210         } else {
6211                 req->cos_rule = cpu_to_le16(0xffff);
6212         }
6213
6214         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
6215                 ring = 0;
6216         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
6217                 ring = vnic_id - 1;
6218         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
6219                 ring = bp->rx_nr_rings - 1;
6220
6221         grp_idx = bp->rx_ring[ring].bnapi->index;
6222         req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
6223         req->lb_rule = cpu_to_le16(0xffff);
6224 vnic_mru:
6225         req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
6226
6227         req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6228 #ifdef CONFIG_BNXT_SRIOV
6229         if (BNXT_VF(bp))
6230                 def_vlan = bp->vf.vlan;
6231 #endif
6232         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
6233                 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
6234         if (!vnic_id && bnxt_ulp_registered(bp->edev))
6235                 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
6236
6237         return hwrm_req_send(bp, req);
6238 }
6239
6240 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
6241 {
6242         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
6243                 struct hwrm_vnic_free_input *req;
6244
6245                 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE))
6246                         return;
6247
6248                 req->vnic_id =
6249                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
6250
6251                 hwrm_req_send(bp, req);
6252                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
6253         }
6254 }
6255
6256 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
6257 {
6258         u16 i;
6259
6260         for (i = 0; i < bp->nr_vnics; i++)
6261                 bnxt_hwrm_vnic_free_one(bp, i);
6262 }
6263
6264 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
6265                                 unsigned int start_rx_ring_idx,
6266                                 unsigned int nr_rings)
6267 {
6268         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
6269         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6270         struct hwrm_vnic_alloc_output *resp;
6271         struct hwrm_vnic_alloc_input *req;
6272         int rc;
6273
6274         rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC);
6275         if (rc)
6276                 return rc;
6277
6278         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6279                 goto vnic_no_ring_grps;
6280
6281         /* map ring groups to this vnic */
6282         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
6283                 grp_idx = bp->rx_ring[i].bnapi->index;
6284                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
6285                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
6286                                    j, nr_rings);
6287                         break;
6288                 }
6289                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
6290         }
6291
6292 vnic_no_ring_grps:
6293         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
6294                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
6295         if (vnic_id == 0)
6296                 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
6297
6298         resp = hwrm_req_hold(bp, req);
6299         rc = hwrm_req_send(bp, req);
6300         if (!rc)
6301                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
6302         hwrm_req_drop(bp, req);
6303         return rc;
6304 }
6305
6306 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
6307 {
6308         struct hwrm_vnic_qcaps_output *resp;
6309         struct hwrm_vnic_qcaps_input *req;
6310         int rc;
6311
6312         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
6313         bp->flags &= ~BNXT_FLAG_ROCE_MIRROR_CAP;
6314         bp->rss_cap &= ~BNXT_RSS_CAP_NEW_RSS_CAP;
6315         if (bp->hwrm_spec_code < 0x10600)
6316                 return 0;
6317
6318         rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS);
6319         if (rc)
6320                 return rc;
6321
6322         resp = hwrm_req_hold(bp, req);
6323         rc = hwrm_req_send(bp, req);
6324         if (!rc) {
6325                 u32 flags = le32_to_cpu(resp->flags);
6326
6327                 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
6328                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
6329                         bp->rss_cap |= BNXT_RSS_CAP_NEW_RSS_CAP;
6330                 if (flags &
6331                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
6332                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
6333
6334                 /* Older P5 fw before EXT_HW_STATS support did not set
6335                  * VLAN_STRIP_CAP properly.
6336                  */
6337                 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
6338                     (BNXT_CHIP_P5(bp) &&
6339                      !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
6340                         bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
6341                 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP)
6342                         bp->rss_cap |= BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA;
6343                 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_PROF_TCAM_MODE_ENABLED)
6344                         bp->rss_cap |= BNXT_RSS_CAP_RSS_TCAM;
6345                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
6346                 if (bp->max_tpa_v2) {
6347                         if (BNXT_CHIP_P5(bp))
6348                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
6349                         else
6350                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P7;
6351                 }
6352                 if (flags & VNIC_QCAPS_RESP_FLAGS_HW_TUNNEL_TPA_CAP)
6353                         bp->fw_cap |= BNXT_FW_CAP_VNIC_TUNNEL_TPA;
6354         }
6355         hwrm_req_drop(bp, req);
6356         return rc;
6357 }
6358
6359 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
6360 {
6361         struct hwrm_ring_grp_alloc_output *resp;
6362         struct hwrm_ring_grp_alloc_input *req;
6363         int rc;
6364         u16 i;
6365
6366         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6367                 return 0;
6368
6369         rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC);
6370         if (rc)
6371                 return rc;
6372
6373         resp = hwrm_req_hold(bp, req);
6374         for (i = 0; i < bp->rx_nr_rings; i++) {
6375                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
6376
6377                 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
6378                 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
6379                 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
6380                 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
6381
6382                 rc = hwrm_req_send(bp, req);
6383
6384                 if (rc)
6385                         break;
6386
6387                 bp->grp_info[grp_idx].fw_grp_id =
6388                         le32_to_cpu(resp->ring_group_id);
6389         }
6390         hwrm_req_drop(bp, req);
6391         return rc;
6392 }
6393
6394 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
6395 {
6396         struct hwrm_ring_grp_free_input *req;
6397         u16 i;
6398
6399         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
6400                 return;
6401
6402         if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE))
6403                 return;
6404
6405         hwrm_req_hold(bp, req);
6406         for (i = 0; i < bp->cp_nr_rings; i++) {
6407                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
6408                         continue;
6409                 req->ring_group_id =
6410                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
6411
6412                 hwrm_req_send(bp, req);
6413                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
6414         }
6415         hwrm_req_drop(bp, req);
6416 }
6417
6418 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
6419                                     struct bnxt_ring_struct *ring,
6420                                     u32 ring_type, u32 map_index)
6421 {
6422         struct hwrm_ring_alloc_output *resp;
6423         struct hwrm_ring_alloc_input *req;
6424         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
6425         struct bnxt_ring_grp_info *grp_info;
6426         int rc, err = 0;
6427         u16 ring_id;
6428
6429         rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC);
6430         if (rc)
6431                 goto exit;
6432
6433         req->enables = 0;
6434         if (rmem->nr_pages > 1) {
6435                 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
6436                 /* Page size is in log2 units */
6437                 req->page_size = BNXT_PAGE_SHIFT;
6438                 req->page_tbl_depth = 1;
6439         } else {
6440                 req->page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
6441         }
6442         req->fbo = 0;
6443         /* Association of ring index with doorbell index and MSIX number */
6444         req->logical_id = cpu_to_le16(map_index);
6445
6446         switch (ring_type) {
6447         case HWRM_RING_ALLOC_TX: {
6448                 struct bnxt_tx_ring_info *txr;
6449
6450                 txr = container_of(ring, struct bnxt_tx_ring_info,
6451                                    tx_ring_struct);
6452                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
6453                 /* Association of transmit ring with completion ring */
6454                 grp_info = &bp->grp_info[ring->grp_idx];
6455                 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
6456                 req->length = cpu_to_le32(bp->tx_ring_mask + 1);
6457                 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6458                 req->queue_id = cpu_to_le16(ring->queue_id);
6459                 if (bp->flags & BNXT_FLAG_TX_COAL_CMPL)
6460                         req->cmpl_coal_cnt =
6461                                 RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_64;
6462                 break;
6463         }
6464         case HWRM_RING_ALLOC_RX:
6465                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
6466                 req->length = cpu_to_le32(bp->rx_ring_mask + 1);
6467                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6468                         u16 flags = 0;
6469
6470                         /* Association of rx ring with stats context */
6471                         grp_info = &bp->grp_info[ring->grp_idx];
6472                         req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
6473                         req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6474                         req->enables |= cpu_to_le32(
6475                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
6476                         if (NET_IP_ALIGN == 2)
6477                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
6478                         req->flags = cpu_to_le16(flags);
6479                 }
6480                 break;
6481         case HWRM_RING_ALLOC_AGG:
6482                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6483                         req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
6484                         /* Association of agg ring with rx ring */
6485                         grp_info = &bp->grp_info[ring->grp_idx];
6486                         req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
6487                         req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
6488                         req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6489                         req->enables |= cpu_to_le32(
6490                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
6491                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
6492                 } else {
6493                         req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
6494                 }
6495                 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
6496                 break;
6497         case HWRM_RING_ALLOC_CMPL:
6498                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
6499                 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
6500                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6501                         /* Association of cp ring with nq */
6502                         grp_info = &bp->grp_info[map_index];
6503                         req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
6504                         req->cq_handle = cpu_to_le64(ring->handle);
6505                         req->enables |= cpu_to_le32(
6506                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
6507                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
6508                         req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
6509                 }
6510                 break;
6511         case HWRM_RING_ALLOC_NQ:
6512                 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
6513                 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
6514                 if (bp->flags & BNXT_FLAG_USING_MSIX)
6515                         req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
6516                 break;
6517         default:
6518                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
6519                            ring_type);
6520                 return -1;
6521         }
6522
6523         resp = hwrm_req_hold(bp, req);
6524         rc = hwrm_req_send(bp, req);
6525         err = le16_to_cpu(resp->error_code);
6526         ring_id = le16_to_cpu(resp->ring_id);
6527         hwrm_req_drop(bp, req);
6528
6529 exit:
6530         if (rc || err) {
6531                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
6532                            ring_type, rc, err);
6533                 return -EIO;
6534         }
6535         ring->fw_ring_id = ring_id;
6536         return rc;
6537 }
6538
6539 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
6540 {
6541         int rc;
6542
6543         if (BNXT_PF(bp)) {
6544                 struct hwrm_func_cfg_input *req;
6545
6546                 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
6547                 if (rc)
6548                         return rc;
6549
6550                 req->fid = cpu_to_le16(0xffff);
6551                 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
6552                 req->async_event_cr = cpu_to_le16(idx);
6553                 return hwrm_req_send(bp, req);
6554         } else {
6555                 struct hwrm_func_vf_cfg_input *req;
6556
6557                 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG);
6558                 if (rc)
6559                         return rc;
6560
6561                 req->enables =
6562                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
6563                 req->async_event_cr = cpu_to_le16(idx);
6564                 return hwrm_req_send(bp, req);
6565         }
6566 }
6567
6568 static void bnxt_set_db_mask(struct bnxt *bp, struct bnxt_db_info *db,
6569                              u32 ring_type)
6570 {
6571         switch (ring_type) {
6572         case HWRM_RING_ALLOC_TX:
6573                 db->db_ring_mask = bp->tx_ring_mask;
6574                 break;
6575         case HWRM_RING_ALLOC_RX:
6576                 db->db_ring_mask = bp->rx_ring_mask;
6577                 break;
6578         case HWRM_RING_ALLOC_AGG:
6579                 db->db_ring_mask = bp->rx_agg_ring_mask;
6580                 break;
6581         case HWRM_RING_ALLOC_CMPL:
6582         case HWRM_RING_ALLOC_NQ:
6583                 db->db_ring_mask = bp->cp_ring_mask;
6584                 break;
6585         }
6586         if (bp->flags & BNXT_FLAG_CHIP_P7) {
6587                 db->db_epoch_mask = db->db_ring_mask + 1;
6588                 db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask);
6589         }
6590 }
6591
6592 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
6593                         u32 map_idx, u32 xid)
6594 {
6595         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6596                 switch (ring_type) {
6597                 case HWRM_RING_ALLOC_TX:
6598                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
6599                         break;
6600                 case HWRM_RING_ALLOC_RX:
6601                 case HWRM_RING_ALLOC_AGG:
6602                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
6603                         break;
6604                 case HWRM_RING_ALLOC_CMPL:
6605                         db->db_key64 = DBR_PATH_L2;
6606                         break;
6607                 case HWRM_RING_ALLOC_NQ:
6608                         db->db_key64 = DBR_PATH_L2;
6609                         break;
6610                 }
6611                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
6612
6613                 if (bp->flags & BNXT_FLAG_CHIP_P7)
6614                         db->db_key64 |= DBR_VALID;
6615
6616                 db->doorbell = bp->bar1 + bp->db_offset;
6617         } else {
6618                 db->doorbell = bp->bar1 + map_idx * 0x80;
6619                 switch (ring_type) {
6620                 case HWRM_RING_ALLOC_TX:
6621                         db->db_key32 = DB_KEY_TX;
6622                         break;
6623                 case HWRM_RING_ALLOC_RX:
6624                 case HWRM_RING_ALLOC_AGG:
6625                         db->db_key32 = DB_KEY_RX;
6626                         break;
6627                 case HWRM_RING_ALLOC_CMPL:
6628                         db->db_key32 = DB_KEY_CP;
6629                         break;
6630                 }
6631         }
6632         bnxt_set_db_mask(bp, db, ring_type);
6633 }
6634
6635 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
6636 {
6637         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
6638         int i, rc = 0;
6639         u32 type;
6640
6641         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6642                 type = HWRM_RING_ALLOC_NQ;
6643         else
6644                 type = HWRM_RING_ALLOC_CMPL;
6645         for (i = 0; i < bp->cp_nr_rings; i++) {
6646                 struct bnxt_napi *bnapi = bp->bnapi[i];
6647                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6648                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
6649                 u32 map_idx = ring->map_idx;
6650                 unsigned int vector;
6651
6652                 vector = bp->irq_tbl[map_idx].vector;
6653                 disable_irq_nosync(vector);
6654                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6655                 if (rc) {
6656                         enable_irq(vector);
6657                         goto err_out;
6658                 }
6659                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
6660                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
6661                 enable_irq(vector);
6662                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
6663
6664                 if (!i) {
6665                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
6666                         if (rc)
6667                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
6668                 }
6669         }
6670
6671         type = HWRM_RING_ALLOC_TX;
6672         for (i = 0; i < bp->tx_nr_rings; i++) {
6673                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6674                 struct bnxt_ring_struct *ring;
6675                 u32 map_idx;
6676
6677                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6678                         struct bnxt_cp_ring_info *cpr2 = txr->tx_cpr;
6679                         struct bnxt_napi *bnapi = txr->bnapi;
6680                         u32 type2 = HWRM_RING_ALLOC_CMPL;
6681
6682                         ring = &cpr2->cp_ring_struct;
6683                         ring->handle = BNXT_SET_NQ_HDL(cpr2);
6684                         map_idx = bnapi->index;
6685                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
6686                         if (rc)
6687                                 goto err_out;
6688                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
6689                                     ring->fw_ring_id);
6690                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
6691                 }
6692                 ring = &txr->tx_ring_struct;
6693                 map_idx = i;
6694                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6695                 if (rc)
6696                         goto err_out;
6697                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
6698         }
6699
6700         type = HWRM_RING_ALLOC_RX;
6701         for (i = 0; i < bp->rx_nr_rings; i++) {
6702                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6703                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6704                 struct bnxt_napi *bnapi = rxr->bnapi;
6705                 u32 map_idx = bnapi->index;
6706
6707                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6708                 if (rc)
6709                         goto err_out;
6710                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
6711                 /* If we have agg rings, post agg buffers first. */
6712                 if (!agg_rings)
6713                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
6714                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
6715                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6716                         struct bnxt_cp_ring_info *cpr2 = rxr->rx_cpr;
6717                         u32 type2 = HWRM_RING_ALLOC_CMPL;
6718
6719                         ring = &cpr2->cp_ring_struct;
6720                         ring->handle = BNXT_SET_NQ_HDL(cpr2);
6721                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
6722                         if (rc)
6723                                 goto err_out;
6724                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
6725                                     ring->fw_ring_id);
6726                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
6727                 }
6728         }
6729
6730         if (agg_rings) {
6731                 type = HWRM_RING_ALLOC_AGG;
6732                 for (i = 0; i < bp->rx_nr_rings; i++) {
6733                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6734                         struct bnxt_ring_struct *ring =
6735                                                 &rxr->rx_agg_ring_struct;
6736                         u32 grp_idx = ring->grp_idx;
6737                         u32 map_idx = grp_idx + bp->rx_nr_rings;
6738
6739                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
6740                         if (rc)
6741                                 goto err_out;
6742
6743                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
6744                                     ring->fw_ring_id);
6745                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
6746                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
6747                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
6748                 }
6749         }
6750 err_out:
6751         return rc;
6752 }
6753
6754 static int hwrm_ring_free_send_msg(struct bnxt *bp,
6755                                    struct bnxt_ring_struct *ring,
6756                                    u32 ring_type, int cmpl_ring_id)
6757 {
6758         struct hwrm_ring_free_output *resp;
6759         struct hwrm_ring_free_input *req;
6760         u16 error_code = 0;
6761         int rc;
6762
6763         if (BNXT_NO_FW_ACCESS(bp))
6764                 return 0;
6765
6766         rc = hwrm_req_init(bp, req, HWRM_RING_FREE);
6767         if (rc)
6768                 goto exit;
6769
6770         req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
6771         req->ring_type = ring_type;
6772         req->ring_id = cpu_to_le16(ring->fw_ring_id);
6773
6774         resp = hwrm_req_hold(bp, req);
6775         rc = hwrm_req_send(bp, req);
6776         error_code = le16_to_cpu(resp->error_code);
6777         hwrm_req_drop(bp, req);
6778 exit:
6779         if (rc || error_code) {
6780                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
6781                            ring_type, rc, error_code);
6782                 return -EIO;
6783         }
6784         return 0;
6785 }
6786
6787 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
6788 {
6789         u32 type;
6790         int i;
6791
6792         if (!bp->bnapi)
6793                 return;
6794
6795         for (i = 0; i < bp->tx_nr_rings; i++) {
6796                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6797                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
6798
6799                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6800                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
6801
6802                         hwrm_ring_free_send_msg(bp, ring,
6803                                                 RING_FREE_REQ_RING_TYPE_TX,
6804                                                 close_path ? cmpl_ring_id :
6805                                                 INVALID_HW_RING_ID);
6806                         ring->fw_ring_id = INVALID_HW_RING_ID;
6807                 }
6808         }
6809
6810         for (i = 0; i < bp->rx_nr_rings; i++) {
6811                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6812                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6813                 u32 grp_idx = rxr->bnapi->index;
6814
6815                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6816                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6817
6818                         hwrm_ring_free_send_msg(bp, ring,
6819                                                 RING_FREE_REQ_RING_TYPE_RX,
6820                                                 close_path ? cmpl_ring_id :
6821                                                 INVALID_HW_RING_ID);
6822                         ring->fw_ring_id = INVALID_HW_RING_ID;
6823                         bp->grp_info[grp_idx].rx_fw_ring_id =
6824                                 INVALID_HW_RING_ID;
6825                 }
6826         }
6827
6828         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6829                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
6830         else
6831                 type = RING_FREE_REQ_RING_TYPE_RX;
6832         for (i = 0; i < bp->rx_nr_rings; i++) {
6833                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6834                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
6835                 u32 grp_idx = rxr->bnapi->index;
6836
6837                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6838                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6839
6840                         hwrm_ring_free_send_msg(bp, ring, type,
6841                                                 close_path ? cmpl_ring_id :
6842                                                 INVALID_HW_RING_ID);
6843                         ring->fw_ring_id = INVALID_HW_RING_ID;
6844                         bp->grp_info[grp_idx].agg_fw_ring_id =
6845                                 INVALID_HW_RING_ID;
6846                 }
6847         }
6848
6849         /* The completion rings are about to be freed.  After that the
6850          * IRQ doorbell will not work anymore.  So we need to disable
6851          * IRQ here.
6852          */
6853         bnxt_disable_int_sync(bp);
6854
6855         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6856                 type = RING_FREE_REQ_RING_TYPE_NQ;
6857         else
6858                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
6859         for (i = 0; i < bp->cp_nr_rings; i++) {
6860                 struct bnxt_napi *bnapi = bp->bnapi[i];
6861                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6862                 struct bnxt_ring_struct *ring;
6863                 int j;
6864
6865                 for (j = 0; j < cpr->cp_ring_count && cpr->cp_ring_arr; j++) {
6866                         struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
6867
6868                         ring = &cpr2->cp_ring_struct;
6869                         if (ring->fw_ring_id == INVALID_HW_RING_ID)
6870                                 continue;
6871                         hwrm_ring_free_send_msg(bp, ring,
6872                                                 RING_FREE_REQ_RING_TYPE_L2_CMPL,
6873                                                 INVALID_HW_RING_ID);
6874                         ring->fw_ring_id = INVALID_HW_RING_ID;
6875                 }
6876                 ring = &cpr->cp_ring_struct;
6877                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6878                         hwrm_ring_free_send_msg(bp, ring, type,
6879                                                 INVALID_HW_RING_ID);
6880                         ring->fw_ring_id = INVALID_HW_RING_ID;
6881                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
6882                 }
6883         }
6884 }
6885
6886 static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
6887                              bool shared);
6888 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
6889                            bool shared);
6890
6891 static int bnxt_hwrm_get_rings(struct bnxt *bp)
6892 {
6893         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6894         struct hwrm_func_qcfg_output *resp;
6895         struct hwrm_func_qcfg_input *req;
6896         int rc;
6897
6898         if (bp->hwrm_spec_code < 0x10601)
6899                 return 0;
6900
6901         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6902         if (rc)
6903                 return rc;
6904
6905         req->fid = cpu_to_le16(0xffff);
6906         resp = hwrm_req_hold(bp, req);
6907         rc = hwrm_req_send(bp, req);
6908         if (rc) {
6909                 hwrm_req_drop(bp, req);
6910                 return rc;
6911         }
6912
6913         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6914         if (BNXT_NEW_RM(bp)) {
6915                 u16 cp, stats;
6916
6917                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
6918                 hw_resc->resv_hw_ring_grps =
6919                         le32_to_cpu(resp->alloc_hw_ring_grps);
6920                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
6921                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
6922                 stats = le16_to_cpu(resp->alloc_stat_ctx);
6923                 hw_resc->resv_irqs = cp;
6924                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6925                         int rx = hw_resc->resv_rx_rings;
6926                         int tx = hw_resc->resv_tx_rings;
6927
6928                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6929                                 rx >>= 1;
6930                         if (cp < (rx + tx)) {
6931                                 rc = __bnxt_trim_rings(bp, &rx, &tx, cp, false);
6932                                 if (rc)
6933                                         goto get_rings_exit;
6934                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6935                                         rx <<= 1;
6936                                 hw_resc->resv_rx_rings = rx;
6937                                 hw_resc->resv_tx_rings = tx;
6938                         }
6939                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
6940                         hw_resc->resv_hw_ring_grps = rx;
6941                 }
6942                 hw_resc->resv_cp_rings = cp;
6943                 hw_resc->resv_stat_ctxs = stats;
6944         }
6945 get_rings_exit:
6946         hwrm_req_drop(bp, req);
6947         return rc;
6948 }
6949
6950 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
6951 {
6952         struct hwrm_func_qcfg_output *resp;
6953         struct hwrm_func_qcfg_input *req;
6954         int rc;
6955
6956         if (bp->hwrm_spec_code < 0x10601)
6957                 return 0;
6958
6959         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6960         if (rc)
6961                 return rc;
6962
6963         req->fid = cpu_to_le16(fid);
6964         resp = hwrm_req_hold(bp, req);
6965         rc = hwrm_req_send(bp, req);
6966         if (!rc)
6967                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6968
6969         hwrm_req_drop(bp, req);
6970         return rc;
6971 }
6972
6973 static bool bnxt_rfs_supported(struct bnxt *bp);
6974
6975 static struct hwrm_func_cfg_input *
6976 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6977                              int ring_grps, int cp_rings, int stats, int vnics)
6978 {
6979         struct hwrm_func_cfg_input *req;
6980         u32 enables = 0;
6981
6982         if (bnxt_hwrm_func_cfg_short_req_init(bp, &req))
6983                 return NULL;
6984
6985         req->fid = cpu_to_le16(0xffff);
6986         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6987         req->num_tx_rings = cpu_to_le16(tx_rings);
6988         if (BNXT_NEW_RM(bp)) {
6989                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
6990                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6991                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6992                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
6993                         enables |= tx_rings + ring_grps ?
6994                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6995                         enables |= rx_rings ?
6996                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6997                 } else {
6998                         enables |= cp_rings ?
6999                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7000                         enables |= ring_grps ?
7001                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
7002                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
7003                 }
7004                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
7005
7006                 req->num_rx_rings = cpu_to_le16(rx_rings);
7007                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7008                         u16 rss_ctx = bnxt_get_nr_rss_ctxs(bp, ring_grps);
7009
7010                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
7011                         req->num_msix = cpu_to_le16(cp_rings);
7012                         req->num_rsscos_ctxs = cpu_to_le16(rss_ctx);
7013                 } else {
7014                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
7015                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
7016                         req->num_rsscos_ctxs = cpu_to_le16(1);
7017                         if (!(bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) &&
7018                             bnxt_rfs_supported(bp))
7019                                 req->num_rsscos_ctxs =
7020                                         cpu_to_le16(ring_grps + 1);
7021                 }
7022                 req->num_stat_ctxs = cpu_to_le16(stats);
7023                 req->num_vnics = cpu_to_le16(vnics);
7024         }
7025         req->enables = cpu_to_le32(enables);
7026         return req;
7027 }
7028
7029 static struct hwrm_func_vf_cfg_input *
7030 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7031                              int ring_grps, int cp_rings, int stats, int vnics)
7032 {
7033         struct hwrm_func_vf_cfg_input *req;
7034         u32 enables = 0;
7035
7036         if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG))
7037                 return NULL;
7038
7039         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
7040         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
7041                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
7042         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
7043         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7044                 enables |= tx_rings + ring_grps ?
7045                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7046         } else {
7047                 enables |= cp_rings ?
7048                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7049                 enables |= ring_grps ?
7050                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
7051         }
7052         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
7053         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
7054
7055         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
7056         req->num_tx_rings = cpu_to_le16(tx_rings);
7057         req->num_rx_rings = cpu_to_le16(rx_rings);
7058         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7059                 u16 rss_ctx = bnxt_get_nr_rss_ctxs(bp, ring_grps);
7060
7061                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
7062                 req->num_rsscos_ctxs = cpu_to_le16(rss_ctx);
7063         } else {
7064                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
7065                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
7066                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
7067         }
7068         req->num_stat_ctxs = cpu_to_le16(stats);
7069         req->num_vnics = cpu_to_le16(vnics);
7070
7071         req->enables = cpu_to_le32(enables);
7072         return req;
7073 }
7074
7075 static int
7076 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7077                            int ring_grps, int cp_rings, int stats, int vnics)
7078 {
7079         struct hwrm_func_cfg_input *req;
7080         int rc;
7081
7082         req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
7083                                            cp_rings, stats, vnics);
7084         if (!req)
7085                 return -ENOMEM;
7086
7087         if (!req->enables) {
7088                 hwrm_req_drop(bp, req);
7089                 return 0;
7090         }
7091
7092         rc = hwrm_req_send(bp, req);
7093         if (rc)
7094                 return rc;
7095
7096         if (bp->hwrm_spec_code < 0x10601)
7097                 bp->hw_resc.resv_tx_rings = tx_rings;
7098
7099         return bnxt_hwrm_get_rings(bp);
7100 }
7101
7102 static int
7103 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7104                            int ring_grps, int cp_rings, int stats, int vnics)
7105 {
7106         struct hwrm_func_vf_cfg_input *req;
7107         int rc;
7108
7109         if (!BNXT_NEW_RM(bp)) {
7110                 bp->hw_resc.resv_tx_rings = tx_rings;
7111                 return 0;
7112         }
7113
7114         req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
7115                                            cp_rings, stats, vnics);
7116         if (!req)
7117                 return -ENOMEM;
7118
7119         rc = hwrm_req_send(bp, req);
7120         if (rc)
7121                 return rc;
7122
7123         return bnxt_hwrm_get_rings(bp);
7124 }
7125
7126 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
7127                                    int cp, int stat, int vnic)
7128 {
7129         if (BNXT_PF(bp))
7130                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
7131                                                   vnic);
7132         else
7133                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
7134                                                   vnic);
7135 }
7136
7137 int bnxt_nq_rings_in_use(struct bnxt *bp)
7138 {
7139         int cp = bp->cp_nr_rings;
7140         int ulp_msix, ulp_base;
7141
7142         ulp_msix = bnxt_get_ulp_msix_num(bp);
7143         if (ulp_msix) {
7144                 ulp_base = bnxt_get_ulp_msix_base(bp);
7145                 cp += ulp_msix;
7146                 if ((ulp_base + ulp_msix) > cp)
7147                         cp = ulp_base + ulp_msix;
7148         }
7149         return cp;
7150 }
7151
7152 static int bnxt_cp_rings_in_use(struct bnxt *bp)
7153 {
7154         int cp;
7155
7156         if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7157                 return bnxt_nq_rings_in_use(bp);
7158
7159         cp = bp->tx_nr_rings + bp->rx_nr_rings;
7160         return cp;
7161 }
7162
7163 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
7164 {
7165         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
7166         int cp = bp->cp_nr_rings;
7167
7168         if (!ulp_stat)
7169                 return cp;
7170
7171         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
7172                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
7173
7174         return cp + ulp_stat;
7175 }
7176
7177 /* Check if a default RSS map needs to be setup.  This function is only
7178  * used on older firmware that does not require reserving RX rings.
7179  */
7180 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
7181 {
7182         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7183
7184         /* The RSS map is valid for RX rings set to resv_rx_rings */
7185         if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
7186                 hw_resc->resv_rx_rings = bp->rx_nr_rings;
7187                 if (!netif_is_rxfh_configured(bp->dev))
7188                         bnxt_set_dflt_rss_indir_tbl(bp);
7189         }
7190 }
7191
7192 static bool bnxt_need_reserve_rings(struct bnxt *bp)
7193 {
7194         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7195         int cp = bnxt_cp_rings_in_use(bp);
7196         int nq = bnxt_nq_rings_in_use(bp);
7197         int rx = bp->rx_nr_rings, stat;
7198         int vnic = 1, grp = rx;
7199
7200         if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
7201             bp->hwrm_spec_code >= 0x10601)
7202                 return true;
7203
7204         /* Old firmware does not need RX ring reservations but we still
7205          * need to setup a default RSS map when needed.  With new firmware
7206          * we go through RX ring reservations first and then set up the
7207          * RSS map for the successfully reserved RX rings when needed.
7208          */
7209         if (!BNXT_NEW_RM(bp)) {
7210                 bnxt_check_rss_tbl_no_rmgr(bp);
7211                 return false;
7212         }
7213         if ((bp->flags & BNXT_FLAG_RFS) &&
7214             !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7215                 vnic = rx + 1;
7216         if (bp->flags & BNXT_FLAG_AGG_RINGS)
7217                 rx <<= 1;
7218         stat = bnxt_get_func_stat_ctxs(bp);
7219         if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
7220             hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
7221             (hw_resc->resv_hw_ring_grps != grp &&
7222              !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)))
7223                 return true;
7224         if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && BNXT_PF(bp) &&
7225             hw_resc->resv_irqs != nq)
7226                 return true;
7227         return false;
7228 }
7229
7230 static int __bnxt_reserve_rings(struct bnxt *bp)
7231 {
7232         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7233         int cp = bnxt_nq_rings_in_use(bp);
7234         int tx = bp->tx_nr_rings;
7235         int rx = bp->rx_nr_rings;
7236         int grp, rx_rings, rc;
7237         int vnic = 1, stat;
7238         bool sh = false;
7239         int tx_cp;
7240
7241         if (!bnxt_need_reserve_rings(bp))
7242                 return 0;
7243
7244         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7245                 sh = true;
7246         if ((bp->flags & BNXT_FLAG_RFS) &&
7247             !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7248                 vnic = rx + 1;
7249         if (bp->flags & BNXT_FLAG_AGG_RINGS)
7250                 rx <<= 1;
7251         grp = bp->rx_nr_rings;
7252         stat = bnxt_get_func_stat_ctxs(bp);
7253
7254         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
7255         if (rc)
7256                 return rc;
7257
7258         tx = hw_resc->resv_tx_rings;
7259         if (BNXT_NEW_RM(bp)) {
7260                 rx = hw_resc->resv_rx_rings;
7261                 cp = hw_resc->resv_irqs;
7262                 grp = hw_resc->resv_hw_ring_grps;
7263                 vnic = hw_resc->resv_vnics;
7264                 stat = hw_resc->resv_stat_ctxs;
7265         }
7266
7267         rx_rings = rx;
7268         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7269                 if (rx >= 2) {
7270                         rx_rings = rx >> 1;
7271                 } else {
7272                         if (netif_running(bp->dev))
7273                                 return -ENOMEM;
7274
7275                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
7276                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
7277                         bp->dev->hw_features &= ~NETIF_F_LRO;
7278                         bp->dev->features &= ~NETIF_F_LRO;
7279                         bnxt_set_ring_params(bp);
7280                 }
7281         }
7282         rx_rings = min_t(int, rx_rings, grp);
7283         cp = min_t(int, cp, bp->cp_nr_rings);
7284         if (stat > bnxt_get_ulp_stat_ctxs(bp))
7285                 stat -= bnxt_get_ulp_stat_ctxs(bp);
7286         cp = min_t(int, cp, stat);
7287         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
7288         if (bp->flags & BNXT_FLAG_AGG_RINGS)
7289                 rx = rx_rings << 1;
7290         tx_cp = bnxt_num_tx_to_cp(bp, tx);
7291         cp = sh ? max_t(int, tx_cp, rx_rings) : tx_cp + rx_rings;
7292         bp->tx_nr_rings = tx;
7293
7294         /* If we cannot reserve all the RX rings, reset the RSS map only
7295          * if absolutely necessary
7296          */
7297         if (rx_rings != bp->rx_nr_rings) {
7298                 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
7299                             rx_rings, bp->rx_nr_rings);
7300                 if (netif_is_rxfh_configured(bp->dev) &&
7301                     (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
7302                      bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
7303                      bnxt_get_max_rss_ring(bp) >= rx_rings)) {
7304                         netdev_warn(bp->dev, "RSS table entries reverting to default\n");
7305                         bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
7306                 }
7307         }
7308         bp->rx_nr_rings = rx_rings;
7309         bp->cp_nr_rings = cp;
7310
7311         if (!tx || !rx || !cp || !grp || !vnic || !stat)
7312                 return -ENOMEM;
7313
7314         if (!netif_is_rxfh_configured(bp->dev))
7315                 bnxt_set_dflt_rss_indir_tbl(bp);
7316
7317         return rc;
7318 }
7319
7320 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7321                                     int ring_grps, int cp_rings, int stats,
7322                                     int vnics)
7323 {
7324         struct hwrm_func_vf_cfg_input *req;
7325         u32 flags;
7326
7327         if (!BNXT_NEW_RM(bp))
7328                 return 0;
7329
7330         req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
7331                                            cp_rings, stats, vnics);
7332         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
7333                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
7334                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
7335                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
7336                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
7337                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
7338         if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7339                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
7340
7341         req->flags = cpu_to_le32(flags);
7342         return hwrm_req_send_silent(bp, req);
7343 }
7344
7345 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7346                                     int ring_grps, int cp_rings, int stats,
7347                                     int vnics)
7348 {
7349         struct hwrm_func_cfg_input *req;
7350         u32 flags;
7351
7352         req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
7353                                            cp_rings, stats, vnics);
7354         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
7355         if (BNXT_NEW_RM(bp)) {
7356                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
7357                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
7358                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
7359                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
7360                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7361                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
7362                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
7363                 else
7364                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
7365         }
7366
7367         req->flags = cpu_to_le32(flags);
7368         return hwrm_req_send_silent(bp, req);
7369 }
7370
7371 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
7372                                  int ring_grps, int cp_rings, int stats,
7373                                  int vnics)
7374 {
7375         if (bp->hwrm_spec_code < 0x10801)
7376                 return 0;
7377
7378         if (BNXT_PF(bp))
7379                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
7380                                                 ring_grps, cp_rings, stats,
7381                                                 vnics);
7382
7383         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
7384                                         cp_rings, stats, vnics);
7385 }
7386
7387 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
7388 {
7389         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7390         struct hwrm_ring_aggint_qcaps_output *resp;
7391         struct hwrm_ring_aggint_qcaps_input *req;
7392         int rc;
7393
7394         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
7395         coal_cap->num_cmpl_dma_aggr_max = 63;
7396         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
7397         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
7398         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
7399         coal_cap->int_lat_tmr_min_max = 65535;
7400         coal_cap->int_lat_tmr_max_max = 65535;
7401         coal_cap->num_cmpl_aggr_int_max = 65535;
7402         coal_cap->timer_units = 80;
7403
7404         if (bp->hwrm_spec_code < 0x10902)
7405                 return;
7406
7407         if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS))
7408                 return;
7409
7410         resp = hwrm_req_hold(bp, req);
7411         rc = hwrm_req_send_silent(bp, req);
7412         if (!rc) {
7413                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
7414                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
7415                 coal_cap->num_cmpl_dma_aggr_max =
7416                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
7417                 coal_cap->num_cmpl_dma_aggr_during_int_max =
7418                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
7419                 coal_cap->cmpl_aggr_dma_tmr_max =
7420                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
7421                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
7422                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
7423                 coal_cap->int_lat_tmr_min_max =
7424                         le16_to_cpu(resp->int_lat_tmr_min_max);
7425                 coal_cap->int_lat_tmr_max_max =
7426                         le16_to_cpu(resp->int_lat_tmr_max_max);
7427                 coal_cap->num_cmpl_aggr_int_max =
7428                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
7429                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
7430         }
7431         hwrm_req_drop(bp, req);
7432 }
7433
7434 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
7435 {
7436         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7437
7438         return usec * 1000 / coal_cap->timer_units;
7439 }
7440
7441 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
7442         struct bnxt_coal *hw_coal,
7443         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7444 {
7445         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7446         u16 val, tmr, max, flags = hw_coal->flags;
7447         u32 cmpl_params = coal_cap->cmpl_params;
7448
7449         max = hw_coal->bufs_per_record * 128;
7450         if (hw_coal->budget)
7451                 max = hw_coal->bufs_per_record * hw_coal->budget;
7452         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
7453
7454         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
7455         req->num_cmpl_aggr_int = cpu_to_le16(val);
7456
7457         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
7458         req->num_cmpl_dma_aggr = cpu_to_le16(val);
7459
7460         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
7461                       coal_cap->num_cmpl_dma_aggr_during_int_max);
7462         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
7463
7464         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
7465         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
7466         req->int_lat_tmr_max = cpu_to_le16(tmr);
7467
7468         /* min timer set to 1/2 of interrupt timer */
7469         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
7470                 val = tmr / 2;
7471                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
7472                 req->int_lat_tmr_min = cpu_to_le16(val);
7473                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
7474         }
7475
7476         /* buf timer set to 1/4 of interrupt timer */
7477         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
7478         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
7479
7480         if (cmpl_params &
7481             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
7482                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
7483                 val = clamp_t(u16, tmr, 1,
7484                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
7485                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
7486                 req->enables |=
7487                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
7488         }
7489
7490         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
7491             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
7492                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
7493         req->flags = cpu_to_le16(flags);
7494         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
7495 }
7496
7497 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
7498                                    struct bnxt_coal *hw_coal)
7499 {
7500         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req;
7501         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7502         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7503         u32 nq_params = coal_cap->nq_params;
7504         u16 tmr;
7505         int rc;
7506
7507         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
7508                 return 0;
7509
7510         rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7511         if (rc)
7512                 return rc;
7513
7514         req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
7515         req->flags =
7516                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
7517
7518         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
7519         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
7520         req->int_lat_tmr_min = cpu_to_le16(tmr);
7521         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
7522         return hwrm_req_send(bp, req);
7523 }
7524
7525 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
7526 {
7527         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx;
7528         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7529         struct bnxt_coal coal;
7530         int rc;
7531
7532         /* Tick values in micro seconds.
7533          * 1 coal_buf x bufs_per_record = 1 completion record.
7534          */
7535         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
7536
7537         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
7538         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
7539
7540         if (!bnapi->rx_ring)
7541                 return -ENODEV;
7542
7543         rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7544         if (rc)
7545                 return rc;
7546
7547         bnxt_hwrm_set_coal_params(bp, &coal, req_rx);
7548
7549         req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
7550
7551         return hwrm_req_send(bp, req_rx);
7552 }
7553
7554 static int
7555 bnxt_hwrm_set_rx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
7556                       struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7557 {
7558         u16 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
7559
7560         req->ring_id = cpu_to_le16(ring_id);
7561         return hwrm_req_send(bp, req);
7562 }
7563
7564 static int
7565 bnxt_hwrm_set_tx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
7566                       struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7567 {
7568         struct bnxt_tx_ring_info *txr;
7569         int i, rc;
7570
7571         bnxt_for_each_napi_tx(i, bnapi, txr) {
7572                 u16 ring_id;
7573
7574                 ring_id = bnxt_cp_ring_for_tx(bp, txr);
7575                 req->ring_id = cpu_to_le16(ring_id);
7576                 rc = hwrm_req_send(bp, req);
7577                 if (rc)
7578                         return rc;
7579                 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7580                         return 0;
7581         }
7582         return 0;
7583 }
7584
7585 int bnxt_hwrm_set_coal(struct bnxt *bp)
7586 {
7587         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx;
7588         int i, rc;
7589
7590         rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7591         if (rc)
7592                 return rc;
7593
7594         rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
7595         if (rc) {
7596                 hwrm_req_drop(bp, req_rx);
7597                 return rc;
7598         }
7599
7600         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx);
7601         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx);
7602
7603         hwrm_req_hold(bp, req_rx);
7604         hwrm_req_hold(bp, req_tx);
7605         for (i = 0; i < bp->cp_nr_rings; i++) {
7606                 struct bnxt_napi *bnapi = bp->bnapi[i];
7607                 struct bnxt_coal *hw_coal;
7608
7609                 if (!bnapi->rx_ring)
7610                         rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
7611                 else
7612                         rc = bnxt_hwrm_set_rx_coal(bp, bnapi, req_rx);
7613                 if (rc)
7614                         break;
7615
7616                 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7617                         continue;
7618
7619                 if (bnapi->rx_ring && bnapi->tx_ring[0]) {
7620                         rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
7621                         if (rc)
7622                                 break;
7623                 }
7624                 if (bnapi->rx_ring)
7625                         hw_coal = &bp->rx_coal;
7626                 else
7627                         hw_coal = &bp->tx_coal;
7628                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
7629         }
7630         hwrm_req_drop(bp, req_rx);
7631         hwrm_req_drop(bp, req_tx);
7632         return rc;
7633 }
7634
7635 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
7636 {
7637         struct hwrm_stat_ctx_clr_stats_input *req0 = NULL;
7638         struct hwrm_stat_ctx_free_input *req;
7639         int i;
7640
7641         if (!bp->bnapi)
7642                 return;
7643
7644         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7645                 return;
7646
7647         if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE))
7648                 return;
7649         if (BNXT_FW_MAJ(bp) <= 20) {
7650                 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) {
7651                         hwrm_req_drop(bp, req);
7652                         return;
7653                 }
7654                 hwrm_req_hold(bp, req0);
7655         }
7656         hwrm_req_hold(bp, req);
7657         for (i = 0; i < bp->cp_nr_rings; i++) {
7658                 struct bnxt_napi *bnapi = bp->bnapi[i];
7659                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7660
7661                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
7662                         req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
7663                         if (req0) {
7664                                 req0->stat_ctx_id = req->stat_ctx_id;
7665                                 hwrm_req_send(bp, req0);
7666                         }
7667                         hwrm_req_send(bp, req);
7668
7669                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
7670                 }
7671         }
7672         hwrm_req_drop(bp, req);
7673         if (req0)
7674                 hwrm_req_drop(bp, req0);
7675 }
7676
7677 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
7678 {
7679         struct hwrm_stat_ctx_alloc_output *resp;
7680         struct hwrm_stat_ctx_alloc_input *req;
7681         int rc, i;
7682
7683         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7684                 return 0;
7685
7686         rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC);
7687         if (rc)
7688                 return rc;
7689
7690         req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
7691         req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
7692
7693         resp = hwrm_req_hold(bp, req);
7694         for (i = 0; i < bp->cp_nr_rings; i++) {
7695                 struct bnxt_napi *bnapi = bp->bnapi[i];
7696                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7697
7698                 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
7699
7700                 rc = hwrm_req_send(bp, req);
7701                 if (rc)
7702                         break;
7703
7704                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
7705
7706                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
7707         }
7708         hwrm_req_drop(bp, req);
7709         return rc;
7710 }
7711
7712 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
7713 {
7714         struct hwrm_func_qcfg_output *resp;
7715         struct hwrm_func_qcfg_input *req;
7716         u16 flags;
7717         int rc;
7718
7719         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
7720         if (rc)
7721                 return rc;
7722
7723         req->fid = cpu_to_le16(0xffff);
7724         resp = hwrm_req_hold(bp, req);
7725         rc = hwrm_req_send(bp, req);
7726         if (rc)
7727                 goto func_qcfg_exit;
7728
7729 #ifdef CONFIG_BNXT_SRIOV
7730         if (BNXT_VF(bp)) {
7731                 struct bnxt_vf_info *vf = &bp->vf;
7732
7733                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
7734         } else {
7735                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
7736         }
7737 #endif
7738         flags = le16_to_cpu(resp->flags);
7739         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
7740                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
7741                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
7742                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
7743                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
7744         }
7745         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
7746                 bp->flags |= BNXT_FLAG_MULTI_HOST;
7747
7748         if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
7749                 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
7750
7751         switch (resp->port_partition_type) {
7752         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
7753         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
7754         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
7755                 bp->port_partition_type = resp->port_partition_type;
7756                 break;
7757         }
7758         if (bp->hwrm_spec_code < 0x10707 ||
7759             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
7760                 bp->br_mode = BRIDGE_MODE_VEB;
7761         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
7762                 bp->br_mode = BRIDGE_MODE_VEPA;
7763         else
7764                 bp->br_mode = BRIDGE_MODE_UNDEF;
7765
7766         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
7767         if (!bp->max_mtu)
7768                 bp->max_mtu = BNXT_MAX_MTU;
7769
7770         if (bp->db_size)
7771                 goto func_qcfg_exit;
7772
7773         bp->db_offset = le16_to_cpu(resp->legacy_l2_db_size_kb) * 1024;
7774         if (BNXT_CHIP_P5(bp)) {
7775                 if (BNXT_PF(bp))
7776                         bp->db_offset = DB_PF_OFFSET_P5;
7777                 else
7778                         bp->db_offset = DB_VF_OFFSET_P5;
7779         }
7780         bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
7781                                  1024);
7782         if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
7783             bp->db_size <= bp->db_offset)
7784                 bp->db_size = pci_resource_len(bp->pdev, 2);
7785
7786 func_qcfg_exit:
7787         hwrm_req_drop(bp, req);
7788         return rc;
7789 }
7790
7791 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_type *ctxm,
7792                                       u8 init_val, u8 init_offset,
7793                                       bool init_mask_set)
7794 {
7795         ctxm->init_value = init_val;
7796         ctxm->init_offset = BNXT_CTX_INIT_INVALID_OFFSET;
7797         if (init_mask_set)
7798                 ctxm->init_offset = init_offset * 4;
7799         else
7800                 ctxm->init_value = 0;
7801 }
7802
7803 static int bnxt_alloc_all_ctx_pg_info(struct bnxt *bp, int ctx_max)
7804 {
7805         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7806         u16 type;
7807
7808         for (type = 0; type < ctx_max; type++) {
7809                 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
7810                 int n = 1;
7811
7812                 if (!ctxm->max_entries)
7813                         continue;
7814
7815                 if (ctxm->instance_bmap)
7816                         n = hweight32(ctxm->instance_bmap);
7817                 ctxm->pg_info = kcalloc(n, sizeof(*ctxm->pg_info), GFP_KERNEL);
7818                 if (!ctxm->pg_info)
7819                         return -ENOMEM;
7820         }
7821         return 0;
7822 }
7823
7824 #define BNXT_CTX_INIT_VALID(flags)      \
7825         (!!((flags) &                   \
7826             FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ENABLE_CTX_KIND_INIT))
7827
7828 static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp)
7829 {
7830         struct hwrm_func_backing_store_qcaps_v2_output *resp;
7831         struct hwrm_func_backing_store_qcaps_v2_input *req;
7832         struct bnxt_ctx_mem_info *ctx;
7833         u16 type;
7834         int rc;
7835
7836         rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS_V2);
7837         if (rc)
7838                 return rc;
7839
7840         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
7841         if (!ctx)
7842                 return -ENOMEM;
7843         bp->ctx = ctx;
7844
7845         resp = hwrm_req_hold(bp, req);
7846
7847         for (type = 0; type < BNXT_CTX_V2_MAX; ) {
7848                 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
7849                 u8 init_val, init_off, i;
7850                 __le32 *p;
7851                 u32 flags;
7852
7853                 req->type = cpu_to_le16(type);
7854                 rc = hwrm_req_send(bp, req);
7855                 if (rc)
7856                         goto ctx_done;
7857                 flags = le32_to_cpu(resp->flags);
7858                 type = le16_to_cpu(resp->next_valid_type);
7859                 if (!(flags & FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_TYPE_VALID))
7860                         continue;
7861
7862                 ctxm->type = le16_to_cpu(resp->type);
7863                 ctxm->entry_size = le16_to_cpu(resp->entry_size);
7864                 ctxm->flags = flags;
7865                 ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map);
7866                 ctxm->entry_multiple = resp->entry_multiple;
7867                 ctxm->max_entries = le32_to_cpu(resp->max_num_entries);
7868                 ctxm->min_entries = le32_to_cpu(resp->min_num_entries);
7869                 init_val = resp->ctx_init_value;
7870                 init_off = resp->ctx_init_offset;
7871                 bnxt_init_ctx_initializer(ctxm, init_val, init_off,
7872                                           BNXT_CTX_INIT_VALID(flags));
7873                 ctxm->split_entry_cnt = min_t(u8, resp->subtype_valid_cnt,
7874                                               BNXT_MAX_SPLIT_ENTRY);
7875                 for (i = 0, p = &resp->split_entry_0; i < ctxm->split_entry_cnt;
7876                      i++, p++)
7877                         ctxm->split[i] = le32_to_cpu(*p);
7878         }
7879         rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_V2_MAX);
7880
7881 ctx_done:
7882         hwrm_req_drop(bp, req);
7883         return rc;
7884 }
7885
7886 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
7887 {
7888         struct hwrm_func_backing_store_qcaps_output *resp;
7889         struct hwrm_func_backing_store_qcaps_input *req;
7890         int rc;
7891
7892         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
7893                 return 0;
7894
7895         if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2)
7896                 return bnxt_hwrm_func_backing_store_qcaps_v2(bp);
7897
7898         rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS);
7899         if (rc)
7900                 return rc;
7901
7902         resp = hwrm_req_hold(bp, req);
7903         rc = hwrm_req_send_silent(bp, req);
7904         if (!rc) {
7905                 struct bnxt_ctx_mem_type *ctxm;
7906                 struct bnxt_ctx_mem_info *ctx;
7907                 u8 init_val, init_idx = 0;
7908                 u16 init_mask;
7909
7910                 ctx = bp->ctx;
7911                 if (!ctx) {
7912                         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
7913                         if (!ctx) {
7914                                 rc = -ENOMEM;
7915                                 goto ctx_err;
7916                         }
7917                         bp->ctx = ctx;
7918                 }
7919                 init_val = resp->ctx_kind_initializer;
7920                 init_mask = le16_to_cpu(resp->ctx_init_mask);
7921
7922                 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
7923                 ctxm->max_entries = le32_to_cpu(resp->qp_max_entries);
7924                 ctxm->qp_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
7925                 ctxm->qp_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
7926                 ctxm->qp_fast_qpmd_entries = le16_to_cpu(resp->fast_qpmd_qp_num_entries);
7927                 ctxm->entry_size = le16_to_cpu(resp->qp_entry_size);
7928                 bnxt_init_ctx_initializer(ctxm, init_val, resp->qp_init_offset,
7929                                           (init_mask & (1 << init_idx++)) != 0);
7930
7931                 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
7932                 ctxm->srq_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
7933                 ctxm->max_entries = le32_to_cpu(resp->srq_max_entries);
7934                 ctxm->entry_size = le16_to_cpu(resp->srq_entry_size);
7935                 bnxt_init_ctx_initializer(ctxm, init_val, resp->srq_init_offset,
7936                                           (init_mask & (1 << init_idx++)) != 0);
7937
7938                 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
7939                 ctxm->cq_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
7940                 ctxm->max_entries = le32_to_cpu(resp->cq_max_entries);
7941                 ctxm->entry_size = le16_to_cpu(resp->cq_entry_size);
7942                 bnxt_init_ctx_initializer(ctxm, init_val, resp->cq_init_offset,
7943                                           (init_mask & (1 << init_idx++)) != 0);
7944
7945                 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
7946                 ctxm->vnic_entries = le16_to_cpu(resp->vnic_max_vnic_entries);
7947                 ctxm->max_entries = ctxm->vnic_entries +
7948                         le16_to_cpu(resp->vnic_max_ring_table_entries);
7949                 ctxm->entry_size = le16_to_cpu(resp->vnic_entry_size);
7950                 bnxt_init_ctx_initializer(ctxm, init_val,
7951                                           resp->vnic_init_offset,
7952                                           (init_mask & (1 << init_idx++)) != 0);
7953
7954                 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
7955                 ctxm->max_entries = le32_to_cpu(resp->stat_max_entries);
7956                 ctxm->entry_size = le16_to_cpu(resp->stat_entry_size);
7957                 bnxt_init_ctx_initializer(ctxm, init_val,
7958                                           resp->stat_init_offset,
7959                                           (init_mask & (1 << init_idx++)) != 0);
7960
7961                 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
7962                 ctxm->entry_size = le16_to_cpu(resp->tqm_entry_size);
7963                 ctxm->min_entries = le32_to_cpu(resp->tqm_min_entries_per_ring);
7964                 ctxm->max_entries = le32_to_cpu(resp->tqm_max_entries_per_ring);
7965                 ctxm->entry_multiple = resp->tqm_entries_multiple;
7966                 if (!ctxm->entry_multiple)
7967                         ctxm->entry_multiple = 1;
7968
7969                 memcpy(&ctx->ctx_arr[BNXT_CTX_FTQM], ctxm, sizeof(*ctxm));
7970
7971                 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
7972                 ctxm->max_entries = le32_to_cpu(resp->mrav_max_entries);
7973                 ctxm->entry_size = le16_to_cpu(resp->mrav_entry_size);
7974                 ctxm->mrav_num_entries_units =
7975                         le16_to_cpu(resp->mrav_num_entries_units);
7976                 bnxt_init_ctx_initializer(ctxm, init_val,
7977                                           resp->mrav_init_offset,
7978                                           (init_mask & (1 << init_idx++)) != 0);
7979
7980                 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
7981                 ctxm->entry_size = le16_to_cpu(resp->tim_entry_size);
7982                 ctxm->max_entries = le32_to_cpu(resp->tim_max_entries);
7983
7984                 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
7985                 if (!ctx->tqm_fp_rings_count)
7986                         ctx->tqm_fp_rings_count = bp->max_q;
7987                 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
7988                         ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
7989
7990                 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM];
7991                 memcpy(ctxm, &ctx->ctx_arr[BNXT_CTX_STQM], sizeof(*ctxm));
7992                 ctxm->instance_bmap = (1 << ctx->tqm_fp_rings_count) - 1;
7993
7994                 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_MAX);
7995         } else {
7996                 rc = 0;
7997         }
7998 ctx_err:
7999         hwrm_req_drop(bp, req);
8000         return rc;
8001 }
8002
8003 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
8004                                   __le64 *pg_dir)
8005 {
8006         if (!rmem->nr_pages)
8007                 return;
8008
8009         BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
8010         if (rmem->depth >= 1) {
8011                 if (rmem->depth == 2)
8012                         *pg_attr |= 2;
8013                 else
8014                         *pg_attr |= 1;
8015                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
8016         } else {
8017                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
8018         }
8019 }
8020
8021 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
8022         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
8023          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
8024          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
8025          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
8026          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
8027
8028 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
8029 {
8030         struct hwrm_func_backing_store_cfg_input *req;
8031         struct bnxt_ctx_mem_info *ctx = bp->ctx;
8032         struct bnxt_ctx_pg_info *ctx_pg;
8033         struct bnxt_ctx_mem_type *ctxm;
8034         void **__req = (void **)&req;
8035         u32 req_len = sizeof(*req);
8036         __le32 *num_entries;
8037         __le64 *pg_dir;
8038         u32 flags = 0;
8039         u8 *pg_attr;
8040         u32 ena;
8041         int rc;
8042         int i;
8043
8044         if (!ctx)
8045                 return 0;
8046
8047         if (req_len > bp->hwrm_max_ext_req_len)
8048                 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
8049         rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len);
8050         if (rc)
8051                 return rc;
8052
8053         req->enables = cpu_to_le32(enables);
8054         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
8055                 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8056                 ctx_pg = ctxm->pg_info;
8057                 req->qp_num_entries = cpu_to_le32(ctx_pg->entries);
8058                 req->qp_num_qp1_entries = cpu_to_le16(ctxm->qp_qp1_entries);
8059                 req->qp_num_l2_entries = cpu_to_le16(ctxm->qp_l2_entries);
8060                 req->qp_entry_size = cpu_to_le16(ctxm->entry_size);
8061                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8062                                       &req->qpc_pg_size_qpc_lvl,
8063                                       &req->qpc_page_dir);
8064
8065                 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD)
8066                         req->qp_num_fast_qpmd_entries = cpu_to_le16(ctxm->qp_fast_qpmd_entries);
8067         }
8068         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
8069                 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8070                 ctx_pg = ctxm->pg_info;
8071                 req->srq_num_entries = cpu_to_le32(ctx_pg->entries);
8072                 req->srq_num_l2_entries = cpu_to_le16(ctxm->srq_l2_entries);
8073                 req->srq_entry_size = cpu_to_le16(ctxm->entry_size);
8074                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8075                                       &req->srq_pg_size_srq_lvl,
8076                                       &req->srq_page_dir);
8077         }
8078         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
8079                 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8080                 ctx_pg = ctxm->pg_info;
8081                 req->cq_num_entries = cpu_to_le32(ctx_pg->entries);
8082                 req->cq_num_l2_entries = cpu_to_le16(ctxm->cq_l2_entries);
8083                 req->cq_entry_size = cpu_to_le16(ctxm->entry_size);
8084                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8085                                       &req->cq_pg_size_cq_lvl,
8086                                       &req->cq_page_dir);
8087         }
8088         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
8089                 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8090                 ctx_pg = ctxm->pg_info;
8091                 req->vnic_num_vnic_entries = cpu_to_le16(ctxm->vnic_entries);
8092                 req->vnic_num_ring_table_entries =
8093                         cpu_to_le16(ctxm->max_entries - ctxm->vnic_entries);
8094                 req->vnic_entry_size = cpu_to_le16(ctxm->entry_size);
8095                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8096                                       &req->vnic_pg_size_vnic_lvl,
8097                                       &req->vnic_page_dir);
8098         }
8099         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
8100                 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8101                 ctx_pg = ctxm->pg_info;
8102                 req->stat_num_entries = cpu_to_le32(ctxm->max_entries);
8103                 req->stat_entry_size = cpu_to_le16(ctxm->entry_size);
8104                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8105                                       &req->stat_pg_size_stat_lvl,
8106                                       &req->stat_page_dir);
8107         }
8108         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
8109                 u32 units;
8110
8111                 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8112                 ctx_pg = ctxm->pg_info;
8113                 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries);
8114                 units = ctxm->mrav_num_entries_units;
8115                 if (units) {
8116                         u32 num_mr, num_ah = ctxm->mrav_av_entries;
8117                         u32 entries;
8118
8119                         num_mr = ctx_pg->entries - num_ah;
8120                         entries = ((num_mr / units) << 16) | (num_ah / units);
8121                         req->mrav_num_entries = cpu_to_le32(entries);
8122                         flags |= FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
8123                 }
8124                 req->mrav_entry_size = cpu_to_le16(ctxm->entry_size);
8125                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8126                                       &req->mrav_pg_size_mrav_lvl,
8127                                       &req->mrav_page_dir);
8128         }
8129         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
8130                 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8131                 ctx_pg = ctxm->pg_info;
8132                 req->tim_num_entries = cpu_to_le32(ctx_pg->entries);
8133                 req->tim_entry_size = cpu_to_le16(ctxm->entry_size);
8134                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8135                                       &req->tim_pg_size_tim_lvl,
8136                                       &req->tim_page_dir);
8137         }
8138         ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8139         for (i = 0, num_entries = &req->tqm_sp_num_entries,
8140              pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl,
8141              pg_dir = &req->tqm_sp_page_dir,
8142              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP,
8143              ctx_pg = ctxm->pg_info;
8144              i < BNXT_MAX_TQM_RINGS;
8145              ctx_pg = &ctx->ctx_arr[BNXT_CTX_FTQM].pg_info[i],
8146              i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
8147                 if (!(enables & ena))
8148                         continue;
8149
8150                 req->tqm_entry_size = cpu_to_le16(ctxm->entry_size);
8151                 *num_entries = cpu_to_le32(ctx_pg->entries);
8152                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
8153         }
8154         req->flags = cpu_to_le32(flags);
8155         return hwrm_req_send(bp, req);
8156 }
8157
8158 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
8159                                   struct bnxt_ctx_pg_info *ctx_pg)
8160 {
8161         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8162
8163         rmem->page_size = BNXT_PAGE_SIZE;
8164         rmem->pg_arr = ctx_pg->ctx_pg_arr;
8165         rmem->dma_arr = ctx_pg->ctx_dma_arr;
8166         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
8167         if (rmem->depth >= 1)
8168                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
8169         return bnxt_alloc_ring(bp, rmem);
8170 }
8171
8172 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
8173                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
8174                                   u8 depth, struct bnxt_ctx_mem_type *ctxm)
8175 {
8176         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8177         int rc;
8178
8179         if (!mem_size)
8180                 return -EINVAL;
8181
8182         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
8183         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
8184                 ctx_pg->nr_pages = 0;
8185                 return -EINVAL;
8186         }
8187         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
8188                 int nr_tbls, i;
8189
8190                 rmem->depth = 2;
8191                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
8192                                              GFP_KERNEL);
8193                 if (!ctx_pg->ctx_pg_tbl)
8194                         return -ENOMEM;
8195                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
8196                 rmem->nr_pages = nr_tbls;
8197                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
8198                 if (rc)
8199                         return rc;
8200                 for (i = 0; i < nr_tbls; i++) {
8201                         struct bnxt_ctx_pg_info *pg_tbl;
8202
8203                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
8204                         if (!pg_tbl)
8205                                 return -ENOMEM;
8206                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
8207                         rmem = &pg_tbl->ring_mem;
8208                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
8209                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
8210                         rmem->depth = 1;
8211                         rmem->nr_pages = MAX_CTX_PAGES;
8212                         rmem->ctx_mem = ctxm;
8213                         if (i == (nr_tbls - 1)) {
8214                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
8215
8216                                 if (rem)
8217                                         rmem->nr_pages = rem;
8218                         }
8219                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
8220                         if (rc)
8221                                 break;
8222                 }
8223         } else {
8224                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
8225                 if (rmem->nr_pages > 1 || depth)
8226                         rmem->depth = 1;
8227                 rmem->ctx_mem = ctxm;
8228                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
8229         }
8230         return rc;
8231 }
8232
8233 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
8234                                   struct bnxt_ctx_pg_info *ctx_pg)
8235 {
8236         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8237
8238         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
8239             ctx_pg->ctx_pg_tbl) {
8240                 int i, nr_tbls = rmem->nr_pages;
8241
8242                 for (i = 0; i < nr_tbls; i++) {
8243                         struct bnxt_ctx_pg_info *pg_tbl;
8244                         struct bnxt_ring_mem_info *rmem2;
8245
8246                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
8247                         if (!pg_tbl)
8248                                 continue;
8249                         rmem2 = &pg_tbl->ring_mem;
8250                         bnxt_free_ring(bp, rmem2);
8251                         ctx_pg->ctx_pg_arr[i] = NULL;
8252                         kfree(pg_tbl);
8253                         ctx_pg->ctx_pg_tbl[i] = NULL;
8254                 }
8255                 kfree(ctx_pg->ctx_pg_tbl);
8256                 ctx_pg->ctx_pg_tbl = NULL;
8257         }
8258         bnxt_free_ring(bp, rmem);
8259         ctx_pg->nr_pages = 0;
8260 }
8261
8262 static int bnxt_setup_ctxm_pg_tbls(struct bnxt *bp,
8263                                    struct bnxt_ctx_mem_type *ctxm, u32 entries,
8264                                    u8 pg_lvl)
8265 {
8266         struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
8267         int i, rc = 0, n = 1;
8268         u32 mem_size;
8269
8270         if (!ctxm->entry_size || !ctx_pg)
8271                 return -EINVAL;
8272         if (ctxm->instance_bmap)
8273                 n = hweight32(ctxm->instance_bmap);
8274         if (ctxm->entry_multiple)
8275                 entries = roundup(entries, ctxm->entry_multiple);
8276         entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries);
8277         mem_size = entries * ctxm->entry_size;
8278         for (i = 0; i < n && !rc; i++) {
8279                 ctx_pg[i].entries = entries;
8280                 rc = bnxt_alloc_ctx_pg_tbls(bp, &ctx_pg[i], mem_size, pg_lvl,
8281                                             ctxm->init_value ? ctxm : NULL);
8282         }
8283         return rc;
8284 }
8285
8286 static int bnxt_hwrm_func_backing_store_cfg_v2(struct bnxt *bp,
8287                                                struct bnxt_ctx_mem_type *ctxm,
8288                                                bool last)
8289 {
8290         struct hwrm_func_backing_store_cfg_v2_input *req;
8291         u32 instance_bmap = ctxm->instance_bmap;
8292         int i, j, rc = 0, n = 1;
8293         __le32 *p;
8294
8295         if (!(ctxm->flags & BNXT_CTX_MEM_TYPE_VALID) || !ctxm->pg_info)
8296                 return 0;
8297
8298         if (instance_bmap)
8299                 n = hweight32(ctxm->instance_bmap);
8300         else
8301                 instance_bmap = 1;
8302
8303         rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_CFG_V2);
8304         if (rc)
8305                 return rc;
8306         hwrm_req_hold(bp, req);
8307         req->type = cpu_to_le16(ctxm->type);
8308         req->entry_size = cpu_to_le16(ctxm->entry_size);
8309         req->subtype_valid_cnt = ctxm->split_entry_cnt;
8310         for (i = 0, p = &req->split_entry_0; i < ctxm->split_entry_cnt; i++)
8311                 p[i] = cpu_to_le32(ctxm->split[i]);
8312         for (i = 0, j = 0; j < n && !rc; i++) {
8313                 struct bnxt_ctx_pg_info *ctx_pg;
8314
8315                 if (!(instance_bmap & (1 << i)))
8316                         continue;
8317                 req->instance = cpu_to_le16(i);
8318                 ctx_pg = &ctxm->pg_info[j++];
8319                 if (!ctx_pg->entries)
8320                         continue;
8321                 req->num_entries = cpu_to_le32(ctx_pg->entries);
8322                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8323                                       &req->page_size_pbl_level,
8324                                       &req->page_dir);
8325                 if (last && j == n)
8326                         req->flags =
8327                                 cpu_to_le32(FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE);
8328                 rc = hwrm_req_send(bp, req);
8329         }
8330         hwrm_req_drop(bp, req);
8331         return rc;
8332 }
8333
8334 static int bnxt_backing_store_cfg_v2(struct bnxt *bp, u32 ena)
8335 {
8336         struct bnxt_ctx_mem_info *ctx = bp->ctx;
8337         struct bnxt_ctx_mem_type *ctxm;
8338         u16 last_type;
8339         int rc = 0;
8340         u16 type;
8341
8342         if (!ena)
8343                 return 0;
8344         else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM)
8345                 last_type = BNXT_CTX_MAX - 1;
8346         else
8347                 last_type = BNXT_CTX_L2_MAX - 1;
8348         ctx->ctx_arr[last_type].last = 1;
8349
8350         for (type = 0 ; type < BNXT_CTX_V2_MAX; type++) {
8351                 ctxm = &ctx->ctx_arr[type];
8352
8353                 rc = bnxt_hwrm_func_backing_store_cfg_v2(bp, ctxm, ctxm->last);
8354                 if (rc)
8355                         return rc;
8356         }
8357         return 0;
8358 }
8359
8360 void bnxt_free_ctx_mem(struct bnxt *bp)
8361 {
8362         struct bnxt_ctx_mem_info *ctx = bp->ctx;
8363         u16 type;
8364
8365         if (!ctx)
8366                 return;
8367
8368         for (type = 0; type < BNXT_CTX_V2_MAX; type++) {
8369                 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
8370                 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
8371                 int i, n = 1;
8372
8373                 if (!ctx_pg)
8374                         continue;
8375                 if (ctxm->instance_bmap)
8376                         n = hweight32(ctxm->instance_bmap);
8377                 for (i = 0; i < n; i++)
8378                         bnxt_free_ctx_pg_tbls(bp, &ctx_pg[i]);
8379
8380                 kfree(ctx_pg);
8381                 ctxm->pg_info = NULL;
8382         }
8383
8384         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
8385         kfree(ctx);
8386         bp->ctx = NULL;
8387 }
8388
8389 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
8390 {
8391         struct bnxt_ctx_mem_type *ctxm;
8392         struct bnxt_ctx_mem_info *ctx;
8393         u32 l2_qps, qp1_qps, max_qps;
8394         u32 ena, entries_sp, entries;
8395         u32 srqs, max_srqs, min;
8396         u32 num_mr, num_ah;
8397         u32 extra_srqs = 0;
8398         u32 extra_qps = 0;
8399         u32 fast_qpmd_qps;
8400         u8 pg_lvl = 1;
8401         int i, rc;
8402
8403         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
8404         if (rc) {
8405                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
8406                            rc);
8407                 return rc;
8408         }
8409         ctx = bp->ctx;
8410         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
8411                 return 0;
8412
8413         ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8414         l2_qps = ctxm->qp_l2_entries;
8415         qp1_qps = ctxm->qp_qp1_entries;
8416         fast_qpmd_qps = ctxm->qp_fast_qpmd_entries;
8417         max_qps = ctxm->max_entries;
8418         ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8419         srqs = ctxm->srq_l2_entries;
8420         max_srqs = ctxm->max_entries;
8421         ena = 0;
8422         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
8423                 pg_lvl = 2;
8424                 extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps);
8425                 /* allocate extra qps if fw supports RoCE fast qp destroy feature */
8426                 extra_qps += fast_qpmd_qps;
8427                 extra_srqs = min_t(u32, 8192, max_srqs - srqs);
8428                 if (fast_qpmd_qps)
8429                         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD;
8430         }
8431
8432         ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8433         rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps,
8434                                      pg_lvl);
8435         if (rc)
8436                 return rc;
8437
8438         ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8439         rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, srqs + extra_srqs, pg_lvl);
8440         if (rc)
8441                 return rc;
8442
8443         ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8444         rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->cq_l2_entries +
8445                                      extra_qps * 2, pg_lvl);
8446         if (rc)
8447                 return rc;
8448
8449         ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8450         rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1);
8451         if (rc)
8452                 return rc;
8453
8454         ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8455         rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1);
8456         if (rc)
8457                 return rc;
8458
8459         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
8460                 goto skip_rdma;
8461
8462         ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8463         /* 128K extra is needed to accommodate static AH context
8464          * allocation by f/w.
8465          */
8466         num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256);
8467         num_ah = min_t(u32, num_mr, 1024 * 128);
8468         ctxm->split_entry_cnt = BNXT_CTX_MRAV_AV_SPLIT_ENTRY + 1;
8469         if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah)
8470                 ctxm->mrav_av_entries = num_ah;
8471
8472         rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, num_mr + num_ah, 2);
8473         if (rc)
8474                 return rc;
8475         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
8476
8477         ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8478         rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps, 1);
8479         if (rc)
8480                 return rc;
8481         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
8482
8483 skip_rdma:
8484         ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8485         min = ctxm->min_entries;
8486         entries_sp = ctx->ctx_arr[BNXT_CTX_VNIC].vnic_entries + l2_qps +
8487                      2 * (extra_qps + qp1_qps) + min;
8488         rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries_sp, 2);
8489         if (rc)
8490                 return rc;
8491
8492         ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM];
8493         entries = l2_qps + 2 * (extra_qps + qp1_qps);
8494         rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries, 2);
8495         if (rc)
8496                 return rc;
8497         for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
8498                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
8499         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
8500
8501         if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2)
8502                 rc = bnxt_backing_store_cfg_v2(bp, ena);
8503         else
8504                 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
8505         if (rc) {
8506                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
8507                            rc);
8508                 return rc;
8509         }
8510         ctx->flags |= BNXT_CTX_FLAG_INITED;
8511         return 0;
8512 }
8513
8514 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
8515 {
8516         struct hwrm_func_resource_qcaps_output *resp;
8517         struct hwrm_func_resource_qcaps_input *req;
8518         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8519         int rc;
8520
8521         rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS);
8522         if (rc)
8523                 return rc;
8524
8525         req->fid = cpu_to_le16(0xffff);
8526         resp = hwrm_req_hold(bp, req);
8527         rc = hwrm_req_send_silent(bp, req);
8528         if (rc)
8529                 goto hwrm_func_resc_qcaps_exit;
8530
8531         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
8532         if (!all)
8533                 goto hwrm_func_resc_qcaps_exit;
8534
8535         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
8536         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
8537         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
8538         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
8539         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
8540         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
8541         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
8542         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
8543         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
8544         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
8545         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
8546         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
8547         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
8548         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
8549         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
8550         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
8551
8552         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
8553                 u16 max_msix = le16_to_cpu(resp->max_msix);
8554
8555                 hw_resc->max_nqs = max_msix;
8556                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
8557         }
8558
8559         if (BNXT_PF(bp)) {
8560                 struct bnxt_pf_info *pf = &bp->pf;
8561
8562                 pf->vf_resv_strategy =
8563                         le16_to_cpu(resp->vf_reservation_strategy);
8564                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
8565                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
8566         }
8567 hwrm_func_resc_qcaps_exit:
8568         hwrm_req_drop(bp, req);
8569         return rc;
8570 }
8571
8572 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
8573 {
8574         struct hwrm_port_mac_ptp_qcfg_output *resp;
8575         struct hwrm_port_mac_ptp_qcfg_input *req;
8576         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
8577         bool phc_cfg;
8578         u8 flags;
8579         int rc;
8580
8581         if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5(bp)) {
8582                 rc = -ENODEV;
8583                 goto no_ptp;
8584         }
8585
8586         rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG);
8587         if (rc)
8588                 goto no_ptp;
8589
8590         req->port_id = cpu_to_le16(bp->pf.port_id);
8591         resp = hwrm_req_hold(bp, req);
8592         rc = hwrm_req_send(bp, req);
8593         if (rc)
8594                 goto exit;
8595
8596         flags = resp->flags;
8597         if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
8598                 rc = -ENODEV;
8599                 goto exit;
8600         }
8601         if (!ptp) {
8602                 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
8603                 if (!ptp) {
8604                         rc = -ENOMEM;
8605                         goto exit;
8606                 }
8607                 ptp->bp = bp;
8608                 bp->ptp_cfg = ptp;
8609         }
8610         if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) {
8611                 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
8612                 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
8613         } else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
8614                 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
8615                 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
8616         } else {
8617                 rc = -ENODEV;
8618                 goto exit;
8619         }
8620         phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0;
8621         rc = bnxt_ptp_init(bp, phc_cfg);
8622         if (rc)
8623                 netdev_warn(bp->dev, "PTP initialization failed.\n");
8624 exit:
8625         hwrm_req_drop(bp, req);
8626         if (!rc)
8627                 return 0;
8628
8629 no_ptp:
8630         bnxt_ptp_clear(bp);
8631         kfree(ptp);
8632         bp->ptp_cfg = NULL;
8633         return rc;
8634 }
8635
8636 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
8637 {
8638         struct hwrm_func_qcaps_output *resp;
8639         struct hwrm_func_qcaps_input *req;
8640         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8641         u32 flags, flags_ext, flags_ext2;
8642         int rc;
8643
8644         rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS);
8645         if (rc)
8646                 return rc;
8647
8648         req->fid = cpu_to_le16(0xffff);
8649         resp = hwrm_req_hold(bp, req);
8650         rc = hwrm_req_send(bp, req);
8651         if (rc)
8652                 goto hwrm_func_qcaps_exit;
8653
8654         flags = le32_to_cpu(resp->flags);
8655         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
8656                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
8657         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
8658                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
8659         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
8660                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
8661         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
8662                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
8663         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
8664                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
8665         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
8666                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
8667         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
8668                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
8669         if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
8670                 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
8671         if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED)
8672                 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS;
8673
8674         flags_ext = le32_to_cpu(resp->flags_ext);
8675         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
8676                 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
8677         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED))
8678                 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS;
8679         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED)
8680                 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC;
8681         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT))
8682                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF;
8683         if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED))
8684                 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH;
8685         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_SUPPORTED)
8686                 bp->fw_cap |= BNXT_FW_CAP_BACKING_STORE_V2;
8687         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_TX_COAL_CMPL_CAP)
8688                 bp->flags |= BNXT_FLAG_TX_COAL_CMPL;
8689
8690         flags_ext2 = le32_to_cpu(resp->flags_ext2);
8691         if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED)
8692                 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS;
8693         if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_UDP_GSO_SUPPORTED)
8694                 bp->flags |= BNXT_FLAG_UDP_GSO_CAP;
8695
8696         bp->tx_push_thresh = 0;
8697         if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
8698             BNXT_FW_MAJ(bp) > 217)
8699                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
8700
8701         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
8702         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
8703         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
8704         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
8705         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
8706         if (!hw_resc->max_hw_ring_grps)
8707                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
8708         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
8709         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
8710         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
8711
8712         if (BNXT_PF(bp)) {
8713                 struct bnxt_pf_info *pf = &bp->pf;
8714
8715                 pf->fw_fid = le16_to_cpu(resp->fid);
8716                 pf->port_id = le16_to_cpu(resp->port_id);
8717                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
8718                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
8719                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
8720                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
8721                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
8722                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
8723                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
8724                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
8725                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
8726                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
8727                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
8728                         bp->flags |= BNXT_FLAG_WOL_CAP;
8729                 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
8730                         bp->fw_cap |= BNXT_FW_CAP_PTP;
8731                 } else {
8732                         bnxt_ptp_clear(bp);
8733                         kfree(bp->ptp_cfg);
8734                         bp->ptp_cfg = NULL;
8735                 }
8736         } else {
8737 #ifdef CONFIG_BNXT_SRIOV
8738                 struct bnxt_vf_info *vf = &bp->vf;
8739
8740                 vf->fw_fid = le16_to_cpu(resp->fid);
8741                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
8742 #endif
8743         }
8744
8745 hwrm_func_qcaps_exit:
8746         hwrm_req_drop(bp, req);
8747         return rc;
8748 }
8749
8750 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp)
8751 {
8752         struct hwrm_dbg_qcaps_output *resp;
8753         struct hwrm_dbg_qcaps_input *req;
8754         int rc;
8755
8756         bp->fw_dbg_cap = 0;
8757         if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS))
8758                 return;
8759
8760         rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS);
8761         if (rc)
8762                 return;
8763
8764         req->fid = cpu_to_le16(0xffff);
8765         resp = hwrm_req_hold(bp, req);
8766         rc = hwrm_req_send(bp, req);
8767         if (rc)
8768                 goto hwrm_dbg_qcaps_exit;
8769
8770         bp->fw_dbg_cap = le32_to_cpu(resp->flags);
8771
8772 hwrm_dbg_qcaps_exit:
8773         hwrm_req_drop(bp, req);
8774 }
8775
8776 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
8777
8778 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
8779 {
8780         int rc;
8781
8782         rc = __bnxt_hwrm_func_qcaps(bp);
8783         if (rc)
8784                 return rc;
8785
8786         bnxt_hwrm_dbg_qcaps(bp);
8787
8788         rc = bnxt_hwrm_queue_qportcfg(bp);
8789         if (rc) {
8790                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
8791                 return rc;
8792         }
8793         if (bp->hwrm_spec_code >= 0x10803) {
8794                 rc = bnxt_alloc_ctx_mem(bp);
8795                 if (rc)
8796                         return rc;
8797                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
8798                 if (!rc)
8799                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
8800         }
8801         return 0;
8802 }
8803
8804 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
8805 {
8806         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
8807         struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req;
8808         u32 flags;
8809         int rc;
8810
8811         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
8812                 return 0;
8813
8814         rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS);
8815         if (rc)
8816                 return rc;
8817
8818         resp = hwrm_req_hold(bp, req);
8819         rc = hwrm_req_send(bp, req);
8820         if (rc)
8821                 goto hwrm_cfa_adv_qcaps_exit;
8822
8823         flags = le32_to_cpu(resp->flags);
8824         if (flags &
8825             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
8826                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
8827
8828 hwrm_cfa_adv_qcaps_exit:
8829         hwrm_req_drop(bp, req);
8830         return rc;
8831 }
8832
8833 static int __bnxt_alloc_fw_health(struct bnxt *bp)
8834 {
8835         if (bp->fw_health)
8836                 return 0;
8837
8838         bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
8839         if (!bp->fw_health)
8840                 return -ENOMEM;
8841
8842         mutex_init(&bp->fw_health->lock);
8843         return 0;
8844 }
8845
8846 static int bnxt_alloc_fw_health(struct bnxt *bp)
8847 {
8848         int rc;
8849
8850         if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
8851             !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
8852                 return 0;
8853
8854         rc = __bnxt_alloc_fw_health(bp);
8855         if (rc) {
8856                 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
8857                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
8858                 return rc;
8859         }
8860
8861         return 0;
8862 }
8863
8864 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
8865 {
8866         writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
8867                                          BNXT_GRCPF_REG_WINDOW_BASE_OUT +
8868                                          BNXT_FW_HEALTH_WIN_MAP_OFF);
8869 }
8870
8871 static void bnxt_inv_fw_health_reg(struct bnxt *bp)
8872 {
8873         struct bnxt_fw_health *fw_health = bp->fw_health;
8874         u32 reg_type;
8875
8876         if (!fw_health)
8877                 return;
8878
8879         reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
8880         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
8881                 fw_health->status_reliable = false;
8882
8883         reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]);
8884         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
8885                 fw_health->resets_reliable = false;
8886 }
8887
8888 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
8889 {
8890         void __iomem *hs;
8891         u32 status_loc;
8892         u32 reg_type;
8893         u32 sig;
8894
8895         if (bp->fw_health)
8896                 bp->fw_health->status_reliable = false;
8897
8898         __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
8899         hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
8900
8901         sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
8902         if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
8903                 if (!bp->chip_num) {
8904                         __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
8905                         bp->chip_num = readl(bp->bar0 +
8906                                              BNXT_FW_HEALTH_WIN_BASE +
8907                                              BNXT_GRC_REG_CHIP_NUM);
8908                 }
8909                 if (!BNXT_CHIP_P5(bp))
8910                         return;
8911
8912                 status_loc = BNXT_GRC_REG_STATUS_P5 |
8913                              BNXT_FW_HEALTH_REG_TYPE_BAR0;
8914         } else {
8915                 status_loc = readl(hs + offsetof(struct hcomm_status,
8916                                                  fw_status_loc));
8917         }
8918
8919         if (__bnxt_alloc_fw_health(bp)) {
8920                 netdev_warn(bp->dev, "no memory for firmware status checks\n");
8921                 return;
8922         }
8923
8924         bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
8925         reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
8926         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
8927                 __bnxt_map_fw_health_reg(bp, status_loc);
8928                 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
8929                         BNXT_FW_HEALTH_WIN_OFF(status_loc);
8930         }
8931
8932         bp->fw_health->status_reliable = true;
8933 }
8934
8935 static int bnxt_map_fw_health_regs(struct bnxt *bp)
8936 {
8937         struct bnxt_fw_health *fw_health = bp->fw_health;
8938         u32 reg_base = 0xffffffff;
8939         int i;
8940
8941         bp->fw_health->status_reliable = false;
8942         bp->fw_health->resets_reliable = false;
8943         /* Only pre-map the monitoring GRC registers using window 3 */
8944         for (i = 0; i < 4; i++) {
8945                 u32 reg = fw_health->regs[i];
8946
8947                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
8948                         continue;
8949                 if (reg_base == 0xffffffff)
8950                         reg_base = reg & BNXT_GRC_BASE_MASK;
8951                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
8952                         return -ERANGE;
8953                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
8954         }
8955         bp->fw_health->status_reliable = true;
8956         bp->fw_health->resets_reliable = true;
8957         if (reg_base == 0xffffffff)
8958                 return 0;
8959
8960         __bnxt_map_fw_health_reg(bp, reg_base);
8961         return 0;
8962 }
8963
8964 static void bnxt_remap_fw_health_regs(struct bnxt *bp)
8965 {
8966         if (!bp->fw_health)
8967                 return;
8968
8969         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) {
8970                 bp->fw_health->status_reliable = true;
8971                 bp->fw_health->resets_reliable = true;
8972         } else {
8973                 bnxt_try_map_fw_health_reg(bp);
8974         }
8975 }
8976
8977 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
8978 {
8979         struct bnxt_fw_health *fw_health = bp->fw_health;
8980         struct hwrm_error_recovery_qcfg_output *resp;
8981         struct hwrm_error_recovery_qcfg_input *req;
8982         int rc, i;
8983
8984         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
8985                 return 0;
8986
8987         rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG);
8988         if (rc)
8989                 return rc;
8990
8991         resp = hwrm_req_hold(bp, req);
8992         rc = hwrm_req_send(bp, req);
8993         if (rc)
8994                 goto err_recovery_out;
8995         fw_health->flags = le32_to_cpu(resp->flags);
8996         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
8997             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
8998                 rc = -EINVAL;
8999                 goto err_recovery_out;
9000         }
9001         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
9002         fw_health->master_func_wait_dsecs =
9003                 le32_to_cpu(resp->master_func_wait_period);
9004         fw_health->normal_func_wait_dsecs =
9005                 le32_to_cpu(resp->normal_func_wait_period);
9006         fw_health->post_reset_wait_dsecs =
9007                 le32_to_cpu(resp->master_func_wait_period_after_reset);
9008         fw_health->post_reset_max_wait_dsecs =
9009                 le32_to_cpu(resp->max_bailout_time_after_reset);
9010         fw_health->regs[BNXT_FW_HEALTH_REG] =
9011                 le32_to_cpu(resp->fw_health_status_reg);
9012         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
9013                 le32_to_cpu(resp->fw_heartbeat_reg);
9014         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
9015                 le32_to_cpu(resp->fw_reset_cnt_reg);
9016         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
9017                 le32_to_cpu(resp->reset_inprogress_reg);
9018         fw_health->fw_reset_inprog_reg_mask =
9019                 le32_to_cpu(resp->reset_inprogress_reg_mask);
9020         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
9021         if (fw_health->fw_reset_seq_cnt >= 16) {
9022                 rc = -EINVAL;
9023                 goto err_recovery_out;
9024         }
9025         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
9026                 fw_health->fw_reset_seq_regs[i] =
9027                         le32_to_cpu(resp->reset_reg[i]);
9028                 fw_health->fw_reset_seq_vals[i] =
9029                         le32_to_cpu(resp->reset_reg_val[i]);
9030                 fw_health->fw_reset_seq_delay_msec[i] =
9031                         resp->delay_after_reset[i];
9032         }
9033 err_recovery_out:
9034         hwrm_req_drop(bp, req);
9035         if (!rc)
9036                 rc = bnxt_map_fw_health_regs(bp);
9037         if (rc)
9038                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
9039         return rc;
9040 }
9041
9042 static int bnxt_hwrm_func_reset(struct bnxt *bp)
9043 {
9044         struct hwrm_func_reset_input *req;
9045         int rc;
9046
9047         rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET);
9048         if (rc)
9049                 return rc;
9050
9051         req->enables = 0;
9052         hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT);
9053         return hwrm_req_send(bp, req);
9054 }
9055
9056 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
9057 {
9058         struct hwrm_nvm_get_dev_info_output nvm_info;
9059
9060         if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
9061                 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
9062                          nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
9063                          nvm_info.nvm_cfg_ver_upd);
9064 }
9065
9066 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
9067 {
9068         struct hwrm_queue_qportcfg_output *resp;
9069         struct hwrm_queue_qportcfg_input *req;
9070         u8 i, j, *qptr;
9071         bool no_rdma;
9072         int rc = 0;
9073
9074         rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG);
9075         if (rc)
9076                 return rc;
9077
9078         resp = hwrm_req_hold(bp, req);
9079         rc = hwrm_req_send(bp, req);
9080         if (rc)
9081                 goto qportcfg_exit;
9082
9083         if (!resp->max_configurable_queues) {
9084                 rc = -EINVAL;
9085                 goto qportcfg_exit;
9086         }
9087         bp->max_tc = resp->max_configurable_queues;
9088         bp->max_lltc = resp->max_configurable_lossless_queues;
9089         if (bp->max_tc > BNXT_MAX_QUEUE)
9090                 bp->max_tc = BNXT_MAX_QUEUE;
9091
9092         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
9093         qptr = &resp->queue_id0;
9094         for (i = 0, j = 0; i < bp->max_tc; i++) {
9095                 bp->q_info[j].queue_id = *qptr;
9096                 bp->q_ids[i] = *qptr++;
9097                 bp->q_info[j].queue_profile = *qptr++;
9098                 bp->tc_to_qidx[j] = j;
9099                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
9100                     (no_rdma && BNXT_PF(bp)))
9101                         j++;
9102         }
9103         bp->max_q = bp->max_tc;
9104         bp->max_tc = max_t(u8, j, 1);
9105
9106         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
9107                 bp->max_tc = 1;
9108
9109         if (bp->max_lltc > bp->max_tc)
9110                 bp->max_lltc = bp->max_tc;
9111
9112 qportcfg_exit:
9113         hwrm_req_drop(bp, req);
9114         return rc;
9115 }
9116
9117 static int bnxt_hwrm_poll(struct bnxt *bp)
9118 {
9119         struct hwrm_ver_get_input *req;
9120         int rc;
9121
9122         rc = hwrm_req_init(bp, req, HWRM_VER_GET);
9123         if (rc)
9124                 return rc;
9125
9126         req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
9127         req->hwrm_intf_min = HWRM_VERSION_MINOR;
9128         req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
9129
9130         hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT);
9131         rc = hwrm_req_send(bp, req);
9132         return rc;
9133 }
9134
9135 static int bnxt_hwrm_ver_get(struct bnxt *bp)
9136 {
9137         struct hwrm_ver_get_output *resp;
9138         struct hwrm_ver_get_input *req;
9139         u16 fw_maj, fw_min, fw_bld, fw_rsv;
9140         u32 dev_caps_cfg, hwrm_ver;
9141         int rc, len;
9142
9143         rc = hwrm_req_init(bp, req, HWRM_VER_GET);
9144         if (rc)
9145                 return rc;
9146
9147         hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
9148         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
9149         req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
9150         req->hwrm_intf_min = HWRM_VERSION_MINOR;
9151         req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
9152
9153         resp = hwrm_req_hold(bp, req);
9154         rc = hwrm_req_send(bp, req);
9155         if (rc)
9156                 goto hwrm_ver_get_exit;
9157
9158         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
9159
9160         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
9161                              resp->hwrm_intf_min_8b << 8 |
9162                              resp->hwrm_intf_upd_8b;
9163         if (resp->hwrm_intf_maj_8b < 1) {
9164                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
9165                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
9166                             resp->hwrm_intf_upd_8b);
9167                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
9168         }
9169
9170         hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
9171                         HWRM_VERSION_UPDATE;
9172
9173         if (bp->hwrm_spec_code > hwrm_ver)
9174                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
9175                          HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
9176                          HWRM_VERSION_UPDATE);
9177         else
9178                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
9179                          resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
9180                          resp->hwrm_intf_upd_8b);
9181
9182         fw_maj = le16_to_cpu(resp->hwrm_fw_major);
9183         if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
9184                 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
9185                 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
9186                 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
9187                 len = FW_VER_STR_LEN;
9188         } else {
9189                 fw_maj = resp->hwrm_fw_maj_8b;
9190                 fw_min = resp->hwrm_fw_min_8b;
9191                 fw_bld = resp->hwrm_fw_bld_8b;
9192                 fw_rsv = resp->hwrm_fw_rsvd_8b;
9193                 len = BC_HWRM_STR_LEN;
9194         }
9195         bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
9196         snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
9197                  fw_rsv);
9198
9199         if (strlen(resp->active_pkg_name)) {
9200                 int fw_ver_len = strlen(bp->fw_ver_str);
9201
9202                 snprintf(bp->fw_ver_str + fw_ver_len,
9203                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
9204                          resp->active_pkg_name);
9205                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
9206         }
9207
9208         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
9209         if (!bp->hwrm_cmd_timeout)
9210                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
9211         bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
9212         if (!bp->hwrm_cmd_max_timeout)
9213                 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
9214         else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
9215                 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
9216                             bp->hwrm_cmd_max_timeout / 1000);
9217
9218         if (resp->hwrm_intf_maj_8b >= 1) {
9219                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
9220                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
9221         }
9222         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
9223                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
9224
9225         bp->chip_num = le16_to_cpu(resp->chip_num);
9226         bp->chip_rev = resp->chip_rev;
9227         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
9228             !resp->chip_metal)
9229                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
9230
9231         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
9232         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
9233             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
9234                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
9235
9236         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
9237                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
9238
9239         if (dev_caps_cfg &
9240             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
9241                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
9242
9243         if (dev_caps_cfg &
9244             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
9245                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
9246
9247         if (dev_caps_cfg &
9248             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
9249                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
9250
9251 hwrm_ver_get_exit:
9252         hwrm_req_drop(bp, req);
9253         return rc;
9254 }
9255
9256 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
9257 {
9258         struct hwrm_fw_set_time_input *req;
9259         struct tm tm;
9260         time64_t now = ktime_get_real_seconds();
9261         int rc;
9262
9263         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
9264             bp->hwrm_spec_code < 0x10400)
9265                 return -EOPNOTSUPP;
9266
9267         time64_to_tm(now, 0, &tm);
9268         rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME);
9269         if (rc)
9270                 return rc;
9271
9272         req->year = cpu_to_le16(1900 + tm.tm_year);
9273         req->month = 1 + tm.tm_mon;
9274         req->day = tm.tm_mday;
9275         req->hour = tm.tm_hour;
9276         req->minute = tm.tm_min;
9277         req->second = tm.tm_sec;
9278         return hwrm_req_send(bp, req);
9279 }
9280
9281 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
9282 {
9283         u64 sw_tmp;
9284
9285         hw &= mask;
9286         sw_tmp = (*sw & ~mask) | hw;
9287         if (hw < (*sw & mask))
9288                 sw_tmp += mask + 1;
9289         WRITE_ONCE(*sw, sw_tmp);
9290 }
9291
9292 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
9293                                     int count, bool ignore_zero)
9294 {
9295         int i;
9296
9297         for (i = 0; i < count; i++) {
9298                 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
9299
9300                 if (ignore_zero && !hw)
9301                         continue;
9302
9303                 if (masks[i] == -1ULL)
9304                         sw_stats[i] = hw;
9305                 else
9306                         bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
9307         }
9308 }
9309
9310 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
9311 {
9312         if (!stats->hw_stats)
9313                 return;
9314
9315         __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
9316                                 stats->hw_masks, stats->len / 8, false);
9317 }
9318
9319 static void bnxt_accumulate_all_stats(struct bnxt *bp)
9320 {
9321         struct bnxt_stats_mem *ring0_stats;
9322         bool ignore_zero = false;
9323         int i;
9324
9325         /* Chip bug.  Counter intermittently becomes 0. */
9326         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9327                 ignore_zero = true;
9328
9329         for (i = 0; i < bp->cp_nr_rings; i++) {
9330                 struct bnxt_napi *bnapi = bp->bnapi[i];
9331                 struct bnxt_cp_ring_info *cpr;
9332                 struct bnxt_stats_mem *stats;
9333
9334                 cpr = &bnapi->cp_ring;
9335                 stats = &cpr->stats;
9336                 if (!i)
9337                         ring0_stats = stats;
9338                 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
9339                                         ring0_stats->hw_masks,
9340                                         ring0_stats->len / 8, ignore_zero);
9341         }
9342         if (bp->flags & BNXT_FLAG_PORT_STATS) {
9343                 struct bnxt_stats_mem *stats = &bp->port_stats;
9344                 __le64 *hw_stats = stats->hw_stats;
9345                 u64 *sw_stats = stats->sw_stats;
9346                 u64 *masks = stats->hw_masks;
9347                 int cnt;
9348
9349                 cnt = sizeof(struct rx_port_stats) / 8;
9350                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
9351
9352                 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9353                 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9354                 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9355                 cnt = sizeof(struct tx_port_stats) / 8;
9356                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
9357         }
9358         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
9359                 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
9360                 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
9361         }
9362 }
9363
9364 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
9365 {
9366         struct hwrm_port_qstats_input *req;
9367         struct bnxt_pf_info *pf = &bp->pf;
9368         int rc;
9369
9370         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
9371                 return 0;
9372
9373         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
9374                 return -EOPNOTSUPP;
9375
9376         rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS);
9377         if (rc)
9378                 return rc;
9379
9380         req->flags = flags;
9381         req->port_id = cpu_to_le16(pf->port_id);
9382         req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
9383                                             BNXT_TX_PORT_STATS_BYTE_OFFSET);
9384         req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
9385         return hwrm_req_send(bp, req);
9386 }
9387
9388 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
9389 {
9390         struct hwrm_queue_pri2cos_qcfg_output *resp_qc;
9391         struct hwrm_queue_pri2cos_qcfg_input *req_qc;
9392         struct hwrm_port_qstats_ext_output *resp_qs;
9393         struct hwrm_port_qstats_ext_input *req_qs;
9394         struct bnxt_pf_info *pf = &bp->pf;
9395         u32 tx_stat_size;
9396         int rc;
9397
9398         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
9399                 return 0;
9400
9401         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
9402                 return -EOPNOTSUPP;
9403
9404         rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT);
9405         if (rc)
9406                 return rc;
9407
9408         req_qs->flags = flags;
9409         req_qs->port_id = cpu_to_le16(pf->port_id);
9410         req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
9411         req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
9412         tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
9413                        sizeof(struct tx_port_stats_ext) : 0;
9414         req_qs->tx_stat_size = cpu_to_le16(tx_stat_size);
9415         req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
9416         resp_qs = hwrm_req_hold(bp, req_qs);
9417         rc = hwrm_req_send(bp, req_qs);
9418         if (!rc) {
9419                 bp->fw_rx_stats_ext_size =
9420                         le16_to_cpu(resp_qs->rx_stat_size) / 8;
9421                 if (BNXT_FW_MAJ(bp) < 220 &&
9422                     bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY)
9423                         bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY;
9424
9425                 bp->fw_tx_stats_ext_size = tx_stat_size ?
9426                         le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0;
9427         } else {
9428                 bp->fw_rx_stats_ext_size = 0;
9429                 bp->fw_tx_stats_ext_size = 0;
9430         }
9431         hwrm_req_drop(bp, req_qs);
9432
9433         if (flags)
9434                 return rc;
9435
9436         if (bp->fw_tx_stats_ext_size <=
9437             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
9438                 bp->pri2cos_valid = 0;
9439                 return rc;
9440         }
9441
9442         rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG);
9443         if (rc)
9444                 return rc;
9445
9446         req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
9447
9448         resp_qc = hwrm_req_hold(bp, req_qc);
9449         rc = hwrm_req_send(bp, req_qc);
9450         if (!rc) {
9451                 u8 *pri2cos;
9452                 int i, j;
9453
9454                 pri2cos = &resp_qc->pri0_cos_queue_id;
9455                 for (i = 0; i < 8; i++) {
9456                         u8 queue_id = pri2cos[i];
9457                         u8 queue_idx;
9458
9459                         /* Per port queue IDs start from 0, 10, 20, etc */
9460                         queue_idx = queue_id % 10;
9461                         if (queue_idx > BNXT_MAX_QUEUE) {
9462                                 bp->pri2cos_valid = false;
9463                                 hwrm_req_drop(bp, req_qc);
9464                                 return rc;
9465                         }
9466                         for (j = 0; j < bp->max_q; j++) {
9467                                 if (bp->q_ids[j] == queue_id)
9468                                         bp->pri2cos_idx[i] = queue_idx;
9469                         }
9470                 }
9471                 bp->pri2cos_valid = true;
9472         }
9473         hwrm_req_drop(bp, req_qc);
9474
9475         return rc;
9476 }
9477
9478 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
9479 {
9480         bnxt_hwrm_tunnel_dst_port_free(bp,
9481                 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
9482         bnxt_hwrm_tunnel_dst_port_free(bp,
9483                 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
9484 }
9485
9486 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
9487 {
9488         int rc, i;
9489         u32 tpa_flags = 0;
9490
9491         if (set_tpa)
9492                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
9493         else if (BNXT_NO_FW_ACCESS(bp))
9494                 return 0;
9495         for (i = 0; i < bp->nr_vnics; i++) {
9496                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
9497                 if (rc) {
9498                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
9499                                    i, rc);
9500                         return rc;
9501                 }
9502         }
9503         return 0;
9504 }
9505
9506 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
9507 {
9508         int i;
9509
9510         for (i = 0; i < bp->nr_vnics; i++)
9511                 bnxt_hwrm_vnic_set_rss(bp, i, false);
9512 }
9513
9514 static void bnxt_clear_vnic(struct bnxt *bp)
9515 {
9516         if (!bp->vnic_info)
9517                 return;
9518
9519         bnxt_hwrm_clear_vnic_filter(bp);
9520         if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) {
9521                 /* clear all RSS setting before free vnic ctx */
9522                 bnxt_hwrm_clear_vnic_rss(bp);
9523                 bnxt_hwrm_vnic_ctx_free(bp);
9524         }
9525         /* before free the vnic, undo the vnic tpa settings */
9526         if (bp->flags & BNXT_FLAG_TPA)
9527                 bnxt_set_tpa(bp, false);
9528         bnxt_hwrm_vnic_free(bp);
9529         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9530                 bnxt_hwrm_vnic_ctx_free(bp);
9531 }
9532
9533 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
9534                                     bool irq_re_init)
9535 {
9536         bnxt_clear_vnic(bp);
9537         bnxt_hwrm_ring_free(bp, close_path);
9538         bnxt_hwrm_ring_grp_free(bp);
9539         if (irq_re_init) {
9540                 bnxt_hwrm_stat_ctx_free(bp);
9541                 bnxt_hwrm_free_tunnel_ports(bp);
9542         }
9543 }
9544
9545 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
9546 {
9547         struct hwrm_func_cfg_input *req;
9548         u8 evb_mode;
9549         int rc;
9550
9551         if (br_mode == BRIDGE_MODE_VEB)
9552                 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
9553         else if (br_mode == BRIDGE_MODE_VEPA)
9554                 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
9555         else
9556                 return -EINVAL;
9557
9558         rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
9559         if (rc)
9560                 return rc;
9561
9562         req->fid = cpu_to_le16(0xffff);
9563         req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
9564         req->evb_mode = evb_mode;
9565         return hwrm_req_send(bp, req);
9566 }
9567
9568 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
9569 {
9570         struct hwrm_func_cfg_input *req;
9571         int rc;
9572
9573         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
9574                 return 0;
9575
9576         rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
9577         if (rc)
9578                 return rc;
9579
9580         req->fid = cpu_to_le16(0xffff);
9581         req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
9582         req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
9583         if (size == 128)
9584                 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
9585
9586         return hwrm_req_send(bp, req);
9587 }
9588
9589 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
9590 {
9591         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
9592         int rc;
9593
9594         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
9595                 goto skip_rss_ctx;
9596
9597         /* allocate context for vnic */
9598         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
9599         if (rc) {
9600                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
9601                            vnic_id, rc);
9602                 goto vnic_setup_err;
9603         }
9604         bp->rsscos_nr_ctxs++;
9605
9606         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9607                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
9608                 if (rc) {
9609                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
9610                                    vnic_id, rc);
9611                         goto vnic_setup_err;
9612                 }
9613                 bp->rsscos_nr_ctxs++;
9614         }
9615
9616 skip_rss_ctx:
9617         /* configure default vnic, ring grp */
9618         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
9619         if (rc) {
9620                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
9621                            vnic_id, rc);
9622                 goto vnic_setup_err;
9623         }
9624
9625         /* Enable RSS hashing on vnic */
9626         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
9627         if (rc) {
9628                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
9629                            vnic_id, rc);
9630                 goto vnic_setup_err;
9631         }
9632
9633         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
9634                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
9635                 if (rc) {
9636                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
9637                                    vnic_id, rc);
9638                 }
9639         }
9640
9641 vnic_setup_err:
9642         return rc;
9643 }
9644
9645 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
9646 {
9647         int rc, i, nr_ctxs;
9648
9649         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
9650         for (i = 0; i < nr_ctxs; i++) {
9651                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
9652                 if (rc) {
9653                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
9654                                    vnic_id, i, rc);
9655                         break;
9656                 }
9657                 bp->rsscos_nr_ctxs++;
9658         }
9659         if (i < nr_ctxs)
9660                 return -ENOMEM;
9661
9662         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
9663         if (rc) {
9664                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
9665                            vnic_id, rc);
9666                 return rc;
9667         }
9668         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
9669         if (rc) {
9670                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
9671                            vnic_id, rc);
9672                 return rc;
9673         }
9674         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
9675                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
9676                 if (rc) {
9677                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
9678                                    vnic_id, rc);
9679                 }
9680         }
9681         return rc;
9682 }
9683
9684 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
9685 {
9686         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9687                 return __bnxt_setup_vnic_p5(bp, vnic_id);
9688         else
9689                 return __bnxt_setup_vnic(bp, vnic_id);
9690 }
9691
9692 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
9693 {
9694         int i, rc = 0;
9695
9696         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
9697                 return 0;
9698
9699         for (i = 0; i < bp->rx_nr_rings; i++) {
9700                 struct bnxt_vnic_info *vnic;
9701                 u16 vnic_id = i + 1;
9702                 u16 ring_id = i;
9703
9704                 if (vnic_id >= bp->nr_vnics)
9705                         break;
9706
9707                 vnic = &bp->vnic_info[vnic_id];
9708                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
9709                 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
9710                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
9711                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
9712                 if (rc) {
9713                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
9714                                    vnic_id, rc);
9715                         break;
9716                 }
9717                 rc = bnxt_setup_vnic(bp, vnic_id);
9718                 if (rc)
9719                         break;
9720         }
9721         return rc;
9722 }
9723
9724 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
9725 static bool bnxt_promisc_ok(struct bnxt *bp)
9726 {
9727 #ifdef CONFIG_BNXT_SRIOV
9728         if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
9729                 return false;
9730 #endif
9731         return true;
9732 }
9733
9734 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
9735 {
9736         unsigned int rc = 0;
9737
9738         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
9739         if (rc) {
9740                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
9741                            rc);
9742                 return rc;
9743         }
9744
9745         rc = bnxt_hwrm_vnic_cfg(bp, 1);
9746         if (rc) {
9747                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
9748                            rc);
9749                 return rc;
9750         }
9751         return rc;
9752 }
9753
9754 static int bnxt_cfg_rx_mode(struct bnxt *);
9755 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
9756
9757 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
9758 {
9759         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9760         int rc = 0;
9761         unsigned int rx_nr_rings = bp->rx_nr_rings;
9762
9763         if (irq_re_init) {
9764                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
9765                 if (rc) {
9766                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
9767                                    rc);
9768                         goto err_out;
9769                 }
9770         }
9771
9772         rc = bnxt_hwrm_ring_alloc(bp);
9773         if (rc) {
9774                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
9775                 goto err_out;
9776         }
9777
9778         rc = bnxt_hwrm_ring_grp_alloc(bp);
9779         if (rc) {
9780                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
9781                 goto err_out;
9782         }
9783
9784         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
9785                 rx_nr_rings--;
9786
9787         /* default vnic 0 */
9788         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
9789         if (rc) {
9790                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
9791                 goto err_out;
9792         }
9793
9794         if (BNXT_VF(bp))
9795                 bnxt_hwrm_func_qcfg(bp);
9796
9797         rc = bnxt_setup_vnic(bp, 0);
9798         if (rc)
9799                 goto err_out;
9800         if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA)
9801                 bnxt_hwrm_update_rss_hash_cfg(bp);
9802
9803         if (bp->flags & BNXT_FLAG_RFS) {
9804                 rc = bnxt_alloc_rfs_vnics(bp);
9805                 if (rc)
9806                         goto err_out;
9807         }
9808
9809         if (bp->flags & BNXT_FLAG_TPA) {
9810                 rc = bnxt_set_tpa(bp, true);
9811                 if (rc)
9812                         goto err_out;
9813         }
9814
9815         if (BNXT_VF(bp))
9816                 bnxt_update_vf_mac(bp);
9817
9818         /* Filter for default vnic 0 */
9819         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
9820         if (rc) {
9821                 if (BNXT_VF(bp) && rc == -ENODEV)
9822                         netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n");
9823                 else
9824                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
9825                 goto err_out;
9826         }
9827         vnic->uc_filter_count = 1;
9828
9829         vnic->rx_mask = 0;
9830         if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
9831                 goto skip_rx_mask;
9832
9833         if (bp->dev->flags & IFF_BROADCAST)
9834                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
9835
9836         if (bp->dev->flags & IFF_PROMISC)
9837                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9838
9839         if (bp->dev->flags & IFF_ALLMULTI) {
9840                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9841                 vnic->mc_list_count = 0;
9842         } else if (bp->dev->flags & IFF_MULTICAST) {
9843                 u32 mask = 0;
9844
9845                 bnxt_mc_list_updated(bp, &mask);
9846                 vnic->rx_mask |= mask;
9847         }
9848
9849         rc = bnxt_cfg_rx_mode(bp);
9850         if (rc)
9851                 goto err_out;
9852
9853 skip_rx_mask:
9854         rc = bnxt_hwrm_set_coal(bp);
9855         if (rc)
9856                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
9857                                 rc);
9858
9859         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9860                 rc = bnxt_setup_nitroa0_vnic(bp);
9861                 if (rc)
9862                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
9863                                    rc);
9864         }
9865
9866         if (BNXT_VF(bp)) {
9867                 bnxt_hwrm_func_qcfg(bp);
9868                 netdev_update_features(bp->dev);
9869         }
9870
9871         return 0;
9872
9873 err_out:
9874         bnxt_hwrm_resource_free(bp, 0, true);
9875
9876         return rc;
9877 }
9878
9879 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
9880 {
9881         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
9882         return 0;
9883 }
9884
9885 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
9886 {
9887         bnxt_init_cp_rings(bp);
9888         bnxt_init_rx_rings(bp);
9889         bnxt_init_tx_rings(bp);
9890         bnxt_init_ring_grps(bp, irq_re_init);
9891         bnxt_init_vnics(bp);
9892
9893         return bnxt_init_chip(bp, irq_re_init);
9894 }
9895
9896 static int bnxt_set_real_num_queues(struct bnxt *bp)
9897 {
9898         int rc;
9899         struct net_device *dev = bp->dev;
9900
9901         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
9902                                           bp->tx_nr_rings_xdp);
9903         if (rc)
9904                 return rc;
9905
9906         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
9907         if (rc)
9908                 return rc;
9909
9910 #ifdef CONFIG_RFS_ACCEL
9911         if (bp->flags & BNXT_FLAG_RFS)
9912                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
9913 #endif
9914
9915         return rc;
9916 }
9917
9918 static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
9919                              bool shared)
9920 {
9921         int _rx = *rx, _tx = *tx;
9922
9923         if (shared) {
9924                 *rx = min_t(int, _rx, max);
9925                 *tx = min_t(int, _tx, max);
9926         } else {
9927                 if (max < 2)
9928                         return -ENOMEM;
9929
9930                 while (_rx + _tx > max) {
9931                         if (_rx > _tx && _rx > 1)
9932                                 _rx--;
9933                         else if (_tx > 1)
9934                                 _tx--;
9935                 }
9936                 *rx = _rx;
9937                 *tx = _tx;
9938         }
9939         return 0;
9940 }
9941
9942 static int __bnxt_num_tx_to_cp(struct bnxt *bp, int tx, int tx_sets, int tx_xdp)
9943 {
9944         return (tx - tx_xdp) / tx_sets + tx_xdp;
9945 }
9946
9947 int bnxt_num_tx_to_cp(struct bnxt *bp, int tx)
9948 {
9949         int tcs = bp->num_tc;
9950
9951         if (!tcs)
9952                 tcs = 1;
9953         return __bnxt_num_tx_to_cp(bp, tx, tcs, bp->tx_nr_rings_xdp);
9954 }
9955
9956 static int bnxt_num_cp_to_tx(struct bnxt *bp, int tx_cp)
9957 {
9958         int tcs = bp->num_tc;
9959
9960         return (tx_cp - bp->tx_nr_rings_xdp) * tcs +
9961                bp->tx_nr_rings_xdp;
9962 }
9963
9964 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
9965                            bool sh)
9966 {
9967         int tx_cp = bnxt_num_tx_to_cp(bp, *tx);
9968
9969         if (tx_cp != *tx) {
9970                 int tx_saved = tx_cp, rc;
9971
9972                 rc = __bnxt_trim_rings(bp, rx, &tx_cp, max, sh);
9973                 if (rc)
9974                         return rc;
9975                 if (tx_cp != tx_saved)
9976                         *tx = bnxt_num_cp_to_tx(bp, tx_cp);
9977                 return 0;
9978         }
9979         return __bnxt_trim_rings(bp, rx, tx, max, sh);
9980 }
9981
9982 static void bnxt_setup_msix(struct bnxt *bp)
9983 {
9984         const int len = sizeof(bp->irq_tbl[0].name);
9985         struct net_device *dev = bp->dev;
9986         int tcs, i;
9987
9988         tcs = bp->num_tc;
9989         if (tcs) {
9990                 int i, off, count;
9991
9992                 for (i = 0; i < tcs; i++) {
9993                         count = bp->tx_nr_rings_per_tc;
9994                         off = BNXT_TC_TO_RING_BASE(bp, i);
9995                         netdev_set_tc_queue(dev, i, count, off);
9996                 }
9997         }
9998
9999         for (i = 0; i < bp->cp_nr_rings; i++) {
10000                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10001                 char *attr;
10002
10003                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
10004                         attr = "TxRx";
10005                 else if (i < bp->rx_nr_rings)
10006                         attr = "rx";
10007                 else
10008                         attr = "tx";
10009
10010                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
10011                          attr, i);
10012                 bp->irq_tbl[map_idx].handler = bnxt_msix;
10013         }
10014 }
10015
10016 static void bnxt_setup_inta(struct bnxt *bp)
10017 {
10018         const int len = sizeof(bp->irq_tbl[0].name);
10019
10020         if (bp->num_tc) {
10021                 netdev_reset_tc(bp->dev);
10022                 bp->num_tc = 0;
10023         }
10024
10025         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
10026                  0);
10027         bp->irq_tbl[0].handler = bnxt_inta;
10028 }
10029
10030 static int bnxt_init_int_mode(struct bnxt *bp);
10031
10032 static int bnxt_setup_int_mode(struct bnxt *bp)
10033 {
10034         int rc;
10035
10036         if (!bp->irq_tbl) {
10037                 rc = bnxt_init_int_mode(bp);
10038                 if (rc || !bp->irq_tbl)
10039                         return rc ?: -ENODEV;
10040         }
10041
10042         if (bp->flags & BNXT_FLAG_USING_MSIX)
10043                 bnxt_setup_msix(bp);
10044         else
10045                 bnxt_setup_inta(bp);
10046
10047         rc = bnxt_set_real_num_queues(bp);
10048         return rc;
10049 }
10050
10051 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
10052 {
10053         return bp->hw_resc.max_rsscos_ctxs;
10054 }
10055
10056 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
10057 {
10058         return bp->hw_resc.max_vnics;
10059 }
10060
10061 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
10062 {
10063         return bp->hw_resc.max_stat_ctxs;
10064 }
10065
10066 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
10067 {
10068         return bp->hw_resc.max_cp_rings;
10069 }
10070
10071 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
10072 {
10073         unsigned int cp = bp->hw_resc.max_cp_rings;
10074
10075         if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
10076                 cp -= bnxt_get_ulp_msix_num(bp);
10077
10078         return cp;
10079 }
10080
10081 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
10082 {
10083         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10084
10085         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10086                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
10087
10088         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
10089 }
10090
10091 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
10092 {
10093         bp->hw_resc.max_irqs = max_irqs;
10094 }
10095
10096 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
10097 {
10098         unsigned int cp;
10099
10100         cp = bnxt_get_max_func_cp_rings_for_en(bp);
10101         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10102                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
10103         else
10104                 return cp - bp->cp_nr_rings;
10105 }
10106
10107 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
10108 {
10109         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
10110 }
10111
10112 int bnxt_get_avail_msix(struct bnxt *bp, int num)
10113 {
10114         int max_cp = bnxt_get_max_func_cp_rings(bp);
10115         int max_irq = bnxt_get_max_func_irqs(bp);
10116         int total_req = bp->cp_nr_rings + num;
10117         int max_idx, avail_msix;
10118
10119         max_idx = bp->total_irqs;
10120         if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
10121                 max_idx = min_t(int, bp->total_irqs, max_cp);
10122         avail_msix = max_idx - bp->cp_nr_rings;
10123         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
10124                 return avail_msix;
10125
10126         if (max_irq < total_req) {
10127                 num = max_irq - bp->cp_nr_rings;
10128                 if (num <= 0)
10129                         return 0;
10130         }
10131         return num;
10132 }
10133
10134 static int bnxt_get_num_msix(struct bnxt *bp)
10135 {
10136         if (!BNXT_NEW_RM(bp))
10137                 return bnxt_get_max_func_irqs(bp);
10138
10139         return bnxt_nq_rings_in_use(bp);
10140 }
10141
10142 static int bnxt_init_msix(struct bnxt *bp)
10143 {
10144         int i, total_vecs, max, rc = 0, min = 1, ulp_msix, tx_cp;
10145         struct msix_entry *msix_ent;
10146
10147         total_vecs = bnxt_get_num_msix(bp);
10148         max = bnxt_get_max_func_irqs(bp);
10149         if (total_vecs > max)
10150                 total_vecs = max;
10151
10152         if (!total_vecs)
10153                 return 0;
10154
10155         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
10156         if (!msix_ent)
10157                 return -ENOMEM;
10158
10159         for (i = 0; i < total_vecs; i++) {
10160                 msix_ent[i].entry = i;
10161                 msix_ent[i].vector = 0;
10162         }
10163
10164         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
10165                 min = 2;
10166
10167         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
10168         ulp_msix = bnxt_get_ulp_msix_num(bp);
10169         if (total_vecs < 0 || total_vecs < ulp_msix) {
10170                 rc = -ENODEV;
10171                 goto msix_setup_exit;
10172         }
10173
10174         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
10175         if (bp->irq_tbl) {
10176                 for (i = 0; i < total_vecs; i++)
10177                         bp->irq_tbl[i].vector = msix_ent[i].vector;
10178
10179                 bp->total_irqs = total_vecs;
10180                 /* Trim rings based upon num of vectors allocated */
10181                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
10182                                      total_vecs - ulp_msix, min == 1);
10183                 if (rc)
10184                         goto msix_setup_exit;
10185
10186                 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
10187                 bp->cp_nr_rings = (min == 1) ?
10188                                   max_t(int, tx_cp, bp->rx_nr_rings) :
10189                                   tx_cp + bp->rx_nr_rings;
10190
10191         } else {
10192                 rc = -ENOMEM;
10193                 goto msix_setup_exit;
10194         }
10195         bp->flags |= BNXT_FLAG_USING_MSIX;
10196         kfree(msix_ent);
10197         return 0;
10198
10199 msix_setup_exit:
10200         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
10201         kfree(bp->irq_tbl);
10202         bp->irq_tbl = NULL;
10203         pci_disable_msix(bp->pdev);
10204         kfree(msix_ent);
10205         return rc;
10206 }
10207
10208 static int bnxt_init_inta(struct bnxt *bp)
10209 {
10210         bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL);
10211         if (!bp->irq_tbl)
10212                 return -ENOMEM;
10213
10214         bp->total_irqs = 1;
10215         bp->rx_nr_rings = 1;
10216         bp->tx_nr_rings = 1;
10217         bp->cp_nr_rings = 1;
10218         bp->flags |= BNXT_FLAG_SHARED_RINGS;
10219         bp->irq_tbl[0].vector = bp->pdev->irq;
10220         return 0;
10221 }
10222
10223 static int bnxt_init_int_mode(struct bnxt *bp)
10224 {
10225         int rc = -ENODEV;
10226
10227         if (bp->flags & BNXT_FLAG_MSIX_CAP)
10228                 rc = bnxt_init_msix(bp);
10229
10230         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
10231                 /* fallback to INTA */
10232                 rc = bnxt_init_inta(bp);
10233         }
10234         return rc;
10235 }
10236
10237 static void bnxt_clear_int_mode(struct bnxt *bp)
10238 {
10239         if (bp->flags & BNXT_FLAG_USING_MSIX)
10240                 pci_disable_msix(bp->pdev);
10241
10242         kfree(bp->irq_tbl);
10243         bp->irq_tbl = NULL;
10244         bp->flags &= ~BNXT_FLAG_USING_MSIX;
10245 }
10246
10247 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
10248 {
10249         bool irq_cleared = false;
10250         int tcs = bp->num_tc;
10251         int rc;
10252
10253         if (!bnxt_need_reserve_rings(bp))
10254                 return 0;
10255
10256         if (irq_re_init && BNXT_NEW_RM(bp) &&
10257             bnxt_get_num_msix(bp) != bp->total_irqs) {
10258                 bnxt_ulp_irq_stop(bp);
10259                 bnxt_clear_int_mode(bp);
10260                 irq_cleared = true;
10261         }
10262         rc = __bnxt_reserve_rings(bp);
10263         if (irq_cleared) {
10264                 if (!rc)
10265                         rc = bnxt_init_int_mode(bp);
10266                 bnxt_ulp_irq_restart(bp, rc);
10267         }
10268         if (rc) {
10269                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
10270                 return rc;
10271         }
10272         if (tcs && (bp->tx_nr_rings_per_tc * tcs !=
10273                     bp->tx_nr_rings - bp->tx_nr_rings_xdp)) {
10274                 netdev_err(bp->dev, "tx ring reservation failure\n");
10275                 netdev_reset_tc(bp->dev);
10276                 bp->num_tc = 0;
10277                 if (bp->tx_nr_rings_xdp)
10278                         bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp;
10279                 else
10280                         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10281                 return -ENOMEM;
10282         }
10283         return 0;
10284 }
10285
10286 static void bnxt_free_irq(struct bnxt *bp)
10287 {
10288         struct bnxt_irq *irq;
10289         int i;
10290
10291 #ifdef CONFIG_RFS_ACCEL
10292         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
10293         bp->dev->rx_cpu_rmap = NULL;
10294 #endif
10295         if (!bp->irq_tbl || !bp->bnapi)
10296                 return;
10297
10298         for (i = 0; i < bp->cp_nr_rings; i++) {
10299                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10300
10301                 irq = &bp->irq_tbl[map_idx];
10302                 if (irq->requested) {
10303                         if (irq->have_cpumask) {
10304                                 irq_set_affinity_hint(irq->vector, NULL);
10305                                 free_cpumask_var(irq->cpu_mask);
10306                                 irq->have_cpumask = 0;
10307                         }
10308                         free_irq(irq->vector, bp->bnapi[i]);
10309                 }
10310
10311                 irq->requested = 0;
10312         }
10313 }
10314
10315 static int bnxt_request_irq(struct bnxt *bp)
10316 {
10317         int i, j, rc = 0;
10318         unsigned long flags = 0;
10319 #ifdef CONFIG_RFS_ACCEL
10320         struct cpu_rmap *rmap;
10321 #endif
10322
10323         rc = bnxt_setup_int_mode(bp);
10324         if (rc) {
10325                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
10326                            rc);
10327                 return rc;
10328         }
10329 #ifdef CONFIG_RFS_ACCEL
10330         rmap = bp->dev->rx_cpu_rmap;
10331 #endif
10332         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
10333                 flags = IRQF_SHARED;
10334
10335         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
10336                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10337                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
10338
10339 #ifdef CONFIG_RFS_ACCEL
10340                 if (rmap && bp->bnapi[i]->rx_ring) {
10341                         rc = irq_cpu_rmap_add(rmap, irq->vector);
10342                         if (rc)
10343                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
10344                                             j);
10345                         j++;
10346                 }
10347 #endif
10348                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
10349                                  bp->bnapi[i]);
10350                 if (rc)
10351                         break;
10352
10353                 netif_napi_set_irq(&bp->bnapi[i]->napi, irq->vector);
10354                 irq->requested = 1;
10355
10356                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
10357                         int numa_node = dev_to_node(&bp->pdev->dev);
10358
10359                         irq->have_cpumask = 1;
10360                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
10361                                         irq->cpu_mask);
10362                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
10363                         if (rc) {
10364                                 netdev_warn(bp->dev,
10365                                             "Set affinity failed, IRQ = %d\n",
10366                                             irq->vector);
10367                                 break;
10368                         }
10369                 }
10370         }
10371         return rc;
10372 }
10373
10374 static void bnxt_del_napi(struct bnxt *bp)
10375 {
10376         int i;
10377
10378         if (!bp->bnapi)
10379                 return;
10380
10381         for (i = 0; i < bp->rx_nr_rings; i++)
10382                 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_RX, NULL);
10383         for (i = 0; i < bp->tx_nr_rings - bp->tx_nr_rings_xdp; i++)
10384                 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_TX, NULL);
10385
10386         for (i = 0; i < bp->cp_nr_rings; i++) {
10387                 struct bnxt_napi *bnapi = bp->bnapi[i];
10388
10389                 __netif_napi_del(&bnapi->napi);
10390         }
10391         /* We called __netif_napi_del(), we need
10392          * to respect an RCU grace period before freeing napi structures.
10393          */
10394         synchronize_net();
10395 }
10396
10397 static void bnxt_init_napi(struct bnxt *bp)
10398 {
10399         int i;
10400         unsigned int cp_nr_rings = bp->cp_nr_rings;
10401         struct bnxt_napi *bnapi;
10402
10403         if (bp->flags & BNXT_FLAG_USING_MSIX) {
10404                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
10405
10406                 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10407                         poll_fn = bnxt_poll_p5;
10408                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
10409                         cp_nr_rings--;
10410                 for (i = 0; i < cp_nr_rings; i++) {
10411                         bnapi = bp->bnapi[i];
10412                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn);
10413                 }
10414                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10415                         bnapi = bp->bnapi[cp_nr_rings];
10416                         netif_napi_add(bp->dev, &bnapi->napi,
10417                                        bnxt_poll_nitroa0);
10418                 }
10419         } else {
10420                 bnapi = bp->bnapi[0];
10421                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll);
10422         }
10423 }
10424
10425 static void bnxt_disable_napi(struct bnxt *bp)
10426 {
10427         int i;
10428
10429         if (!bp->bnapi ||
10430             test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
10431                 return;
10432
10433         for (i = 0; i < bp->cp_nr_rings; i++) {
10434                 struct bnxt_napi *bnapi = bp->bnapi[i];
10435                 struct bnxt_cp_ring_info *cpr;
10436
10437                 cpr = &bnapi->cp_ring;
10438                 if (bnapi->tx_fault)
10439                         cpr->sw_stats.tx.tx_resets++;
10440                 if (bnapi->in_reset)
10441                         cpr->sw_stats.rx.rx_resets++;
10442                 napi_disable(&bnapi->napi);
10443                 if (bnapi->rx_ring)
10444                         cancel_work_sync(&cpr->dim.work);
10445         }
10446 }
10447
10448 static void bnxt_enable_napi(struct bnxt *bp)
10449 {
10450         int i;
10451
10452         clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
10453         for (i = 0; i < bp->cp_nr_rings; i++) {
10454                 struct bnxt_napi *bnapi = bp->bnapi[i];
10455                 struct bnxt_cp_ring_info *cpr;
10456
10457                 bnapi->tx_fault = 0;
10458
10459                 cpr = &bnapi->cp_ring;
10460                 bnapi->in_reset = false;
10461
10462                 if (bnapi->rx_ring) {
10463                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
10464                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
10465                 }
10466                 napi_enable(&bnapi->napi);
10467         }
10468 }
10469
10470 void bnxt_tx_disable(struct bnxt *bp)
10471 {
10472         int i;
10473         struct bnxt_tx_ring_info *txr;
10474
10475         if (bp->tx_ring) {
10476                 for (i = 0; i < bp->tx_nr_rings; i++) {
10477                         txr = &bp->tx_ring[i];
10478                         WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING);
10479                 }
10480         }
10481         /* Make sure napi polls see @dev_state change */
10482         synchronize_net();
10483         /* Drop carrier first to prevent TX timeout */
10484         netif_carrier_off(bp->dev);
10485         /* Stop all TX queues */
10486         netif_tx_disable(bp->dev);
10487 }
10488
10489 void bnxt_tx_enable(struct bnxt *bp)
10490 {
10491         int i;
10492         struct bnxt_tx_ring_info *txr;
10493
10494         for (i = 0; i < bp->tx_nr_rings; i++) {
10495                 txr = &bp->tx_ring[i];
10496                 WRITE_ONCE(txr->dev_state, 0);
10497         }
10498         /* Make sure napi polls see @dev_state change */
10499         synchronize_net();
10500         netif_tx_wake_all_queues(bp->dev);
10501         if (BNXT_LINK_IS_UP(bp))
10502                 netif_carrier_on(bp->dev);
10503 }
10504
10505 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
10506 {
10507         u8 active_fec = link_info->active_fec_sig_mode &
10508                         PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
10509
10510         switch (active_fec) {
10511         default:
10512         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
10513                 return "None";
10514         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
10515                 return "Clause 74 BaseR";
10516         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
10517                 return "Clause 91 RS(528,514)";
10518         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
10519                 return "Clause 91 RS544_1XN";
10520         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
10521                 return "Clause 91 RS(544,514)";
10522         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
10523                 return "Clause 91 RS272_1XN";
10524         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
10525                 return "Clause 91 RS(272,257)";
10526         }
10527 }
10528
10529 void bnxt_report_link(struct bnxt *bp)
10530 {
10531         if (BNXT_LINK_IS_UP(bp)) {
10532                 const char *signal = "";
10533                 const char *flow_ctrl;
10534                 const char *duplex;
10535                 u32 speed;
10536                 u16 fec;
10537
10538                 netif_carrier_on(bp->dev);
10539                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
10540                 if (speed == SPEED_UNKNOWN) {
10541                         netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
10542                         return;
10543                 }
10544                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
10545                         duplex = "full";
10546                 else
10547                         duplex = "half";
10548                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
10549                         flow_ctrl = "ON - receive & transmit";
10550                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
10551                         flow_ctrl = "ON - transmit";
10552                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
10553                         flow_ctrl = "ON - receive";
10554                 else
10555                         flow_ctrl = "none";
10556                 if (bp->link_info.phy_qcfg_resp.option_flags &
10557                     PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
10558                         u8 sig_mode = bp->link_info.active_fec_sig_mode &
10559                                       PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
10560                         switch (sig_mode) {
10561                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
10562                                 signal = "(NRZ) ";
10563                                 break;
10564                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
10565                                 signal = "(PAM4 56Gbps) ";
10566                                 break;
10567                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112:
10568                                 signal = "(PAM4 112Gbps) ";
10569                                 break;
10570                         default:
10571                                 break;
10572                         }
10573                 }
10574                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
10575                             speed, signal, duplex, flow_ctrl);
10576                 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
10577                         netdev_info(bp->dev, "EEE is %s\n",
10578                                     bp->eee.eee_active ? "active" :
10579                                                          "not active");
10580                 fec = bp->link_info.fec_cfg;
10581                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
10582                         netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
10583                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
10584                                     bnxt_report_fec(&bp->link_info));
10585         } else {
10586                 netif_carrier_off(bp->dev);
10587                 netdev_err(bp->dev, "NIC Link is Down\n");
10588         }
10589 }
10590
10591 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
10592 {
10593         if (!resp->supported_speeds_auto_mode &&
10594             !resp->supported_speeds_force_mode &&
10595             !resp->supported_pam4_speeds_auto_mode &&
10596             !resp->supported_pam4_speeds_force_mode &&
10597             !resp->supported_speeds2_auto_mode &&
10598             !resp->supported_speeds2_force_mode)
10599                 return true;
10600         return false;
10601 }
10602
10603 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
10604 {
10605         struct bnxt_link_info *link_info = &bp->link_info;
10606         struct hwrm_port_phy_qcaps_output *resp;
10607         struct hwrm_port_phy_qcaps_input *req;
10608         int rc = 0;
10609
10610         if (bp->hwrm_spec_code < 0x10201)
10611                 return 0;
10612
10613         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS);
10614         if (rc)
10615                 return rc;
10616
10617         resp = hwrm_req_hold(bp, req);
10618         rc = hwrm_req_send(bp, req);
10619         if (rc)
10620                 goto hwrm_phy_qcaps_exit;
10621
10622         bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8);
10623         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
10624                 struct ethtool_keee *eee = &bp->eee;
10625                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
10626
10627                 eee->supported_u32 = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
10628                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
10629                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
10630                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
10631                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
10632         }
10633
10634         if (bp->hwrm_spec_code >= 0x10a01) {
10635                 if (bnxt_phy_qcaps_no_speed(resp)) {
10636                         link_info->phy_state = BNXT_PHY_STATE_DISABLED;
10637                         netdev_warn(bp->dev, "Ethernet link disabled\n");
10638                 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
10639                         link_info->phy_state = BNXT_PHY_STATE_ENABLED;
10640                         netdev_info(bp->dev, "Ethernet link enabled\n");
10641                         /* Phy re-enabled, reprobe the speeds */
10642                         link_info->support_auto_speeds = 0;
10643                         link_info->support_pam4_auto_speeds = 0;
10644                         link_info->support_auto_speeds2 = 0;
10645                 }
10646         }
10647         if (resp->supported_speeds_auto_mode)
10648                 link_info->support_auto_speeds =
10649                         le16_to_cpu(resp->supported_speeds_auto_mode);
10650         if (resp->supported_pam4_speeds_auto_mode)
10651                 link_info->support_pam4_auto_speeds =
10652                         le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
10653         if (resp->supported_speeds2_auto_mode)
10654                 link_info->support_auto_speeds2 =
10655                         le16_to_cpu(resp->supported_speeds2_auto_mode);
10656
10657         bp->port_count = resp->port_cnt;
10658
10659 hwrm_phy_qcaps_exit:
10660         hwrm_req_drop(bp, req);
10661         return rc;
10662 }
10663
10664 static bool bnxt_support_dropped(u16 advertising, u16 supported)
10665 {
10666         u16 diff = advertising ^ supported;
10667
10668         return ((supported | diff) != supported);
10669 }
10670
10671 static bool bnxt_support_speed_dropped(struct bnxt_link_info *link_info)
10672 {
10673         struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
10674
10675         /* Check if any advertised speeds are no longer supported. The caller
10676          * holds the link_lock mutex, so we can modify link_info settings.
10677          */
10678         if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
10679                 if (bnxt_support_dropped(link_info->advertising,
10680                                          link_info->support_auto_speeds2)) {
10681                         link_info->advertising = link_info->support_auto_speeds2;
10682                         return true;
10683                 }
10684                 return false;
10685         }
10686         if (bnxt_support_dropped(link_info->advertising,
10687                                  link_info->support_auto_speeds)) {
10688                 link_info->advertising = link_info->support_auto_speeds;
10689                 return true;
10690         }
10691         if (bnxt_support_dropped(link_info->advertising_pam4,
10692                                  link_info->support_pam4_auto_speeds)) {
10693                 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
10694                 return true;
10695         }
10696         return false;
10697 }
10698
10699 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
10700 {
10701         struct bnxt_link_info *link_info = &bp->link_info;
10702         struct hwrm_port_phy_qcfg_output *resp;
10703         struct hwrm_port_phy_qcfg_input *req;
10704         u8 link_state = link_info->link_state;
10705         bool support_changed;
10706         int rc;
10707
10708         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG);
10709         if (rc)
10710                 return rc;
10711
10712         resp = hwrm_req_hold(bp, req);
10713         rc = hwrm_req_send(bp, req);
10714         if (rc) {
10715                 hwrm_req_drop(bp, req);
10716                 if (BNXT_VF(bp) && rc == -ENODEV) {
10717                         netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n");
10718                         rc = 0;
10719                 }
10720                 return rc;
10721         }
10722
10723         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
10724         link_info->phy_link_status = resp->link;
10725         link_info->duplex = resp->duplex_cfg;
10726         if (bp->hwrm_spec_code >= 0x10800)
10727                 link_info->duplex = resp->duplex_state;
10728         link_info->pause = resp->pause;
10729         link_info->auto_mode = resp->auto_mode;
10730         link_info->auto_pause_setting = resp->auto_pause;
10731         link_info->lp_pause = resp->link_partner_adv_pause;
10732         link_info->force_pause_setting = resp->force_pause;
10733         link_info->duplex_setting = resp->duplex_cfg;
10734         if (link_info->phy_link_status == BNXT_LINK_LINK) {
10735                 link_info->link_speed = le16_to_cpu(resp->link_speed);
10736                 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2)
10737                         link_info->active_lanes = resp->active_lanes;
10738         } else {
10739                 link_info->link_speed = 0;
10740                 link_info->active_lanes = 0;
10741         }
10742         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
10743         link_info->force_pam4_link_speed =
10744                 le16_to_cpu(resp->force_pam4_link_speed);
10745         link_info->force_link_speed2 = le16_to_cpu(resp->force_link_speeds2);
10746         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
10747         link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
10748         link_info->support_speeds2 = le16_to_cpu(resp->support_speeds2);
10749         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
10750         link_info->auto_pam4_link_speeds =
10751                 le16_to_cpu(resp->auto_pam4_link_speed_mask);
10752         link_info->auto_link_speeds2 = le16_to_cpu(resp->auto_link_speeds2);
10753         link_info->lp_auto_link_speeds =
10754                 le16_to_cpu(resp->link_partner_adv_speeds);
10755         link_info->lp_auto_pam4_link_speeds =
10756                 resp->link_partner_pam4_adv_speeds;
10757         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
10758         link_info->phy_ver[0] = resp->phy_maj;
10759         link_info->phy_ver[1] = resp->phy_min;
10760         link_info->phy_ver[2] = resp->phy_bld;
10761         link_info->media_type = resp->media_type;
10762         link_info->phy_type = resp->phy_type;
10763         link_info->transceiver = resp->xcvr_pkg_type;
10764         link_info->phy_addr = resp->eee_config_phy_addr &
10765                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
10766         link_info->module_status = resp->module_status;
10767
10768         if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
10769                 struct ethtool_keee *eee = &bp->eee;
10770                 u16 fw_speeds;
10771
10772                 eee->eee_active = 0;
10773                 if (resp->eee_config_phy_addr &
10774                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
10775                         eee->eee_active = 1;
10776                         fw_speeds = le16_to_cpu(
10777                                 resp->link_partner_adv_eee_link_speed_mask);
10778                         eee->lp_advertised_u32 =
10779                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
10780                 }
10781
10782                 /* Pull initial EEE config */
10783                 if (!chng_link_state) {
10784                         if (resp->eee_config_phy_addr &
10785                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
10786                                 eee->eee_enabled = 1;
10787
10788                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
10789                         eee->advertised_u32 =
10790                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
10791
10792                         if (resp->eee_config_phy_addr &
10793                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
10794                                 __le32 tmr;
10795
10796                                 eee->tx_lpi_enabled = 1;
10797                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
10798                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
10799                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
10800                         }
10801                 }
10802         }
10803
10804         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
10805         if (bp->hwrm_spec_code >= 0x10504) {
10806                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
10807                 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
10808         }
10809         /* TODO: need to add more logic to report VF link */
10810         if (chng_link_state) {
10811                 if (link_info->phy_link_status == BNXT_LINK_LINK)
10812                         link_info->link_state = BNXT_LINK_STATE_UP;
10813                 else
10814                         link_info->link_state = BNXT_LINK_STATE_DOWN;
10815                 if (link_state != link_info->link_state)
10816                         bnxt_report_link(bp);
10817         } else {
10818                 /* always link down if not require to update link state */
10819                 link_info->link_state = BNXT_LINK_STATE_DOWN;
10820         }
10821         hwrm_req_drop(bp, req);
10822
10823         if (!BNXT_PHY_CFG_ABLE(bp))
10824                 return 0;
10825
10826         support_changed = bnxt_support_speed_dropped(link_info);
10827         if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
10828                 bnxt_hwrm_set_link_setting(bp, true, false);
10829         return 0;
10830 }
10831
10832 static void bnxt_get_port_module_status(struct bnxt *bp)
10833 {
10834         struct bnxt_link_info *link_info = &bp->link_info;
10835         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
10836         u8 module_status;
10837
10838         if (bnxt_update_link(bp, true))
10839                 return;
10840
10841         module_status = link_info->module_status;
10842         switch (module_status) {
10843         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
10844         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
10845         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
10846                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
10847                             bp->pf.port_id);
10848                 if (bp->hwrm_spec_code >= 0x10201) {
10849                         netdev_warn(bp->dev, "Module part number %s\n",
10850                                     resp->phy_vendor_partnumber);
10851                 }
10852                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
10853                         netdev_warn(bp->dev, "TX is disabled\n");
10854                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
10855                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
10856         }
10857 }
10858
10859 static void
10860 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
10861 {
10862         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
10863                 if (bp->hwrm_spec_code >= 0x10201)
10864                         req->auto_pause =
10865                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
10866                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
10867                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
10868                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
10869                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
10870                 req->enables |=
10871                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
10872         } else {
10873                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
10874                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
10875                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
10876                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
10877                 req->enables |=
10878                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
10879                 if (bp->hwrm_spec_code >= 0x10201) {
10880                         req->auto_pause = req->force_pause;
10881                         req->enables |= cpu_to_le32(
10882                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
10883                 }
10884         }
10885 }
10886
10887 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
10888 {
10889         if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
10890                 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
10891                 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
10892                         req->enables |=
10893                                 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK);
10894                         req->auto_link_speeds2_mask = cpu_to_le16(bp->link_info.advertising);
10895                 } else if (bp->link_info.advertising) {
10896                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
10897                         req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
10898                 }
10899                 if (bp->link_info.advertising_pam4) {
10900                         req->enables |=
10901                                 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
10902                         req->auto_link_pam4_speed_mask =
10903                                 cpu_to_le16(bp->link_info.advertising_pam4);
10904                 }
10905                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
10906                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
10907         } else {
10908                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
10909                 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
10910                         req->force_link_speeds2 = cpu_to_le16(bp->link_info.req_link_speed);
10911                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2);
10912                         netif_info(bp, link, bp->dev, "Forcing FW speed2: %d\n",
10913                                    (u32)bp->link_info.req_link_speed);
10914                 } else if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
10915                         req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
10916                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
10917                 } else {
10918                         req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
10919                 }
10920         }
10921
10922         /* tell chimp that the setting takes effect immediately */
10923         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
10924 }
10925
10926 int bnxt_hwrm_set_pause(struct bnxt *bp)
10927 {
10928         struct hwrm_port_phy_cfg_input *req;
10929         int rc;
10930
10931         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
10932         if (rc)
10933                 return rc;
10934
10935         bnxt_hwrm_set_pause_common(bp, req);
10936
10937         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
10938             bp->link_info.force_link_chng)
10939                 bnxt_hwrm_set_link_common(bp, req);
10940
10941         rc = hwrm_req_send(bp, req);
10942         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
10943                 /* since changing of pause setting doesn't trigger any link
10944                  * change event, the driver needs to update the current pause
10945                  * result upon successfully return of the phy_cfg command
10946                  */
10947                 bp->link_info.pause =
10948                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
10949                 bp->link_info.auto_pause_setting = 0;
10950                 if (!bp->link_info.force_link_chng)
10951                         bnxt_report_link(bp);
10952         }
10953         bp->link_info.force_link_chng = false;
10954         return rc;
10955 }
10956
10957 static void bnxt_hwrm_set_eee(struct bnxt *bp,
10958                               struct hwrm_port_phy_cfg_input *req)
10959 {
10960         struct ethtool_keee *eee = &bp->eee;
10961
10962         if (eee->eee_enabled) {
10963                 u16 eee_speeds;
10964                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
10965
10966                 if (eee->tx_lpi_enabled)
10967                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
10968                 else
10969                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
10970
10971                 req->flags |= cpu_to_le32(flags);
10972                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised_u32);
10973                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
10974                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
10975         } else {
10976                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
10977         }
10978 }
10979
10980 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
10981 {
10982         struct hwrm_port_phy_cfg_input *req;
10983         int rc;
10984
10985         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
10986         if (rc)
10987                 return rc;
10988
10989         if (set_pause)
10990                 bnxt_hwrm_set_pause_common(bp, req);
10991
10992         bnxt_hwrm_set_link_common(bp, req);
10993
10994         if (set_eee)
10995                 bnxt_hwrm_set_eee(bp, req);
10996         return hwrm_req_send(bp, req);
10997 }
10998
10999 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
11000 {
11001         struct hwrm_port_phy_cfg_input *req;
11002         int rc;
11003
11004         if (!BNXT_SINGLE_PF(bp))
11005                 return 0;
11006
11007         if (pci_num_vf(bp->pdev) &&
11008             !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
11009                 return 0;
11010
11011         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11012         if (rc)
11013                 return rc;
11014
11015         req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
11016         rc = hwrm_req_send(bp, req);
11017         if (!rc) {
11018                 mutex_lock(&bp->link_lock);
11019                 /* Device is not obliged link down in certain scenarios, even
11020                  * when forced. Setting the state unknown is consistent with
11021                  * driver startup and will force link state to be reported
11022                  * during subsequent open based on PORT_PHY_QCFG.
11023                  */
11024                 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN;
11025                 mutex_unlock(&bp->link_lock);
11026         }
11027         return rc;
11028 }
11029
11030 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
11031 {
11032 #ifdef CONFIG_TEE_BNXT_FW
11033         int rc = tee_bnxt_fw_load();
11034
11035         if (rc)
11036                 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
11037
11038         return rc;
11039 #else
11040         netdev_err(bp->dev, "OP-TEE not supported\n");
11041         return -ENODEV;
11042 #endif
11043 }
11044
11045 static int bnxt_try_recover_fw(struct bnxt *bp)
11046 {
11047         if (bp->fw_health && bp->fw_health->status_reliable) {
11048                 int retry = 0, rc;
11049                 u32 sts;
11050
11051                 do {
11052                         sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11053                         rc = bnxt_hwrm_poll(bp);
11054                         if (!BNXT_FW_IS_BOOTING(sts) &&
11055                             !BNXT_FW_IS_RECOVERING(sts))
11056                                 break;
11057                         retry++;
11058                 } while (rc == -EBUSY && retry < BNXT_FW_RETRY);
11059
11060                 if (!BNXT_FW_IS_HEALTHY(sts)) {
11061                         netdev_err(bp->dev,
11062                                    "Firmware not responding, status: 0x%x\n",
11063                                    sts);
11064                         rc = -ENODEV;
11065                 }
11066                 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
11067                         netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
11068                         return bnxt_fw_reset_via_optee(bp);
11069                 }
11070                 return rc;
11071         }
11072
11073         return -ENODEV;
11074 }
11075
11076 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset)
11077 {
11078         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11079
11080         if (!BNXT_NEW_RM(bp))
11081                 return; /* no resource reservations required */
11082
11083         hw_resc->resv_cp_rings = 0;
11084         hw_resc->resv_stat_ctxs = 0;
11085         hw_resc->resv_irqs = 0;
11086         hw_resc->resv_tx_rings = 0;
11087         hw_resc->resv_rx_rings = 0;
11088         hw_resc->resv_hw_ring_grps = 0;
11089         hw_resc->resv_vnics = 0;
11090         if (!fw_reset) {
11091                 bp->tx_nr_rings = 0;
11092                 bp->rx_nr_rings = 0;
11093         }
11094 }
11095
11096 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset)
11097 {
11098         int rc;
11099
11100         if (!BNXT_NEW_RM(bp))
11101                 return 0; /* no resource reservations required */
11102
11103         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
11104         if (rc)
11105                 netdev_err(bp->dev, "resc_qcaps failed\n");
11106
11107         bnxt_clear_reservations(bp, fw_reset);
11108
11109         return rc;
11110 }
11111
11112 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
11113 {
11114         struct hwrm_func_drv_if_change_output *resp;
11115         struct hwrm_func_drv_if_change_input *req;
11116         bool fw_reset = !bp->irq_tbl;
11117         bool resc_reinit = false;
11118         int rc, retry = 0;
11119         u32 flags = 0;
11120
11121         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
11122                 return 0;
11123
11124         rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE);
11125         if (rc)
11126                 return rc;
11127
11128         if (up)
11129                 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
11130         resp = hwrm_req_hold(bp, req);
11131
11132         hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
11133         while (retry < BNXT_FW_IF_RETRY) {
11134                 rc = hwrm_req_send(bp, req);
11135                 if (rc != -EAGAIN)
11136                         break;
11137
11138                 msleep(50);
11139                 retry++;
11140         }
11141
11142         if (rc == -EAGAIN) {
11143                 hwrm_req_drop(bp, req);
11144                 return rc;
11145         } else if (!rc) {
11146                 flags = le32_to_cpu(resp->flags);
11147         } else if (up) {
11148                 rc = bnxt_try_recover_fw(bp);
11149                 fw_reset = true;
11150         }
11151         hwrm_req_drop(bp, req);
11152         if (rc)
11153                 return rc;
11154
11155         if (!up) {
11156                 bnxt_inv_fw_health_reg(bp);
11157                 return 0;
11158         }
11159
11160         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
11161                 resc_reinit = true;
11162         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE ||
11163             test_bit(BNXT_STATE_FW_RESET_DET, &bp->state))
11164                 fw_reset = true;
11165         else
11166                 bnxt_remap_fw_health_regs(bp);
11167
11168         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
11169                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
11170                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11171                 return -ENODEV;
11172         }
11173         if (resc_reinit || fw_reset) {
11174                 if (fw_reset) {
11175                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11176                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11177                                 bnxt_ulp_stop(bp);
11178                         bnxt_free_ctx_mem(bp);
11179                         bnxt_dcb_free(bp);
11180                         rc = bnxt_fw_init_one(bp);
11181                         if (rc) {
11182                                 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11183                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11184                                 return rc;
11185                         }
11186                         bnxt_clear_int_mode(bp);
11187                         rc = bnxt_init_int_mode(bp);
11188                         if (rc) {
11189                                 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11190                                 netdev_err(bp->dev, "init int mode failed\n");
11191                                 return rc;
11192                         }
11193                 }
11194                 rc = bnxt_cancel_reservations(bp, fw_reset);
11195         }
11196         return rc;
11197 }
11198
11199 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
11200 {
11201         struct hwrm_port_led_qcaps_output *resp;
11202         struct hwrm_port_led_qcaps_input *req;
11203         struct bnxt_pf_info *pf = &bp->pf;
11204         int rc;
11205
11206         bp->num_leds = 0;
11207         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
11208                 return 0;
11209
11210         rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS);
11211         if (rc)
11212                 return rc;
11213
11214         req->port_id = cpu_to_le16(pf->port_id);
11215         resp = hwrm_req_hold(bp, req);
11216         rc = hwrm_req_send(bp, req);
11217         if (rc) {
11218                 hwrm_req_drop(bp, req);
11219                 return rc;
11220         }
11221         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
11222                 int i;
11223
11224                 bp->num_leds = resp->num_leds;
11225                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
11226                                                  bp->num_leds);
11227                 for (i = 0; i < bp->num_leds; i++) {
11228                         struct bnxt_led_info *led = &bp->leds[i];
11229                         __le16 caps = led->led_state_caps;
11230
11231                         if (!led->led_group_id ||
11232                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
11233                                 bp->num_leds = 0;
11234                                 break;
11235                         }
11236                 }
11237         }
11238         hwrm_req_drop(bp, req);
11239         return 0;
11240 }
11241
11242 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
11243 {
11244         struct hwrm_wol_filter_alloc_output *resp;
11245         struct hwrm_wol_filter_alloc_input *req;
11246         int rc;
11247
11248         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC);
11249         if (rc)
11250                 return rc;
11251
11252         req->port_id = cpu_to_le16(bp->pf.port_id);
11253         req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
11254         req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
11255         memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN);
11256
11257         resp = hwrm_req_hold(bp, req);
11258         rc = hwrm_req_send(bp, req);
11259         if (!rc)
11260                 bp->wol_filter_id = resp->wol_filter_id;
11261         hwrm_req_drop(bp, req);
11262         return rc;
11263 }
11264
11265 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
11266 {
11267         struct hwrm_wol_filter_free_input *req;
11268         int rc;
11269
11270         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE);
11271         if (rc)
11272                 return rc;
11273
11274         req->port_id = cpu_to_le16(bp->pf.port_id);
11275         req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
11276         req->wol_filter_id = bp->wol_filter_id;
11277
11278         return hwrm_req_send(bp, req);
11279 }
11280
11281 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
11282 {
11283         struct hwrm_wol_filter_qcfg_output *resp;
11284         struct hwrm_wol_filter_qcfg_input *req;
11285         u16 next_handle = 0;
11286         int rc;
11287
11288         rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG);
11289         if (rc)
11290                 return rc;
11291
11292         req->port_id = cpu_to_le16(bp->pf.port_id);
11293         req->handle = cpu_to_le16(handle);
11294         resp = hwrm_req_hold(bp, req);
11295         rc = hwrm_req_send(bp, req);
11296         if (!rc) {
11297                 next_handle = le16_to_cpu(resp->next_handle);
11298                 if (next_handle != 0) {
11299                         if (resp->wol_type ==
11300                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
11301                                 bp->wol = 1;
11302                                 bp->wol_filter_id = resp->wol_filter_id;
11303                         }
11304                 }
11305         }
11306         hwrm_req_drop(bp, req);
11307         return next_handle;
11308 }
11309
11310 static void bnxt_get_wol_settings(struct bnxt *bp)
11311 {
11312         u16 handle = 0;
11313
11314         bp->wol = 0;
11315         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
11316                 return;
11317
11318         do {
11319                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
11320         } while (handle && handle != 0xffff);
11321 }
11322
11323 static bool bnxt_eee_config_ok(struct bnxt *bp)
11324 {
11325         struct ethtool_keee *eee = &bp->eee;
11326         struct bnxt_link_info *link_info = &bp->link_info;
11327
11328         if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
11329                 return true;
11330
11331         if (eee->eee_enabled) {
11332                 u32 advertising =
11333                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
11334
11335                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
11336                         eee->eee_enabled = 0;
11337                         return false;
11338                 }
11339                 if (eee->advertised_u32 & ~advertising) {
11340                         eee->advertised_u32 = advertising & eee->supported_u32;
11341                         return false;
11342                 }
11343         }
11344         return true;
11345 }
11346
11347 static int bnxt_update_phy_setting(struct bnxt *bp)
11348 {
11349         int rc;
11350         bool update_link = false;
11351         bool update_pause = false;
11352         bool update_eee = false;
11353         struct bnxt_link_info *link_info = &bp->link_info;
11354
11355         rc = bnxt_update_link(bp, true);
11356         if (rc) {
11357                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
11358                            rc);
11359                 return rc;
11360         }
11361         if (!BNXT_SINGLE_PF(bp))
11362                 return 0;
11363
11364         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
11365             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
11366             link_info->req_flow_ctrl)
11367                 update_pause = true;
11368         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
11369             link_info->force_pause_setting != link_info->req_flow_ctrl)
11370                 update_pause = true;
11371         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
11372                 if (BNXT_AUTO_MODE(link_info->auto_mode))
11373                         update_link = true;
11374                 if (bnxt_force_speed_updated(link_info))
11375                         update_link = true;
11376                 if (link_info->req_duplex != link_info->duplex_setting)
11377                         update_link = true;
11378         } else {
11379                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
11380                         update_link = true;
11381                 if (bnxt_auto_speed_updated(link_info))
11382                         update_link = true;
11383         }
11384
11385         /* The last close may have shutdown the link, so need to call
11386          * PHY_CFG to bring it back up.
11387          */
11388         if (!BNXT_LINK_IS_UP(bp))
11389                 update_link = true;
11390
11391         if (!bnxt_eee_config_ok(bp))
11392                 update_eee = true;
11393
11394         if (update_link)
11395                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
11396         else if (update_pause)
11397                 rc = bnxt_hwrm_set_pause(bp);
11398         if (rc) {
11399                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
11400                            rc);
11401                 return rc;
11402         }
11403
11404         return rc;
11405 }
11406
11407 /* Common routine to pre-map certain register block to different GRC window.
11408  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
11409  * in PF and 3 windows in VF that can be customized to map in different
11410  * register blocks.
11411  */
11412 static void bnxt_preset_reg_win(struct bnxt *bp)
11413 {
11414         if (BNXT_PF(bp)) {
11415                 /* CAG registers map to GRC window #4 */
11416                 writel(BNXT_CAG_REG_BASE,
11417                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
11418         }
11419 }
11420
11421 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
11422
11423 static int bnxt_reinit_after_abort(struct bnxt *bp)
11424 {
11425         int rc;
11426
11427         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11428                 return -EBUSY;
11429
11430         if (bp->dev->reg_state == NETREG_UNREGISTERED)
11431                 return -ENODEV;
11432
11433         rc = bnxt_fw_init_one(bp);
11434         if (!rc) {
11435                 bnxt_clear_int_mode(bp);
11436                 rc = bnxt_init_int_mode(bp);
11437                 if (!rc) {
11438                         clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11439                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11440                 }
11441         }
11442         return rc;
11443 }
11444
11445 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
11446 {
11447         int rc = 0;
11448
11449         bnxt_preset_reg_win(bp);
11450         netif_carrier_off(bp->dev);
11451         if (irq_re_init) {
11452                 /* Reserve rings now if none were reserved at driver probe. */
11453                 rc = bnxt_init_dflt_ring_mode(bp);
11454                 if (rc) {
11455                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
11456                         return rc;
11457                 }
11458         }
11459         rc = bnxt_reserve_rings(bp, irq_re_init);
11460         if (rc)
11461                 return rc;
11462         if ((bp->flags & BNXT_FLAG_RFS) &&
11463             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
11464                 /* disable RFS if falling back to INTA */
11465                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
11466                 bp->flags &= ~BNXT_FLAG_RFS;
11467         }
11468
11469         rc = bnxt_alloc_mem(bp, irq_re_init);
11470         if (rc) {
11471                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
11472                 goto open_err_free_mem;
11473         }
11474
11475         if (irq_re_init) {
11476                 bnxt_init_napi(bp);
11477                 rc = bnxt_request_irq(bp);
11478                 if (rc) {
11479                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
11480                         goto open_err_irq;
11481                 }
11482         }
11483
11484         rc = bnxt_init_nic(bp, irq_re_init);
11485         if (rc) {
11486                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
11487                 goto open_err_irq;
11488         }
11489
11490         bnxt_enable_napi(bp);
11491         bnxt_debug_dev_init(bp);
11492
11493         if (link_re_init) {
11494                 mutex_lock(&bp->link_lock);
11495                 rc = bnxt_update_phy_setting(bp);
11496                 mutex_unlock(&bp->link_lock);
11497                 if (rc) {
11498                         netdev_warn(bp->dev, "failed to update phy settings\n");
11499                         if (BNXT_SINGLE_PF(bp)) {
11500                                 bp->link_info.phy_retry = true;
11501                                 bp->link_info.phy_retry_expires =
11502                                         jiffies + 5 * HZ;
11503                         }
11504                 }
11505         }
11506
11507         if (irq_re_init)
11508                 udp_tunnel_nic_reset_ntf(bp->dev);
11509
11510         if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
11511                 if (!static_key_enabled(&bnxt_xdp_locking_key))
11512                         static_branch_enable(&bnxt_xdp_locking_key);
11513         } else if (static_key_enabled(&bnxt_xdp_locking_key)) {
11514                 static_branch_disable(&bnxt_xdp_locking_key);
11515         }
11516         set_bit(BNXT_STATE_OPEN, &bp->state);
11517         bnxt_enable_int(bp);
11518         /* Enable TX queues */
11519         bnxt_tx_enable(bp);
11520         mod_timer(&bp->timer, jiffies + bp->current_interval);
11521         /* Poll link status and check for SFP+ module status */
11522         mutex_lock(&bp->link_lock);
11523         bnxt_get_port_module_status(bp);
11524         mutex_unlock(&bp->link_lock);
11525
11526         /* VF-reps may need to be re-opened after the PF is re-opened */
11527         if (BNXT_PF(bp))
11528                 bnxt_vf_reps_open(bp);
11529         bnxt_ptp_init_rtc(bp, true);
11530         bnxt_ptp_cfg_tstamp_filters(bp);
11531         return 0;
11532
11533 open_err_irq:
11534         bnxt_del_napi(bp);
11535
11536 open_err_free_mem:
11537         bnxt_free_skbs(bp);
11538         bnxt_free_irq(bp);
11539         bnxt_free_mem(bp, true);
11540         return rc;
11541 }
11542
11543 /* rtnl_lock held */
11544 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
11545 {
11546         int rc = 0;
11547
11548         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
11549                 rc = -EIO;
11550         if (!rc)
11551                 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
11552         if (rc) {
11553                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
11554                 dev_close(bp->dev);
11555         }
11556         return rc;
11557 }
11558
11559 /* rtnl_lock held, open the NIC half way by allocating all resources, but
11560  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
11561  * self tests.
11562  */
11563 int bnxt_half_open_nic(struct bnxt *bp)
11564 {
11565         int rc = 0;
11566
11567         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
11568                 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n");
11569                 rc = -ENODEV;
11570                 goto half_open_err;
11571         }
11572
11573         rc = bnxt_alloc_mem(bp, true);
11574         if (rc) {
11575                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
11576                 goto half_open_err;
11577         }
11578         bnxt_init_napi(bp);
11579         set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
11580         rc = bnxt_init_nic(bp, true);
11581         if (rc) {
11582                 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
11583                 bnxt_del_napi(bp);
11584                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
11585                 goto half_open_err;
11586         }
11587         return 0;
11588
11589 half_open_err:
11590         bnxt_free_skbs(bp);
11591         bnxt_free_mem(bp, true);
11592         dev_close(bp->dev);
11593         return rc;
11594 }
11595
11596 /* rtnl_lock held, this call can only be made after a previous successful
11597  * call to bnxt_half_open_nic().
11598  */
11599 void bnxt_half_close_nic(struct bnxt *bp)
11600 {
11601         bnxt_hwrm_resource_free(bp, false, true);
11602         bnxt_del_napi(bp);
11603         bnxt_free_skbs(bp);
11604         bnxt_free_mem(bp, true);
11605         clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
11606 }
11607
11608 void bnxt_reenable_sriov(struct bnxt *bp)
11609 {
11610         if (BNXT_PF(bp)) {
11611                 struct bnxt_pf_info *pf = &bp->pf;
11612                 int n = pf->active_vfs;
11613
11614                 if (n)
11615                         bnxt_cfg_hw_sriov(bp, &n, true);
11616         }
11617 }
11618
11619 static int bnxt_open(struct net_device *dev)
11620 {
11621         struct bnxt *bp = netdev_priv(dev);
11622         int rc;
11623
11624         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
11625                 rc = bnxt_reinit_after_abort(bp);
11626                 if (rc) {
11627                         if (rc == -EBUSY)
11628                                 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
11629                         else
11630                                 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
11631                         return -ENODEV;
11632                 }
11633         }
11634
11635         rc = bnxt_hwrm_if_change(bp, true);
11636         if (rc)
11637                 return rc;
11638
11639         rc = __bnxt_open_nic(bp, true, true);
11640         if (rc) {
11641                 bnxt_hwrm_if_change(bp, false);
11642         } else {
11643                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
11644                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11645                                 bnxt_ulp_start(bp, 0);
11646                                 bnxt_reenable_sriov(bp);
11647                         }
11648                 }
11649         }
11650
11651         return rc;
11652 }
11653
11654 static bool bnxt_drv_busy(struct bnxt *bp)
11655 {
11656         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
11657                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
11658 }
11659
11660 static void bnxt_get_ring_stats(struct bnxt *bp,
11661                                 struct rtnl_link_stats64 *stats);
11662
11663 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
11664                              bool link_re_init)
11665 {
11666         /* Close the VF-reps before closing PF */
11667         if (BNXT_PF(bp))
11668                 bnxt_vf_reps_close(bp);
11669
11670         /* Change device state to avoid TX queue wake up's */
11671         bnxt_tx_disable(bp);
11672
11673         clear_bit(BNXT_STATE_OPEN, &bp->state);
11674         smp_mb__after_atomic();
11675         while (bnxt_drv_busy(bp))
11676                 msleep(20);
11677
11678         /* Flush rings and disable interrupts */
11679         bnxt_shutdown_nic(bp, irq_re_init);
11680
11681         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
11682
11683         bnxt_debug_dev_exit(bp);
11684         bnxt_disable_napi(bp);
11685         del_timer_sync(&bp->timer);
11686         bnxt_free_skbs(bp);
11687
11688         /* Save ring stats before shutdown */
11689         if (bp->bnapi && irq_re_init) {
11690                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
11691                 bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev);
11692         }
11693         if (irq_re_init) {
11694                 bnxt_free_irq(bp);
11695                 bnxt_del_napi(bp);
11696         }
11697         bnxt_free_mem(bp, irq_re_init);
11698 }
11699
11700 void bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
11701 {
11702         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11703                 /* If we get here, it means firmware reset is in progress
11704                  * while we are trying to close.  We can safely proceed with
11705                  * the close because we are holding rtnl_lock().  Some firmware
11706                  * messages may fail as we proceed to close.  We set the
11707                  * ABORT_ERR flag here so that the FW reset thread will later
11708                  * abort when it gets the rtnl_lock() and sees the flag.
11709                  */
11710                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
11711                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11712         }
11713
11714 #ifdef CONFIG_BNXT_SRIOV
11715         if (bp->sriov_cfg) {
11716                 int rc;
11717
11718                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
11719                                                       !bp->sriov_cfg,
11720                                                       BNXT_SRIOV_CFG_WAIT_TMO);
11721                 if (!rc)
11722                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete, proceeding to close!\n");
11723                 else if (rc < 0)
11724                         netdev_warn(bp->dev, "SRIOV config operation interrupted, proceeding to close!\n");
11725         }
11726 #endif
11727         __bnxt_close_nic(bp, irq_re_init, link_re_init);
11728 }
11729
11730 static int bnxt_close(struct net_device *dev)
11731 {
11732         struct bnxt *bp = netdev_priv(dev);
11733
11734         bnxt_close_nic(bp, true, true);
11735         bnxt_hwrm_shutdown_link(bp);
11736         bnxt_hwrm_if_change(bp, false);
11737         return 0;
11738 }
11739
11740 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
11741                                    u16 *val)
11742 {
11743         struct hwrm_port_phy_mdio_read_output *resp;
11744         struct hwrm_port_phy_mdio_read_input *req;
11745         int rc;
11746
11747         if (bp->hwrm_spec_code < 0x10a00)
11748                 return -EOPNOTSUPP;
11749
11750         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ);
11751         if (rc)
11752                 return rc;
11753
11754         req->port_id = cpu_to_le16(bp->pf.port_id);
11755         req->phy_addr = phy_addr;
11756         req->reg_addr = cpu_to_le16(reg & 0x1f);
11757         if (mdio_phy_id_is_c45(phy_addr)) {
11758                 req->cl45_mdio = 1;
11759                 req->phy_addr = mdio_phy_id_prtad(phy_addr);
11760                 req->dev_addr = mdio_phy_id_devad(phy_addr);
11761                 req->reg_addr = cpu_to_le16(reg);
11762         }
11763
11764         resp = hwrm_req_hold(bp, req);
11765         rc = hwrm_req_send(bp, req);
11766         if (!rc)
11767                 *val = le16_to_cpu(resp->reg_data);
11768         hwrm_req_drop(bp, req);
11769         return rc;
11770 }
11771
11772 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
11773                                     u16 val)
11774 {
11775         struct hwrm_port_phy_mdio_write_input *req;
11776         int rc;
11777
11778         if (bp->hwrm_spec_code < 0x10a00)
11779                 return -EOPNOTSUPP;
11780
11781         rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE);
11782         if (rc)
11783                 return rc;
11784
11785         req->port_id = cpu_to_le16(bp->pf.port_id);
11786         req->phy_addr = phy_addr;
11787         req->reg_addr = cpu_to_le16(reg & 0x1f);
11788         if (mdio_phy_id_is_c45(phy_addr)) {
11789                 req->cl45_mdio = 1;
11790                 req->phy_addr = mdio_phy_id_prtad(phy_addr);
11791                 req->dev_addr = mdio_phy_id_devad(phy_addr);
11792                 req->reg_addr = cpu_to_le16(reg);
11793         }
11794         req->reg_data = cpu_to_le16(val);
11795
11796         return hwrm_req_send(bp, req);
11797 }
11798
11799 /* rtnl_lock held */
11800 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
11801 {
11802         struct mii_ioctl_data *mdio = if_mii(ifr);
11803         struct bnxt *bp = netdev_priv(dev);
11804         int rc;
11805
11806         switch (cmd) {
11807         case SIOCGMIIPHY:
11808                 mdio->phy_id = bp->link_info.phy_addr;
11809
11810                 fallthrough;
11811         case SIOCGMIIREG: {
11812                 u16 mii_regval = 0;
11813
11814                 if (!netif_running(dev))
11815                         return -EAGAIN;
11816
11817                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
11818                                              &mii_regval);
11819                 mdio->val_out = mii_regval;
11820                 return rc;
11821         }
11822
11823         case SIOCSMIIREG:
11824                 if (!netif_running(dev))
11825                         return -EAGAIN;
11826
11827                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
11828                                                 mdio->val_in);
11829
11830         case SIOCSHWTSTAMP:
11831                 return bnxt_hwtstamp_set(dev, ifr);
11832
11833         case SIOCGHWTSTAMP:
11834                 return bnxt_hwtstamp_get(dev, ifr);
11835
11836         default:
11837                 /* do nothing */
11838                 break;
11839         }
11840         return -EOPNOTSUPP;
11841 }
11842
11843 static void bnxt_get_ring_stats(struct bnxt *bp,
11844                                 struct rtnl_link_stats64 *stats)
11845 {
11846         int i;
11847
11848         for (i = 0; i < bp->cp_nr_rings; i++) {
11849                 struct bnxt_napi *bnapi = bp->bnapi[i];
11850                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
11851                 u64 *sw = cpr->stats.sw_stats;
11852
11853                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
11854                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
11855                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
11856
11857                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
11858                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
11859                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
11860
11861                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
11862                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
11863                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
11864
11865                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
11866                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
11867                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
11868
11869                 stats->rx_missed_errors +=
11870                         BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
11871
11872                 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
11873
11874                 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
11875
11876                 stats->rx_dropped +=
11877                         cpr->sw_stats.rx.rx_netpoll_discards +
11878                         cpr->sw_stats.rx.rx_oom_discards;
11879         }
11880 }
11881
11882 static void bnxt_add_prev_stats(struct bnxt *bp,
11883                                 struct rtnl_link_stats64 *stats)
11884 {
11885         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
11886
11887         stats->rx_packets += prev_stats->rx_packets;
11888         stats->tx_packets += prev_stats->tx_packets;
11889         stats->rx_bytes += prev_stats->rx_bytes;
11890         stats->tx_bytes += prev_stats->tx_bytes;
11891         stats->rx_missed_errors += prev_stats->rx_missed_errors;
11892         stats->multicast += prev_stats->multicast;
11893         stats->rx_dropped += prev_stats->rx_dropped;
11894         stats->tx_dropped += prev_stats->tx_dropped;
11895 }
11896
11897 static void
11898 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
11899 {
11900         struct bnxt *bp = netdev_priv(dev);
11901
11902         set_bit(BNXT_STATE_READ_STATS, &bp->state);
11903         /* Make sure bnxt_close_nic() sees that we are reading stats before
11904          * we check the BNXT_STATE_OPEN flag.
11905          */
11906         smp_mb__after_atomic();
11907         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11908                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
11909                 *stats = bp->net_stats_prev;
11910                 return;
11911         }
11912
11913         bnxt_get_ring_stats(bp, stats);
11914         bnxt_add_prev_stats(bp, stats);
11915
11916         if (bp->flags & BNXT_FLAG_PORT_STATS) {
11917                 u64 *rx = bp->port_stats.sw_stats;
11918                 u64 *tx = bp->port_stats.sw_stats +
11919                           BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
11920
11921                 stats->rx_crc_errors =
11922                         BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
11923                 stats->rx_frame_errors =
11924                         BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
11925                 stats->rx_length_errors =
11926                         BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
11927                         BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
11928                         BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
11929                 stats->rx_errors =
11930                         BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
11931                         BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
11932                 stats->collisions =
11933                         BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
11934                 stats->tx_fifo_errors =
11935                         BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
11936                 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
11937         }
11938         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
11939 }
11940
11941 static void bnxt_get_one_ring_err_stats(struct bnxt *bp,
11942                                         struct bnxt_total_ring_err_stats *stats,
11943                                         struct bnxt_cp_ring_info *cpr)
11944 {
11945         struct bnxt_sw_stats *sw_stats = &cpr->sw_stats;
11946         u64 *hw_stats = cpr->stats.sw_stats;
11947
11948         stats->rx_total_l4_csum_errors += sw_stats->rx.rx_l4_csum_errors;
11949         stats->rx_total_resets += sw_stats->rx.rx_resets;
11950         stats->rx_total_buf_errors += sw_stats->rx.rx_buf_errors;
11951         stats->rx_total_oom_discards += sw_stats->rx.rx_oom_discards;
11952         stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards;
11953         stats->rx_total_ring_discards +=
11954                 BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts);
11955         stats->tx_total_resets += sw_stats->tx.tx_resets;
11956         stats->tx_total_ring_discards +=
11957                 BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts);
11958         stats->total_missed_irqs += sw_stats->cmn.missed_irqs;
11959 }
11960
11961 void bnxt_get_ring_err_stats(struct bnxt *bp,
11962                              struct bnxt_total_ring_err_stats *stats)
11963 {
11964         int i;
11965
11966         for (i = 0; i < bp->cp_nr_rings; i++)
11967                 bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring);
11968 }
11969
11970 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
11971 {
11972         struct net_device *dev = bp->dev;
11973         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11974         struct netdev_hw_addr *ha;
11975         u8 *haddr;
11976         int mc_count = 0;
11977         bool update = false;
11978         int off = 0;
11979
11980         netdev_for_each_mc_addr(ha, dev) {
11981                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
11982                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11983                         vnic->mc_list_count = 0;
11984                         return false;
11985                 }
11986                 haddr = ha->addr;
11987                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
11988                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
11989                         update = true;
11990                 }
11991                 off += ETH_ALEN;
11992                 mc_count++;
11993         }
11994         if (mc_count)
11995                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
11996
11997         if (mc_count != vnic->mc_list_count) {
11998                 vnic->mc_list_count = mc_count;
11999                 update = true;
12000         }
12001         return update;
12002 }
12003
12004 static bool bnxt_uc_list_updated(struct bnxt *bp)
12005 {
12006         struct net_device *dev = bp->dev;
12007         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12008         struct netdev_hw_addr *ha;
12009         int off = 0;
12010
12011         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
12012                 return true;
12013
12014         netdev_for_each_uc_addr(ha, dev) {
12015                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
12016                         return true;
12017
12018                 off += ETH_ALEN;
12019         }
12020         return false;
12021 }
12022
12023 static void bnxt_set_rx_mode(struct net_device *dev)
12024 {
12025         struct bnxt *bp = netdev_priv(dev);
12026         struct bnxt_vnic_info *vnic;
12027         bool mc_update = false;
12028         bool uc_update;
12029         u32 mask;
12030
12031         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
12032                 return;
12033
12034         vnic = &bp->vnic_info[0];
12035         mask = vnic->rx_mask;
12036         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
12037                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
12038                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
12039                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
12040
12041         if (dev->flags & IFF_PROMISC)
12042                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12043
12044         uc_update = bnxt_uc_list_updated(bp);
12045
12046         if (dev->flags & IFF_BROADCAST)
12047                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
12048         if (dev->flags & IFF_ALLMULTI) {
12049                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12050                 vnic->mc_list_count = 0;
12051         } else if (dev->flags & IFF_MULTICAST) {
12052                 mc_update = bnxt_mc_list_updated(bp, &mask);
12053         }
12054
12055         if (mask != vnic->rx_mask || uc_update || mc_update) {
12056                 vnic->rx_mask = mask;
12057
12058                 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
12059         }
12060 }
12061
12062 static int bnxt_cfg_rx_mode(struct bnxt *bp)
12063 {
12064         struct net_device *dev = bp->dev;
12065         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12066         struct netdev_hw_addr *ha;
12067         int i, off = 0, rc;
12068         bool uc_update;
12069
12070         netif_addr_lock_bh(dev);
12071         uc_update = bnxt_uc_list_updated(bp);
12072         netif_addr_unlock_bh(dev);
12073
12074         if (!uc_update)
12075                 goto skip_uc;
12076
12077         for (i = 1; i < vnic->uc_filter_count; i++) {
12078                 struct bnxt_l2_filter *fltr = vnic->l2_filters[i];
12079
12080                 bnxt_hwrm_l2_filter_free(bp, fltr);
12081                 bnxt_del_l2_filter(bp, fltr);
12082         }
12083
12084         vnic->uc_filter_count = 1;
12085
12086         netif_addr_lock_bh(dev);
12087         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
12088                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12089         } else {
12090                 netdev_for_each_uc_addr(ha, dev) {
12091                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
12092                         off += ETH_ALEN;
12093                         vnic->uc_filter_count++;
12094                 }
12095         }
12096         netif_addr_unlock_bh(dev);
12097
12098         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
12099                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
12100                 if (rc) {
12101                         if (BNXT_VF(bp) && rc == -ENODEV) {
12102                                 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12103                                         netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n");
12104                                 else
12105                                         netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n");
12106                                 rc = 0;
12107                         } else {
12108                                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
12109                         }
12110                         vnic->uc_filter_count = i;
12111                         return rc;
12112                 }
12113         }
12114         if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12115                 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n");
12116
12117 skip_uc:
12118         if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
12119             !bnxt_promisc_ok(bp))
12120                 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12121         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
12122         if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
12123                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
12124                             rc);
12125                 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
12126                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12127                 vnic->mc_list_count = 0;
12128                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
12129         }
12130         if (rc)
12131                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
12132                            rc);
12133
12134         return rc;
12135 }
12136
12137 static bool bnxt_can_reserve_rings(struct bnxt *bp)
12138 {
12139 #ifdef CONFIG_BNXT_SRIOV
12140         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
12141                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
12142
12143                 /* No minimum rings were provisioned by the PF.  Don't
12144                  * reserve rings by default when device is down.
12145                  */
12146                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
12147                         return true;
12148
12149                 if (!netif_running(bp->dev))
12150                         return false;
12151         }
12152 #endif
12153         return true;
12154 }
12155
12156 /* If the chip and firmware supports RFS */
12157 static bool bnxt_rfs_supported(struct bnxt *bp)
12158 {
12159         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
12160                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
12161                         return true;
12162                 return false;
12163         }
12164         /* 212 firmware is broken for aRFS */
12165         if (BNXT_FW_MAJ(bp) == 212)
12166                 return false;
12167         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
12168                 return true;
12169         if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
12170                 return true;
12171         return false;
12172 }
12173
12174 /* If runtime conditions support RFS */
12175 static bool bnxt_rfs_capable(struct bnxt *bp)
12176 {
12177         int vnics, max_vnics, max_rss_ctxs;
12178
12179         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
12180                 return bnxt_rfs_supported(bp);
12181         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
12182                 return false;
12183
12184         vnics = 1 + bp->rx_nr_rings;
12185         max_vnics = bnxt_get_max_func_vnics(bp);
12186         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
12187
12188         /* RSS contexts not a limiting factor */
12189         if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
12190                 max_rss_ctxs = max_vnics;
12191         if (vnics > max_vnics || vnics > max_rss_ctxs) {
12192                 if (bp->rx_nr_rings > 1)
12193                         netdev_warn(bp->dev,
12194                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
12195                                     min(max_rss_ctxs - 1, max_vnics - 1));
12196                 return false;
12197         }
12198
12199         if (!BNXT_NEW_RM(bp))
12200                 return true;
12201
12202         if (vnics == bp->hw_resc.resv_vnics)
12203                 return true;
12204
12205         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
12206         if (vnics <= bp->hw_resc.resv_vnics)
12207                 return true;
12208
12209         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
12210         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
12211         return false;
12212 }
12213
12214 static netdev_features_t bnxt_fix_features(struct net_device *dev,
12215                                            netdev_features_t features)
12216 {
12217         struct bnxt *bp = netdev_priv(dev);
12218         netdev_features_t vlan_features;
12219
12220         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
12221                 features &= ~NETIF_F_NTUPLE;
12222
12223         if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog)
12224                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12225
12226         if (!(features & NETIF_F_GRO))
12227                 features &= ~NETIF_F_GRO_HW;
12228
12229         if (features & NETIF_F_GRO_HW)
12230                 features &= ~NETIF_F_LRO;
12231
12232         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
12233          * turned on or off together.
12234          */
12235         vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
12236         if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
12237                 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12238                         features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
12239                 else if (vlan_features)
12240                         features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12241         }
12242 #ifdef CONFIG_BNXT_SRIOV
12243         if (BNXT_VF(bp) && bp->vf.vlan)
12244                 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
12245 #endif
12246         return features;
12247 }
12248
12249 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
12250 {
12251         struct bnxt *bp = netdev_priv(dev);
12252         u32 flags = bp->flags;
12253         u32 changes;
12254         int rc = 0;
12255         bool re_init = false;
12256         bool update_tpa = false;
12257
12258         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
12259         if (features & NETIF_F_GRO_HW)
12260                 flags |= BNXT_FLAG_GRO;
12261         else if (features & NETIF_F_LRO)
12262                 flags |= BNXT_FLAG_LRO;
12263
12264         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
12265                 flags &= ~BNXT_FLAG_TPA;
12266
12267         if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12268                 flags |= BNXT_FLAG_STRIP_VLAN;
12269
12270         if (features & NETIF_F_NTUPLE)
12271                 flags |= BNXT_FLAG_RFS;
12272
12273         changes = flags ^ bp->flags;
12274         if (changes & BNXT_FLAG_TPA) {
12275                 update_tpa = true;
12276                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
12277                     (flags & BNXT_FLAG_TPA) == 0 ||
12278                     (bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
12279                         re_init = true;
12280         }
12281
12282         if (changes & ~BNXT_FLAG_TPA)
12283                 re_init = true;
12284
12285         if (flags != bp->flags) {
12286                 u32 old_flags = bp->flags;
12287
12288                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12289                         bp->flags = flags;
12290                         if (update_tpa)
12291                                 bnxt_set_ring_params(bp);
12292                         return rc;
12293                 }
12294
12295                 if (re_init) {
12296                         bnxt_close_nic(bp, false, false);
12297                         bp->flags = flags;
12298                         if (update_tpa)
12299                                 bnxt_set_ring_params(bp);
12300
12301                         return bnxt_open_nic(bp, false, false);
12302                 }
12303                 if (update_tpa) {
12304                         bp->flags = flags;
12305                         rc = bnxt_set_tpa(bp,
12306                                           (flags & BNXT_FLAG_TPA) ?
12307                                           true : false);
12308                         if (rc)
12309                                 bp->flags = old_flags;
12310                 }
12311         }
12312         return rc;
12313 }
12314
12315 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
12316                               u8 **nextp)
12317 {
12318         struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
12319         struct hop_jumbo_hdr *jhdr;
12320         int hdr_count = 0;
12321         u8 *nexthdr;
12322         int start;
12323
12324         /* Check that there are at most 2 IPv6 extension headers, no
12325          * fragment header, and each is <= 64 bytes.
12326          */
12327         start = nw_off + sizeof(*ip6h);
12328         nexthdr = &ip6h->nexthdr;
12329         while (ipv6_ext_hdr(*nexthdr)) {
12330                 struct ipv6_opt_hdr *hp;
12331                 int hdrlen;
12332
12333                 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
12334                     *nexthdr == NEXTHDR_FRAGMENT)
12335                         return false;
12336                 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
12337                                           skb_headlen(skb), NULL);
12338                 if (!hp)
12339                         return false;
12340                 if (*nexthdr == NEXTHDR_AUTH)
12341                         hdrlen = ipv6_authlen(hp);
12342                 else
12343                         hdrlen = ipv6_optlen(hp);
12344
12345                 if (hdrlen > 64)
12346                         return false;
12347
12348                 /* The ext header may be a hop-by-hop header inserted for
12349                  * big TCP purposes. This will be removed before sending
12350                  * from NIC, so do not count it.
12351                  */
12352                 if (*nexthdr == NEXTHDR_HOP) {
12353                         if (likely(skb->len <= GRO_LEGACY_MAX_SIZE))
12354                                 goto increment_hdr;
12355
12356                         jhdr = (struct hop_jumbo_hdr *)hp;
12357                         if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 ||
12358                             jhdr->nexthdr != IPPROTO_TCP)
12359                                 goto increment_hdr;
12360
12361                         goto next_hdr;
12362                 }
12363 increment_hdr:
12364                 hdr_count++;
12365 next_hdr:
12366                 nexthdr = &hp->nexthdr;
12367                 start += hdrlen;
12368         }
12369         if (nextp) {
12370                 /* Caller will check inner protocol */
12371                 if (skb->encapsulation) {
12372                         *nextp = nexthdr;
12373                         return true;
12374                 }
12375                 *nextp = NULL;
12376         }
12377         /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
12378         return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
12379 }
12380
12381 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
12382 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
12383 {
12384         struct udphdr *uh = udp_hdr(skb);
12385         __be16 udp_port = uh->dest;
12386
12387         if (udp_port != bp->vxlan_port && udp_port != bp->nge_port &&
12388             udp_port != bp->vxlan_gpe_port)
12389                 return false;
12390         if (skb->inner_protocol == htons(ETH_P_TEB)) {
12391                 struct ethhdr *eh = inner_eth_hdr(skb);
12392
12393                 switch (eh->h_proto) {
12394                 case htons(ETH_P_IP):
12395                         return true;
12396                 case htons(ETH_P_IPV6):
12397                         return bnxt_exthdr_check(bp, skb,
12398                                                  skb_inner_network_offset(skb),
12399                                                  NULL);
12400                 }
12401         } else if (skb->inner_protocol == htons(ETH_P_IP)) {
12402                 return true;
12403         } else if (skb->inner_protocol == htons(ETH_P_IPV6)) {
12404                 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
12405                                          NULL);
12406         }
12407         return false;
12408 }
12409
12410 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
12411 {
12412         switch (l4_proto) {
12413         case IPPROTO_UDP:
12414                 return bnxt_udp_tunl_check(bp, skb);
12415         case IPPROTO_IPIP:
12416                 return true;
12417         case IPPROTO_GRE: {
12418                 switch (skb->inner_protocol) {
12419                 default:
12420                         return false;
12421                 case htons(ETH_P_IP):
12422                         return true;
12423                 case htons(ETH_P_IPV6):
12424                         fallthrough;
12425                 }
12426         }
12427         case IPPROTO_IPV6:
12428                 /* Check ext headers of inner ipv6 */
12429                 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
12430                                          NULL);
12431         }
12432         return false;
12433 }
12434
12435 static netdev_features_t bnxt_features_check(struct sk_buff *skb,
12436                                              struct net_device *dev,
12437                                              netdev_features_t features)
12438 {
12439         struct bnxt *bp = netdev_priv(dev);
12440         u8 *l4_proto;
12441
12442         features = vlan_features_check(skb, features);
12443         switch (vlan_get_protocol(skb)) {
12444         case htons(ETH_P_IP):
12445                 if (!skb->encapsulation)
12446                         return features;
12447                 l4_proto = &ip_hdr(skb)->protocol;
12448                 if (bnxt_tunl_check(bp, skb, *l4_proto))
12449                         return features;
12450                 break;
12451         case htons(ETH_P_IPV6):
12452                 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
12453                                        &l4_proto))
12454                         break;
12455                 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
12456                         return features;
12457                 break;
12458         }
12459         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
12460 }
12461
12462 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
12463                          u32 *reg_buf)
12464 {
12465         struct hwrm_dbg_read_direct_output *resp;
12466         struct hwrm_dbg_read_direct_input *req;
12467         __le32 *dbg_reg_buf;
12468         dma_addr_t mapping;
12469         int rc, i;
12470
12471         rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT);
12472         if (rc)
12473                 return rc;
12474
12475         dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4,
12476                                          &mapping);
12477         if (!dbg_reg_buf) {
12478                 rc = -ENOMEM;
12479                 goto dbg_rd_reg_exit;
12480         }
12481
12482         req->host_dest_addr = cpu_to_le64(mapping);
12483
12484         resp = hwrm_req_hold(bp, req);
12485         req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
12486         req->read_len32 = cpu_to_le32(num_words);
12487
12488         rc = hwrm_req_send(bp, req);
12489         if (rc || resp->error_code) {
12490                 rc = -EIO;
12491                 goto dbg_rd_reg_exit;
12492         }
12493         for (i = 0; i < num_words; i++)
12494                 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
12495
12496 dbg_rd_reg_exit:
12497         hwrm_req_drop(bp, req);
12498         return rc;
12499 }
12500
12501 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
12502                                        u32 ring_id, u32 *prod, u32 *cons)
12503 {
12504         struct hwrm_dbg_ring_info_get_output *resp;
12505         struct hwrm_dbg_ring_info_get_input *req;
12506         int rc;
12507
12508         rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET);
12509         if (rc)
12510                 return rc;
12511
12512         req->ring_type = ring_type;
12513         req->fw_ring_id = cpu_to_le32(ring_id);
12514         resp = hwrm_req_hold(bp, req);
12515         rc = hwrm_req_send(bp, req);
12516         if (!rc) {
12517                 *prod = le32_to_cpu(resp->producer_index);
12518                 *cons = le32_to_cpu(resp->consumer_index);
12519         }
12520         hwrm_req_drop(bp, req);
12521         return rc;
12522 }
12523
12524 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
12525 {
12526         struct bnxt_tx_ring_info *txr;
12527         int i = bnapi->index, j;
12528
12529         bnxt_for_each_napi_tx(j, bnapi, txr)
12530                 netdev_info(bnapi->bp->dev, "[%d.%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
12531                             i, j, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
12532                             txr->tx_cons);
12533 }
12534
12535 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
12536 {
12537         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
12538         int i = bnapi->index;
12539
12540         if (!rxr)
12541                 return;
12542
12543         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
12544                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
12545                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
12546                     rxr->rx_sw_agg_prod);
12547 }
12548
12549 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
12550 {
12551         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
12552         int i = bnapi->index;
12553
12554         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
12555                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
12556 }
12557
12558 static void bnxt_dbg_dump_states(struct bnxt *bp)
12559 {
12560         int i;
12561         struct bnxt_napi *bnapi;
12562
12563         for (i = 0; i < bp->cp_nr_rings; i++) {
12564                 bnapi = bp->bnapi[i];
12565                 if (netif_msg_drv(bp)) {
12566                         bnxt_dump_tx_sw_state(bnapi);
12567                         bnxt_dump_rx_sw_state(bnapi);
12568                         bnxt_dump_cp_sw_state(bnapi);
12569                 }
12570         }
12571 }
12572
12573 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
12574 {
12575         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
12576         struct hwrm_ring_reset_input *req;
12577         struct bnxt_napi *bnapi = rxr->bnapi;
12578         struct bnxt_cp_ring_info *cpr;
12579         u16 cp_ring_id;
12580         int rc;
12581
12582         rc = hwrm_req_init(bp, req, HWRM_RING_RESET);
12583         if (rc)
12584                 return rc;
12585
12586         cpr = &bnapi->cp_ring;
12587         cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
12588         req->cmpl_ring = cpu_to_le16(cp_ring_id);
12589         req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
12590         req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
12591         return hwrm_req_send_silent(bp, req);
12592 }
12593
12594 static void bnxt_reset_task(struct bnxt *bp, bool silent)
12595 {
12596         if (!silent)
12597                 bnxt_dbg_dump_states(bp);
12598         if (netif_running(bp->dev)) {
12599                 int rc;
12600
12601                 if (silent) {
12602                         bnxt_close_nic(bp, false, false);
12603                         bnxt_open_nic(bp, false, false);
12604                 } else {
12605                         bnxt_ulp_stop(bp);
12606                         bnxt_close_nic(bp, true, false);
12607                         rc = bnxt_open_nic(bp, true, false);
12608                         bnxt_ulp_start(bp, rc);
12609                 }
12610         }
12611 }
12612
12613 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
12614 {
12615         struct bnxt *bp = netdev_priv(dev);
12616
12617         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
12618         bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
12619 }
12620
12621 static void bnxt_fw_health_check(struct bnxt *bp)
12622 {
12623         struct bnxt_fw_health *fw_health = bp->fw_health;
12624         struct pci_dev *pdev = bp->pdev;
12625         u32 val;
12626
12627         if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
12628                 return;
12629
12630         /* Make sure it is enabled before checking the tmr_counter. */
12631         smp_rmb();
12632         if (fw_health->tmr_counter) {
12633                 fw_health->tmr_counter--;
12634                 return;
12635         }
12636
12637         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
12638         if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) {
12639                 fw_health->arrests++;
12640                 goto fw_reset;
12641         }
12642
12643         fw_health->last_fw_heartbeat = val;
12644
12645         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
12646         if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) {
12647                 fw_health->discoveries++;
12648                 goto fw_reset;
12649         }
12650
12651         fw_health->tmr_counter = fw_health->tmr_multiplier;
12652         return;
12653
12654 fw_reset:
12655         bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT);
12656 }
12657
12658 static void bnxt_timer(struct timer_list *t)
12659 {
12660         struct bnxt *bp = from_timer(bp, t, timer);
12661         struct net_device *dev = bp->dev;
12662
12663         if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
12664                 return;
12665
12666         if (atomic_read(&bp->intr_sem) != 0)
12667                 goto bnxt_restart_timer;
12668
12669         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
12670                 bnxt_fw_health_check(bp);
12671
12672         if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks)
12673                 bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT);
12674
12675         if (bnxt_tc_flower_enabled(bp))
12676                 bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT);
12677
12678 #ifdef CONFIG_RFS_ACCEL
12679         if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count)
12680                 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
12681 #endif /*CONFIG_RFS_ACCEL*/
12682
12683         if (bp->link_info.phy_retry) {
12684                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
12685                         bp->link_info.phy_retry = false;
12686                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
12687                 } else {
12688                         bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT);
12689                 }
12690         }
12691
12692         if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12693                 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
12694
12695         if ((BNXT_CHIP_P5(bp)) && !bp->chip_rev && netif_carrier_ok(dev))
12696                 bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT);
12697
12698 bnxt_restart_timer:
12699         mod_timer(&bp->timer, jiffies + bp->current_interval);
12700 }
12701
12702 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
12703 {
12704         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
12705          * set.  If the device is being closed, bnxt_close() may be holding
12706          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
12707          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
12708          */
12709         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12710         rtnl_lock();
12711 }
12712
12713 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
12714 {
12715         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12716         rtnl_unlock();
12717 }
12718
12719 /* Only called from bnxt_sp_task() */
12720 static void bnxt_reset(struct bnxt *bp, bool silent)
12721 {
12722         bnxt_rtnl_lock_sp(bp);
12723         if (test_bit(BNXT_STATE_OPEN, &bp->state))
12724                 bnxt_reset_task(bp, silent);
12725         bnxt_rtnl_unlock_sp(bp);
12726 }
12727
12728 /* Only called from bnxt_sp_task() */
12729 static void bnxt_rx_ring_reset(struct bnxt *bp)
12730 {
12731         int i;
12732
12733         bnxt_rtnl_lock_sp(bp);
12734         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12735                 bnxt_rtnl_unlock_sp(bp);
12736                 return;
12737         }
12738         /* Disable and flush TPA before resetting the RX ring */
12739         if (bp->flags & BNXT_FLAG_TPA)
12740                 bnxt_set_tpa(bp, false);
12741         for (i = 0; i < bp->rx_nr_rings; i++) {
12742                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
12743                 struct bnxt_cp_ring_info *cpr;
12744                 int rc;
12745
12746                 if (!rxr->bnapi->in_reset)
12747                         continue;
12748
12749                 rc = bnxt_hwrm_rx_ring_reset(bp, i);
12750                 if (rc) {
12751                         if (rc == -EINVAL || rc == -EOPNOTSUPP)
12752                                 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
12753                         else
12754                                 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
12755                                             rc);
12756                         bnxt_reset_task(bp, true);
12757                         break;
12758                 }
12759                 bnxt_free_one_rx_ring_skbs(bp, i);
12760                 rxr->rx_prod = 0;
12761                 rxr->rx_agg_prod = 0;
12762                 rxr->rx_sw_agg_prod = 0;
12763                 rxr->rx_next_cons = 0;
12764                 rxr->bnapi->in_reset = false;
12765                 bnxt_alloc_one_rx_ring(bp, i);
12766                 cpr = &rxr->bnapi->cp_ring;
12767                 cpr->sw_stats.rx.rx_resets++;
12768                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
12769                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
12770                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
12771         }
12772         if (bp->flags & BNXT_FLAG_TPA)
12773                 bnxt_set_tpa(bp, true);
12774         bnxt_rtnl_unlock_sp(bp);
12775 }
12776
12777 static void bnxt_fw_reset_close(struct bnxt *bp)
12778 {
12779         bnxt_ulp_stop(bp);
12780         /* When firmware is in fatal state, quiesce device and disable
12781          * bus master to prevent any potential bad DMAs before freeing
12782          * kernel memory.
12783          */
12784         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
12785                 u16 val = 0;
12786
12787                 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
12788                 if (val == 0xffff)
12789                         bp->fw_reset_min_dsecs = 0;
12790                 bnxt_tx_disable(bp);
12791                 bnxt_disable_napi(bp);
12792                 bnxt_disable_int_sync(bp);
12793                 bnxt_free_irq(bp);
12794                 bnxt_clear_int_mode(bp);
12795                 pci_disable_device(bp->pdev);
12796         }
12797         __bnxt_close_nic(bp, true, false);
12798         bnxt_vf_reps_free(bp);
12799         bnxt_clear_int_mode(bp);
12800         bnxt_hwrm_func_drv_unrgtr(bp);
12801         if (pci_is_enabled(bp->pdev))
12802                 pci_disable_device(bp->pdev);
12803         bnxt_free_ctx_mem(bp);
12804 }
12805
12806 static bool is_bnxt_fw_ok(struct bnxt *bp)
12807 {
12808         struct bnxt_fw_health *fw_health = bp->fw_health;
12809         bool no_heartbeat = false, has_reset = false;
12810         u32 val;
12811
12812         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
12813         if (val == fw_health->last_fw_heartbeat)
12814                 no_heartbeat = true;
12815
12816         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
12817         if (val != fw_health->last_fw_reset_cnt)
12818                 has_reset = true;
12819
12820         if (!no_heartbeat && has_reset)
12821                 return true;
12822
12823         return false;
12824 }
12825
12826 /* rtnl_lock is acquired before calling this function */
12827 static void bnxt_force_fw_reset(struct bnxt *bp)
12828 {
12829         struct bnxt_fw_health *fw_health = bp->fw_health;
12830         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
12831         u32 wait_dsecs;
12832
12833         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
12834             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
12835                 return;
12836
12837         if (ptp) {
12838                 spin_lock_bh(&ptp->ptp_lock);
12839                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12840                 spin_unlock_bh(&ptp->ptp_lock);
12841         } else {
12842                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12843         }
12844         bnxt_fw_reset_close(bp);
12845         wait_dsecs = fw_health->master_func_wait_dsecs;
12846         if (fw_health->primary) {
12847                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
12848                         wait_dsecs = 0;
12849                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
12850         } else {
12851                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
12852                 wait_dsecs = fw_health->normal_func_wait_dsecs;
12853                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12854         }
12855
12856         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
12857         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
12858         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
12859 }
12860
12861 void bnxt_fw_exception(struct bnxt *bp)
12862 {
12863         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
12864         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
12865         bnxt_rtnl_lock_sp(bp);
12866         bnxt_force_fw_reset(bp);
12867         bnxt_rtnl_unlock_sp(bp);
12868 }
12869
12870 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
12871  * < 0 on error.
12872  */
12873 static int bnxt_get_registered_vfs(struct bnxt *bp)
12874 {
12875 #ifdef CONFIG_BNXT_SRIOV
12876         int rc;
12877
12878         if (!BNXT_PF(bp))
12879                 return 0;
12880
12881         rc = bnxt_hwrm_func_qcfg(bp);
12882         if (rc) {
12883                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
12884                 return rc;
12885         }
12886         if (bp->pf.registered_vfs)
12887                 return bp->pf.registered_vfs;
12888         if (bp->sriov_cfg)
12889                 return 1;
12890 #endif
12891         return 0;
12892 }
12893
12894 void bnxt_fw_reset(struct bnxt *bp)
12895 {
12896         bnxt_rtnl_lock_sp(bp);
12897         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
12898             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12899                 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
12900                 int n = 0, tmo;
12901
12902                 if (ptp) {
12903                         spin_lock_bh(&ptp->ptp_lock);
12904                         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12905                         spin_unlock_bh(&ptp->ptp_lock);
12906                 } else {
12907                         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12908                 }
12909                 if (bp->pf.active_vfs &&
12910                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
12911                         n = bnxt_get_registered_vfs(bp);
12912                 if (n < 0) {
12913                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
12914                                    n);
12915                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12916                         dev_close(bp->dev);
12917                         goto fw_reset_exit;
12918                 } else if (n > 0) {
12919                         u16 vf_tmo_dsecs = n * 10;
12920
12921                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
12922                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
12923                         bp->fw_reset_state =
12924                                 BNXT_FW_RESET_STATE_POLL_VF;
12925                         bnxt_queue_fw_reset_work(bp, HZ / 10);
12926                         goto fw_reset_exit;
12927                 }
12928                 bnxt_fw_reset_close(bp);
12929                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12930                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
12931                         tmo = HZ / 10;
12932                 } else {
12933                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12934                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
12935                 }
12936                 bnxt_queue_fw_reset_work(bp, tmo);
12937         }
12938 fw_reset_exit:
12939         bnxt_rtnl_unlock_sp(bp);
12940 }
12941
12942 static void bnxt_chk_missed_irq(struct bnxt *bp)
12943 {
12944         int i;
12945
12946         if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
12947                 return;
12948
12949         for (i = 0; i < bp->cp_nr_rings; i++) {
12950                 struct bnxt_napi *bnapi = bp->bnapi[i];
12951                 struct bnxt_cp_ring_info *cpr;
12952                 u32 fw_ring_id;
12953                 int j;
12954
12955                 if (!bnapi)
12956                         continue;
12957
12958                 cpr = &bnapi->cp_ring;
12959                 for (j = 0; j < cpr->cp_ring_count; j++) {
12960                         struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
12961                         u32 val[2];
12962
12963                         if (cpr2->has_more_work || !bnxt_has_work(bp, cpr2))
12964                                 continue;
12965
12966                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
12967                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
12968                                 continue;
12969                         }
12970                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
12971                         bnxt_dbg_hwrm_ring_info_get(bp,
12972                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
12973                                 fw_ring_id, &val[0], &val[1]);
12974                         cpr->sw_stats.cmn.missed_irqs++;
12975                 }
12976         }
12977 }
12978
12979 static void bnxt_cfg_ntp_filters(struct bnxt *);
12980
12981 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
12982 {
12983         struct bnxt_link_info *link_info = &bp->link_info;
12984
12985         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
12986                 link_info->autoneg = BNXT_AUTONEG_SPEED;
12987                 if (bp->hwrm_spec_code >= 0x10201) {
12988                         if (link_info->auto_pause_setting &
12989                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
12990                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
12991                 } else {
12992                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
12993                 }
12994                 bnxt_set_auto_speed(link_info);
12995         } else {
12996                 bnxt_set_force_speed(link_info);
12997                 link_info->req_duplex = link_info->duplex_setting;
12998         }
12999         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
13000                 link_info->req_flow_ctrl =
13001                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
13002         else
13003                 link_info->req_flow_ctrl = link_info->force_pause_setting;
13004 }
13005
13006 static void bnxt_fw_echo_reply(struct bnxt *bp)
13007 {
13008         struct bnxt_fw_health *fw_health = bp->fw_health;
13009         struct hwrm_func_echo_response_input *req;
13010         int rc;
13011
13012         rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE);
13013         if (rc)
13014                 return;
13015         req->event_data1 = cpu_to_le32(fw_health->echo_req_data1);
13016         req->event_data2 = cpu_to_le32(fw_health->echo_req_data2);
13017         hwrm_req_send(bp, req);
13018 }
13019
13020 static void bnxt_sp_task(struct work_struct *work)
13021 {
13022         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
13023
13024         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13025         smp_mb__after_atomic();
13026         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
13027                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13028                 return;
13029         }
13030
13031         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
13032                 bnxt_cfg_rx_mode(bp);
13033
13034         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
13035                 bnxt_cfg_ntp_filters(bp);
13036         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
13037                 bnxt_hwrm_exec_fwd_req(bp);
13038         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
13039                 netdev_info(bp->dev, "Receive PF driver unload event!\n");
13040         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
13041                 bnxt_hwrm_port_qstats(bp, 0);
13042                 bnxt_hwrm_port_qstats_ext(bp, 0);
13043                 bnxt_accumulate_all_stats(bp);
13044         }
13045
13046         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
13047                 int rc;
13048
13049                 mutex_lock(&bp->link_lock);
13050                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
13051                                        &bp->sp_event))
13052                         bnxt_hwrm_phy_qcaps(bp);
13053
13054                 rc = bnxt_update_link(bp, true);
13055                 if (rc)
13056                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
13057                                    rc);
13058
13059                 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
13060                                        &bp->sp_event))
13061                         bnxt_init_ethtool_link_settings(bp);
13062                 mutex_unlock(&bp->link_lock);
13063         }
13064         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
13065                 int rc;
13066
13067                 mutex_lock(&bp->link_lock);
13068                 rc = bnxt_update_phy_setting(bp);
13069                 mutex_unlock(&bp->link_lock);
13070                 if (rc) {
13071                         netdev_warn(bp->dev, "update phy settings retry failed\n");
13072                 } else {
13073                         bp->link_info.phy_retry = false;
13074                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
13075                 }
13076         }
13077         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
13078                 mutex_lock(&bp->link_lock);
13079                 bnxt_get_port_module_status(bp);
13080                 mutex_unlock(&bp->link_lock);
13081         }
13082
13083         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
13084                 bnxt_tc_flow_stats_work(bp);
13085
13086         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
13087                 bnxt_chk_missed_irq(bp);
13088
13089         if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
13090                 bnxt_fw_echo_reply(bp);
13091
13092         if (test_and_clear_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event))
13093                 bnxt_hwmon_notify_event(bp);
13094
13095         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
13096          * must be the last functions to be called before exiting.
13097          */
13098         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
13099                 bnxt_reset(bp, false);
13100
13101         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
13102                 bnxt_reset(bp, true);
13103
13104         if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
13105                 bnxt_rx_ring_reset(bp);
13106
13107         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) {
13108                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) ||
13109                     test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state))
13110                         bnxt_devlink_health_fw_report(bp);
13111                 else
13112                         bnxt_fw_reset(bp);
13113         }
13114
13115         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
13116                 if (!is_bnxt_fw_ok(bp))
13117                         bnxt_devlink_health_fw_report(bp);
13118         }
13119
13120         smp_mb__before_atomic();
13121         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13122 }
13123
13124 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13125                                 int *max_cp);
13126
13127 /* Under rtnl_lock */
13128 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
13129                      int tx_xdp)
13130 {
13131         int max_rx, max_tx, max_cp, tx_sets = 1, tx_cp;
13132         int tx_rings_needed, stats;
13133         int rx_rings = rx;
13134         int cp, vnics;
13135
13136         if (tcs)
13137                 tx_sets = tcs;
13138
13139         _bnxt_get_max_rings(bp, &max_rx, &max_tx, &max_cp);
13140
13141         if (max_rx < rx_rings)
13142                 return -ENOMEM;
13143
13144         if (bp->flags & BNXT_FLAG_AGG_RINGS)
13145                 rx_rings <<= 1;
13146
13147         tx_rings_needed = tx * tx_sets + tx_xdp;
13148         if (max_tx < tx_rings_needed)
13149                 return -ENOMEM;
13150
13151         vnics = 1;
13152         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5_PLUS)) ==
13153             BNXT_FLAG_RFS)
13154                 vnics += rx;
13155
13156         tx_cp = __bnxt_num_tx_to_cp(bp, tx_rings_needed, tx_sets, tx_xdp);
13157         cp = sh ? max_t(int, tx_cp, rx) : tx_cp + rx;
13158         if (max_cp < cp)
13159                 return -ENOMEM;
13160         stats = cp;
13161         if (BNXT_NEW_RM(bp)) {
13162                 cp += bnxt_get_ulp_msix_num(bp);
13163                 stats += bnxt_get_ulp_stat_ctxs(bp);
13164         }
13165         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
13166                                      stats, vnics);
13167 }
13168
13169 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
13170 {
13171         if (bp->bar2) {
13172                 pci_iounmap(pdev, bp->bar2);
13173                 bp->bar2 = NULL;
13174         }
13175
13176         if (bp->bar1) {
13177                 pci_iounmap(pdev, bp->bar1);
13178                 bp->bar1 = NULL;
13179         }
13180
13181         if (bp->bar0) {
13182                 pci_iounmap(pdev, bp->bar0);
13183                 bp->bar0 = NULL;
13184         }
13185 }
13186
13187 static void bnxt_cleanup_pci(struct bnxt *bp)
13188 {
13189         bnxt_unmap_bars(bp, bp->pdev);
13190         pci_release_regions(bp->pdev);
13191         if (pci_is_enabled(bp->pdev))
13192                 pci_disable_device(bp->pdev);
13193 }
13194
13195 static void bnxt_init_dflt_coal(struct bnxt *bp)
13196 {
13197         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
13198         struct bnxt_coal *coal;
13199         u16 flags = 0;
13200
13201         if (coal_cap->cmpl_params &
13202             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
13203                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
13204
13205         /* Tick values in micro seconds.
13206          * 1 coal_buf x bufs_per_record = 1 completion record.
13207          */
13208         coal = &bp->rx_coal;
13209         coal->coal_ticks = 10;
13210         coal->coal_bufs = 30;
13211         coal->coal_ticks_irq = 1;
13212         coal->coal_bufs_irq = 2;
13213         coal->idle_thresh = 50;
13214         coal->bufs_per_record = 2;
13215         coal->budget = 64;              /* NAPI budget */
13216         coal->flags = flags;
13217
13218         coal = &bp->tx_coal;
13219         coal->coal_ticks = 28;
13220         coal->coal_bufs = 30;
13221         coal->coal_ticks_irq = 2;
13222         coal->coal_bufs_irq = 2;
13223         coal->bufs_per_record = 1;
13224         coal->flags = flags;
13225
13226         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
13227 }
13228
13229 /* FW that pre-reserves 1 VNIC per function */
13230 static bool bnxt_fw_pre_resv_vnics(struct bnxt *bp)
13231 {
13232         u16 fw_maj = BNXT_FW_MAJ(bp), fw_bld = BNXT_FW_BLD(bp);
13233
13234         if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
13235             (fw_maj > 218 || (fw_maj == 218 && fw_bld >= 18)))
13236                 return true;
13237         if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
13238             (fw_maj > 216 || (fw_maj == 216 && fw_bld >= 172)))
13239                 return true;
13240         return false;
13241 }
13242
13243 static int bnxt_fw_init_one_p1(struct bnxt *bp)
13244 {
13245         int rc;
13246
13247         bp->fw_cap = 0;
13248         rc = bnxt_hwrm_ver_get(bp);
13249         /* FW may be unresponsive after FLR. FLR must complete within 100 msec
13250          * so wait before continuing with recovery.
13251          */
13252         if (rc)
13253                 msleep(100);
13254         bnxt_try_map_fw_health_reg(bp);
13255         if (rc) {
13256                 rc = bnxt_try_recover_fw(bp);
13257                 if (rc)
13258                         return rc;
13259                 rc = bnxt_hwrm_ver_get(bp);
13260                 if (rc)
13261                         return rc;
13262         }
13263
13264         bnxt_nvm_cfg_ver_get(bp);
13265
13266         rc = bnxt_hwrm_func_reset(bp);
13267         if (rc)
13268                 return -ENODEV;
13269
13270         bnxt_hwrm_fw_set_time(bp);
13271         return 0;
13272 }
13273
13274 static int bnxt_fw_init_one_p2(struct bnxt *bp)
13275 {
13276         int rc;
13277
13278         /* Get the MAX capabilities for this function */
13279         rc = bnxt_hwrm_func_qcaps(bp);
13280         if (rc) {
13281                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
13282                            rc);
13283                 return -ENODEV;
13284         }
13285
13286         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
13287         if (rc)
13288                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
13289                             rc);
13290
13291         if (bnxt_alloc_fw_health(bp)) {
13292                 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
13293         } else {
13294                 rc = bnxt_hwrm_error_recovery_qcfg(bp);
13295                 if (rc)
13296                         netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
13297                                     rc);
13298         }
13299
13300         rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
13301         if (rc)
13302                 return -ENODEV;
13303
13304         if (bnxt_fw_pre_resv_vnics(bp))
13305                 bp->fw_cap |= BNXT_FW_CAP_PRE_RESV_VNICS;
13306
13307         bnxt_hwrm_func_qcfg(bp);
13308         bnxt_hwrm_vnic_qcaps(bp);
13309         bnxt_hwrm_port_led_qcaps(bp);
13310         bnxt_ethtool_init(bp);
13311         if (bp->fw_cap & BNXT_FW_CAP_PTP)
13312                 __bnxt_hwrm_ptp_qcfg(bp);
13313         bnxt_dcb_init(bp);
13314         bnxt_hwmon_init(bp);
13315         return 0;
13316 }
13317
13318 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
13319 {
13320         bp->rss_cap &= ~BNXT_RSS_CAP_UDP_RSS_CAP;
13321         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
13322                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
13323                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
13324                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
13325         if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA)
13326                 bp->rss_hash_delta = bp->rss_hash_cfg;
13327         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
13328                 bp->rss_cap |= BNXT_RSS_CAP_UDP_RSS_CAP;
13329                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
13330                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
13331         }
13332 }
13333
13334 static void bnxt_set_dflt_rfs(struct bnxt *bp)
13335 {
13336         struct net_device *dev = bp->dev;
13337
13338         dev->hw_features &= ~NETIF_F_NTUPLE;
13339         dev->features &= ~NETIF_F_NTUPLE;
13340         bp->flags &= ~BNXT_FLAG_RFS;
13341         if (bnxt_rfs_supported(bp)) {
13342                 dev->hw_features |= NETIF_F_NTUPLE;
13343                 if (bnxt_rfs_capable(bp)) {
13344                         bp->flags |= BNXT_FLAG_RFS;
13345                         dev->features |= NETIF_F_NTUPLE;
13346                 }
13347         }
13348 }
13349
13350 static void bnxt_fw_init_one_p3(struct bnxt *bp)
13351 {
13352         struct pci_dev *pdev = bp->pdev;
13353
13354         bnxt_set_dflt_rss_hash_type(bp);
13355         bnxt_set_dflt_rfs(bp);
13356
13357         bnxt_get_wol_settings(bp);
13358         if (bp->flags & BNXT_FLAG_WOL_CAP)
13359                 device_set_wakeup_enable(&pdev->dev, bp->wol);
13360         else
13361                 device_set_wakeup_capable(&pdev->dev, false);
13362
13363         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
13364         bnxt_hwrm_coal_params_qcaps(bp);
13365 }
13366
13367 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
13368
13369 int bnxt_fw_init_one(struct bnxt *bp)
13370 {
13371         int rc;
13372
13373         rc = bnxt_fw_init_one_p1(bp);
13374         if (rc) {
13375                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
13376                 return rc;
13377         }
13378         rc = bnxt_fw_init_one_p2(bp);
13379         if (rc) {
13380                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
13381                 return rc;
13382         }
13383         rc = bnxt_probe_phy(bp, false);
13384         if (rc)
13385                 return rc;
13386         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
13387         if (rc)
13388                 return rc;
13389
13390         bnxt_fw_init_one_p3(bp);
13391         return 0;
13392 }
13393
13394 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
13395 {
13396         struct bnxt_fw_health *fw_health = bp->fw_health;
13397         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
13398         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
13399         u32 reg_type, reg_off, delay_msecs;
13400
13401         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
13402         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
13403         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
13404         switch (reg_type) {
13405         case BNXT_FW_HEALTH_REG_TYPE_CFG:
13406                 pci_write_config_dword(bp->pdev, reg_off, val);
13407                 break;
13408         case BNXT_FW_HEALTH_REG_TYPE_GRC:
13409                 writel(reg_off & BNXT_GRC_BASE_MASK,
13410                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
13411                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
13412                 fallthrough;
13413         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
13414                 writel(val, bp->bar0 + reg_off);
13415                 break;
13416         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
13417                 writel(val, bp->bar1 + reg_off);
13418                 break;
13419         }
13420         if (delay_msecs) {
13421                 pci_read_config_dword(bp->pdev, 0, &val);
13422                 msleep(delay_msecs);
13423         }
13424 }
13425
13426 bool bnxt_hwrm_reset_permitted(struct bnxt *bp)
13427 {
13428         struct hwrm_func_qcfg_output *resp;
13429         struct hwrm_func_qcfg_input *req;
13430         bool result = true; /* firmware will enforce if unknown */
13431
13432         if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF)
13433                 return result;
13434
13435         if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG))
13436                 return result;
13437
13438         req->fid = cpu_to_le16(0xffff);
13439         resp = hwrm_req_hold(bp, req);
13440         if (!hwrm_req_send(bp, req))
13441                 result = !!(le16_to_cpu(resp->flags) &
13442                             FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED);
13443         hwrm_req_drop(bp, req);
13444         return result;
13445 }
13446
13447 static void bnxt_reset_all(struct bnxt *bp)
13448 {
13449         struct bnxt_fw_health *fw_health = bp->fw_health;
13450         int i, rc;
13451
13452         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
13453                 bnxt_fw_reset_via_optee(bp);
13454                 bp->fw_reset_timestamp = jiffies;
13455                 return;
13456         }
13457
13458         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
13459                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
13460                         bnxt_fw_reset_writel(bp, i);
13461         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
13462                 struct hwrm_fw_reset_input *req;
13463
13464                 rc = hwrm_req_init(bp, req, HWRM_FW_RESET);
13465                 if (!rc) {
13466                         req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG);
13467                         req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
13468                         req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
13469                         req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
13470                         rc = hwrm_req_send(bp, req);
13471                 }
13472                 if (rc != -ENODEV)
13473                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
13474         }
13475         bp->fw_reset_timestamp = jiffies;
13476 }
13477
13478 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
13479 {
13480         return time_after(jiffies, bp->fw_reset_timestamp +
13481                           (bp->fw_reset_max_dsecs * HZ / 10));
13482 }
13483
13484 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc)
13485 {
13486         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13487         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) {
13488                 bnxt_ulp_start(bp, rc);
13489                 bnxt_dl_health_fw_status_update(bp, false);
13490         }
13491         bp->fw_reset_state = 0;
13492         dev_close(bp->dev);
13493 }
13494
13495 static void bnxt_fw_reset_task(struct work_struct *work)
13496 {
13497         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
13498         int rc = 0;
13499
13500         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
13501                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
13502                 return;
13503         }
13504
13505         switch (bp->fw_reset_state) {
13506         case BNXT_FW_RESET_STATE_POLL_VF: {
13507                 int n = bnxt_get_registered_vfs(bp);
13508                 int tmo;
13509
13510                 if (n < 0) {
13511                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
13512                                    n, jiffies_to_msecs(jiffies -
13513                                    bp->fw_reset_timestamp));
13514                         goto fw_reset_abort;
13515                 } else if (n > 0) {
13516                         if (bnxt_fw_reset_timeout(bp)) {
13517                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13518                                 bp->fw_reset_state = 0;
13519                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
13520                                            n);
13521                                 return;
13522                         }
13523                         bnxt_queue_fw_reset_work(bp, HZ / 10);
13524                         return;
13525                 }
13526                 bp->fw_reset_timestamp = jiffies;
13527                 rtnl_lock();
13528                 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
13529                         bnxt_fw_reset_abort(bp, rc);
13530                         rtnl_unlock();
13531                         return;
13532                 }
13533                 bnxt_fw_reset_close(bp);
13534                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
13535                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
13536                         tmo = HZ / 10;
13537                 } else {
13538                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13539                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
13540                 }
13541                 rtnl_unlock();
13542                 bnxt_queue_fw_reset_work(bp, tmo);
13543                 return;
13544         }
13545         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
13546                 u32 val;
13547
13548                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
13549                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
13550                     !bnxt_fw_reset_timeout(bp)) {
13551                         bnxt_queue_fw_reset_work(bp, HZ / 5);
13552                         return;
13553                 }
13554
13555                 if (!bp->fw_health->primary) {
13556                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
13557
13558                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13559                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
13560                         return;
13561                 }
13562                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
13563         }
13564                 fallthrough;
13565         case BNXT_FW_RESET_STATE_RESET_FW:
13566                 bnxt_reset_all(bp);
13567                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13568                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
13569                 return;
13570         case BNXT_FW_RESET_STATE_ENABLE_DEV:
13571                 bnxt_inv_fw_health_reg(bp);
13572                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
13573                     !bp->fw_reset_min_dsecs) {
13574                         u16 val;
13575
13576                         pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
13577                         if (val == 0xffff) {
13578                                 if (bnxt_fw_reset_timeout(bp)) {
13579                                         netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
13580                                         rc = -ETIMEDOUT;
13581                                         goto fw_reset_abort;
13582                                 }
13583                                 bnxt_queue_fw_reset_work(bp, HZ / 1000);
13584                                 return;
13585                         }
13586                 }
13587                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
13588                 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
13589                 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) &&
13590                     !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state))
13591                         bnxt_dl_remote_reload(bp);
13592                 if (pci_enable_device(bp->pdev)) {
13593                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
13594                         rc = -ENODEV;
13595                         goto fw_reset_abort;
13596                 }
13597                 pci_set_master(bp->pdev);
13598                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
13599                 fallthrough;
13600         case BNXT_FW_RESET_STATE_POLL_FW:
13601                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
13602                 rc = bnxt_hwrm_poll(bp);
13603                 if (rc) {
13604                         if (bnxt_fw_reset_timeout(bp)) {
13605                                 netdev_err(bp->dev, "Firmware reset aborted\n");
13606                                 goto fw_reset_abort_status;
13607                         }
13608                         bnxt_queue_fw_reset_work(bp, HZ / 5);
13609                         return;
13610                 }
13611                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
13612                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
13613                 fallthrough;
13614         case BNXT_FW_RESET_STATE_OPENING:
13615                 while (!rtnl_trylock()) {
13616                         bnxt_queue_fw_reset_work(bp, HZ / 10);
13617                         return;
13618                 }
13619                 rc = bnxt_open(bp->dev);
13620                 if (rc) {
13621                         netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
13622                         bnxt_fw_reset_abort(bp, rc);
13623                         rtnl_unlock();
13624                         return;
13625                 }
13626
13627                 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) &&
13628                     bp->fw_health->enabled) {
13629                         bp->fw_health->last_fw_reset_cnt =
13630                                 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
13631                 }
13632                 bp->fw_reset_state = 0;
13633                 /* Make sure fw_reset_state is 0 before clearing the flag */
13634                 smp_mb__before_atomic();
13635                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13636                 bnxt_ulp_start(bp, 0);
13637                 bnxt_reenable_sriov(bp);
13638                 bnxt_vf_reps_alloc(bp);
13639                 bnxt_vf_reps_open(bp);
13640                 bnxt_ptp_reapply_pps(bp);
13641                 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state);
13642                 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) {
13643                         bnxt_dl_health_fw_recovery_done(bp);
13644                         bnxt_dl_health_fw_status_update(bp, true);
13645                 }
13646                 rtnl_unlock();
13647                 break;
13648         }
13649         return;
13650
13651 fw_reset_abort_status:
13652         if (bp->fw_health->status_reliable ||
13653             (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
13654                 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
13655
13656                 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
13657         }
13658 fw_reset_abort:
13659         rtnl_lock();
13660         bnxt_fw_reset_abort(bp, rc);
13661         rtnl_unlock();
13662 }
13663
13664 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
13665 {
13666         int rc;
13667         struct bnxt *bp = netdev_priv(dev);
13668
13669         SET_NETDEV_DEV(dev, &pdev->dev);
13670
13671         /* enable device (incl. PCI PM wakeup), and bus-mastering */
13672         rc = pci_enable_device(pdev);
13673         if (rc) {
13674                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
13675                 goto init_err;
13676         }
13677
13678         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
13679                 dev_err(&pdev->dev,
13680                         "Cannot find PCI device base address, aborting\n");
13681                 rc = -ENODEV;
13682                 goto init_err_disable;
13683         }
13684
13685         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
13686         if (rc) {
13687                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
13688                 goto init_err_disable;
13689         }
13690
13691         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
13692             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
13693                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
13694                 rc = -EIO;
13695                 goto init_err_release;
13696         }
13697
13698         pci_set_master(pdev);
13699
13700         bp->dev = dev;
13701         bp->pdev = pdev;
13702
13703         /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
13704          * determines the BAR size.
13705          */
13706         bp->bar0 = pci_ioremap_bar(pdev, 0);
13707         if (!bp->bar0) {
13708                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
13709                 rc = -ENOMEM;
13710                 goto init_err_release;
13711         }
13712
13713         bp->bar2 = pci_ioremap_bar(pdev, 4);
13714         if (!bp->bar2) {
13715                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
13716                 rc = -ENOMEM;
13717                 goto init_err_release;
13718         }
13719
13720         INIT_WORK(&bp->sp_task, bnxt_sp_task);
13721         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
13722
13723         spin_lock_init(&bp->ntp_fltr_lock);
13724 #if BITS_PER_LONG == 32
13725         spin_lock_init(&bp->db_lock);
13726 #endif
13727
13728         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
13729         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
13730
13731         timer_setup(&bp->timer, bnxt_timer, 0);
13732         bp->current_interval = BNXT_TIMER_INTERVAL;
13733
13734         bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
13735         bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
13736
13737         clear_bit(BNXT_STATE_OPEN, &bp->state);
13738         return 0;
13739
13740 init_err_release:
13741         bnxt_unmap_bars(bp, pdev);
13742         pci_release_regions(pdev);
13743
13744 init_err_disable:
13745         pci_disable_device(pdev);
13746
13747 init_err:
13748         return rc;
13749 }
13750
13751 /* rtnl_lock held */
13752 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
13753 {
13754         struct sockaddr *addr = p;
13755         struct bnxt *bp = netdev_priv(dev);
13756         int rc = 0;
13757
13758         if (!is_valid_ether_addr(addr->sa_data))
13759                 return -EADDRNOTAVAIL;
13760
13761         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
13762                 return 0;
13763
13764         rc = bnxt_approve_mac(bp, addr->sa_data, true);
13765         if (rc)
13766                 return rc;
13767
13768         eth_hw_addr_set(dev, addr->sa_data);
13769         if (netif_running(dev)) {
13770                 bnxt_close_nic(bp, false, false);
13771                 rc = bnxt_open_nic(bp, false, false);
13772         }
13773
13774         return rc;
13775 }
13776
13777 /* rtnl_lock held */
13778 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
13779 {
13780         struct bnxt *bp = netdev_priv(dev);
13781
13782         if (netif_running(dev))
13783                 bnxt_close_nic(bp, true, false);
13784
13785         dev->mtu = new_mtu;
13786         bnxt_set_ring_params(bp);
13787
13788         if (netif_running(dev))
13789                 return bnxt_open_nic(bp, true, false);
13790
13791         return 0;
13792 }
13793
13794 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
13795 {
13796         struct bnxt *bp = netdev_priv(dev);
13797         bool sh = false;
13798         int rc, tx_cp;
13799
13800         if (tc > bp->max_tc) {
13801                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
13802                            tc, bp->max_tc);
13803                 return -EINVAL;
13804         }
13805
13806         if (bp->num_tc == tc)
13807                 return 0;
13808
13809         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
13810                 sh = true;
13811
13812         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
13813                               sh, tc, bp->tx_nr_rings_xdp);
13814         if (rc)
13815                 return rc;
13816
13817         /* Needs to close the device and do hw resource re-allocations */
13818         if (netif_running(bp->dev))
13819                 bnxt_close_nic(bp, true, false);
13820
13821         if (tc) {
13822                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
13823                 netdev_set_num_tc(dev, tc);
13824                 bp->num_tc = tc;
13825         } else {
13826                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13827                 netdev_reset_tc(dev);
13828                 bp->num_tc = 0;
13829         }
13830         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
13831         tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
13832         bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) :
13833                                tx_cp + bp->rx_nr_rings;
13834
13835         if (netif_running(bp->dev))
13836                 return bnxt_open_nic(bp, true, false);
13837
13838         return 0;
13839 }
13840
13841 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
13842                                   void *cb_priv)
13843 {
13844         struct bnxt *bp = cb_priv;
13845
13846         if (!bnxt_tc_flower_enabled(bp) ||
13847             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
13848                 return -EOPNOTSUPP;
13849
13850         switch (type) {
13851         case TC_SETUP_CLSFLOWER:
13852                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
13853         default:
13854                 return -EOPNOTSUPP;
13855         }
13856 }
13857
13858 LIST_HEAD(bnxt_block_cb_list);
13859
13860 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
13861                          void *type_data)
13862 {
13863         struct bnxt *bp = netdev_priv(dev);
13864
13865         switch (type) {
13866         case TC_SETUP_BLOCK:
13867                 return flow_block_cb_setup_simple(type_data,
13868                                                   &bnxt_block_cb_list,
13869                                                   bnxt_setup_tc_block_cb,
13870                                                   bp, bp, true);
13871         case TC_SETUP_QDISC_MQPRIO: {
13872                 struct tc_mqprio_qopt *mqprio = type_data;
13873
13874                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
13875
13876                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
13877         }
13878         default:
13879                 return -EOPNOTSUPP;
13880         }
13881 }
13882
13883 u32 bnxt_get_ntp_filter_idx(struct bnxt *bp, struct flow_keys *fkeys,
13884                             const struct sk_buff *skb)
13885 {
13886         struct bnxt_vnic_info *vnic;
13887
13888         if (skb)
13889                 return skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
13890
13891         vnic = &bp->vnic_info[0];
13892         return bnxt_toeplitz(bp, fkeys, (void *)vnic->rss_hash_key);
13893 }
13894
13895 int bnxt_insert_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr,
13896                            u32 idx)
13897 {
13898         struct hlist_head *head;
13899         int bit_id;
13900
13901         spin_lock_bh(&bp->ntp_fltr_lock);
13902         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, BNXT_MAX_FLTR, 0);
13903         if (bit_id < 0) {
13904                 spin_unlock_bh(&bp->ntp_fltr_lock);
13905                 return -ENOMEM;
13906         }
13907
13908         fltr->base.sw_id = (u16)bit_id;
13909         fltr->base.type = BNXT_FLTR_TYPE_NTUPLE;
13910         fltr->base.flags |= BNXT_ACT_RING_DST;
13911         head = &bp->ntp_fltr_hash_tbl[idx];
13912         hlist_add_head_rcu(&fltr->base.hash, head);
13913         set_bit(BNXT_FLTR_INSERTED, &fltr->base.state);
13914         bp->ntp_fltr_count++;
13915         spin_unlock_bh(&bp->ntp_fltr_lock);
13916         return 0;
13917 }
13918
13919 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
13920                             struct bnxt_ntuple_filter *f2)
13921 {
13922         struct flow_keys *keys1 = &f1->fkeys;
13923         struct flow_keys *keys2 = &f2->fkeys;
13924
13925         if (f1->ntuple_flags != f2->ntuple_flags)
13926                 return false;
13927
13928         if (keys1->basic.n_proto != keys2->basic.n_proto ||
13929             keys1->basic.ip_proto != keys2->basic.ip_proto)
13930                 return false;
13931
13932         if (keys1->basic.n_proto == htons(ETH_P_IP)) {
13933                 if (((f1->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_IP) &&
13934                      keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src) ||
13935                     ((f1->ntuple_flags & BNXT_NTUPLE_MATCH_DST_IP) &&
13936                      keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst))
13937                         return false;
13938         } else {
13939                 if (((f1->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_IP) &&
13940                      memcmp(&keys1->addrs.v6addrs.src,
13941                             &keys2->addrs.v6addrs.src,
13942                             sizeof(keys1->addrs.v6addrs.src))) ||
13943                     ((f1->ntuple_flags & BNXT_NTUPLE_MATCH_DST_IP) &&
13944                      memcmp(&keys1->addrs.v6addrs.dst,
13945                             &keys2->addrs.v6addrs.dst,
13946                             sizeof(keys1->addrs.v6addrs.dst))))
13947                         return false;
13948         }
13949
13950         if (((f1->ntuple_flags & BNXT_NTUPLE_MATCH_SRC_PORT) &&
13951              keys1->ports.src != keys2->ports.src) ||
13952             ((f1->ntuple_flags & BNXT_NTUPLE_MATCH_DST_PORT) &&
13953              keys1->ports.dst != keys2->ports.dst))
13954                 return false;
13955
13956         if (keys1->control.flags == keys2->control.flags &&
13957             f1->l2_fltr == f2->l2_fltr)
13958                 return true;
13959
13960         return false;
13961 }
13962
13963 struct bnxt_ntuple_filter *
13964 bnxt_lookup_ntp_filter_from_idx(struct bnxt *bp,
13965                                 struct bnxt_ntuple_filter *fltr, u32 idx)
13966 {
13967         struct bnxt_ntuple_filter *f;
13968         struct hlist_head *head;
13969
13970         head = &bp->ntp_fltr_hash_tbl[idx];
13971         hlist_for_each_entry_rcu(f, head, base.hash) {
13972                 if (bnxt_fltr_match(f, fltr))
13973                         return f;
13974         }
13975         return NULL;
13976 }
13977
13978 #ifdef CONFIG_RFS_ACCEL
13979 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
13980                               u16 rxq_index, u32 flow_id)
13981 {
13982         struct bnxt *bp = netdev_priv(dev);
13983         struct bnxt_ntuple_filter *fltr, *new_fltr;
13984         struct flow_keys *fkeys;
13985         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
13986         struct bnxt_l2_filter *l2_fltr;
13987         int rc = 0, idx;
13988         u32 flags;
13989
13990         if (ether_addr_equal(dev->dev_addr, eth->h_dest)) {
13991                 l2_fltr = bp->vnic_info[0].l2_filters[0];
13992                 atomic_inc(&l2_fltr->refcnt);
13993         } else {
13994                 struct bnxt_l2_key key;
13995
13996                 ether_addr_copy(key.dst_mac_addr, eth->h_dest);
13997                 key.vlan = 0;
13998                 l2_fltr = bnxt_lookup_l2_filter_from_key(bp, &key);
13999                 if (!l2_fltr)
14000                         return -EINVAL;
14001                 if (l2_fltr->base.flags & BNXT_ACT_FUNC_DST) {
14002                         bnxt_del_l2_filter(bp, l2_fltr);
14003                         return -EINVAL;
14004                 }
14005         }
14006         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
14007         if (!new_fltr) {
14008                 bnxt_del_l2_filter(bp, l2_fltr);
14009                 return -ENOMEM;
14010         }
14011
14012         fkeys = &new_fltr->fkeys;
14013         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
14014                 rc = -EPROTONOSUPPORT;
14015                 goto err_free;
14016         }
14017
14018         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
14019              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
14020             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
14021              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
14022                 rc = -EPROTONOSUPPORT;
14023                 goto err_free;
14024         }
14025         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
14026             bp->hwrm_spec_code < 0x10601) {
14027                 rc = -EPROTONOSUPPORT;
14028                 goto err_free;
14029         }
14030         flags = fkeys->control.flags;
14031         if (((flags & FLOW_DIS_ENCAPSULATION) &&
14032              bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
14033                 rc = -EPROTONOSUPPORT;
14034                 goto err_free;
14035         }
14036
14037         new_fltr->l2_fltr = l2_fltr;
14038         new_fltr->ntuple_flags = BNXT_NTUPLE_MATCH_ALL;
14039
14040         idx = bnxt_get_ntp_filter_idx(bp, fkeys, skb);
14041         rcu_read_lock();
14042         fltr = bnxt_lookup_ntp_filter_from_idx(bp, new_fltr, idx);
14043         if (fltr) {
14044                 rc = fltr->base.sw_id;
14045                 rcu_read_unlock();
14046                 goto err_free;
14047         }
14048         rcu_read_unlock();
14049
14050         new_fltr->flow_id = flow_id;
14051         new_fltr->base.rxq = rxq_index;
14052         rc = bnxt_insert_ntp_filter(bp, new_fltr, idx);
14053         if (!rc) {
14054                 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
14055                 return new_fltr->base.sw_id;
14056         }
14057
14058 err_free:
14059         bnxt_del_l2_filter(bp, l2_fltr);
14060         kfree(new_fltr);
14061         return rc;
14062 }
14063 #endif
14064
14065 void bnxt_del_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr)
14066 {
14067         spin_lock_bh(&bp->ntp_fltr_lock);
14068         if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) {
14069                 spin_unlock_bh(&bp->ntp_fltr_lock);
14070                 return;
14071         }
14072         hlist_del_rcu(&fltr->base.hash);
14073         bp->ntp_fltr_count--;
14074         spin_unlock_bh(&bp->ntp_fltr_lock);
14075         bnxt_del_l2_filter(bp, fltr->l2_fltr);
14076         clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
14077         kfree_rcu(fltr, base.rcu);
14078 }
14079
14080 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
14081 {
14082 #ifdef CONFIG_RFS_ACCEL
14083         int i;
14084
14085         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
14086                 struct hlist_head *head;
14087                 struct hlist_node *tmp;
14088                 struct bnxt_ntuple_filter *fltr;
14089                 int rc;
14090
14091                 head = &bp->ntp_fltr_hash_tbl[i];
14092                 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
14093                         bool del = false;
14094
14095                         if (test_bit(BNXT_FLTR_VALID, &fltr->base.state)) {
14096                                 if (fltr->base.flags & BNXT_ACT_NO_AGING)
14097                                         continue;
14098                                 if (rps_may_expire_flow(bp->dev, fltr->base.rxq,
14099                                                         fltr->flow_id,
14100                                                         fltr->base.sw_id)) {
14101                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
14102                                                                          fltr);
14103                                         del = true;
14104                                 }
14105                         } else {
14106                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
14107                                                                        fltr);
14108                                 if (rc)
14109                                         del = true;
14110                                 else
14111                                         set_bit(BNXT_FLTR_VALID, &fltr->base.state);
14112                         }
14113
14114                         if (del)
14115                                 bnxt_del_ntp_filter(bp, fltr);
14116                 }
14117         }
14118 #endif
14119 }
14120
14121 static int bnxt_udp_tunnel_set_port(struct net_device *netdev, unsigned int table,
14122                                     unsigned int entry, struct udp_tunnel_info *ti)
14123 {
14124         struct bnxt *bp = netdev_priv(netdev);
14125         unsigned int cmd;
14126
14127         if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
14128                 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
14129         else if (ti->type == UDP_TUNNEL_TYPE_GENEVE)
14130                 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE;
14131         else
14132                 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE;
14133
14134         return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti->port, cmd);
14135 }
14136
14137 static int bnxt_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table,
14138                                       unsigned int entry, struct udp_tunnel_info *ti)
14139 {
14140         struct bnxt *bp = netdev_priv(netdev);
14141         unsigned int cmd;
14142
14143         if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
14144                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
14145         else if (ti->type == UDP_TUNNEL_TYPE_GENEVE)
14146                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
14147         else
14148                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE;
14149
14150         return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
14151 }
14152
14153 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
14154         .set_port       = bnxt_udp_tunnel_set_port,
14155         .unset_port     = bnxt_udp_tunnel_unset_port,
14156         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
14157                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
14158         .tables         = {
14159                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
14160                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
14161         },
14162 }, bnxt_udp_tunnels_p7 = {
14163         .set_port       = bnxt_udp_tunnel_set_port,
14164         .unset_port     = bnxt_udp_tunnel_unset_port,
14165         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
14166                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
14167         .tables         = {
14168                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
14169                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
14170                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN_GPE, },
14171         },
14172 };
14173
14174 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
14175                                struct net_device *dev, u32 filter_mask,
14176                                int nlflags)
14177 {
14178         struct bnxt *bp = netdev_priv(dev);
14179
14180         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
14181                                        nlflags, filter_mask, NULL);
14182 }
14183
14184 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
14185                                u16 flags, struct netlink_ext_ack *extack)
14186 {
14187         struct bnxt *bp = netdev_priv(dev);
14188         struct nlattr *attr, *br_spec;
14189         int rem, rc = 0;
14190
14191         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
14192                 return -EOPNOTSUPP;
14193
14194         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
14195         if (!br_spec)
14196                 return -EINVAL;
14197
14198         nla_for_each_nested(attr, br_spec, rem) {
14199                 u16 mode;
14200
14201                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
14202                         continue;
14203
14204                 mode = nla_get_u16(attr);
14205                 if (mode == bp->br_mode)
14206                         break;
14207
14208                 rc = bnxt_hwrm_set_br_mode(bp, mode);
14209                 if (!rc)
14210                         bp->br_mode = mode;
14211                 break;
14212         }
14213         return rc;
14214 }
14215
14216 int bnxt_get_port_parent_id(struct net_device *dev,
14217                             struct netdev_phys_item_id *ppid)
14218 {
14219         struct bnxt *bp = netdev_priv(dev);
14220
14221         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
14222                 return -EOPNOTSUPP;
14223
14224         /* The PF and it's VF-reps only support the switchdev framework */
14225         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
14226                 return -EOPNOTSUPP;
14227
14228         ppid->id_len = sizeof(bp->dsn);
14229         memcpy(ppid->id, bp->dsn, ppid->id_len);
14230
14231         return 0;
14232 }
14233
14234 static const struct net_device_ops bnxt_netdev_ops = {
14235         .ndo_open               = bnxt_open,
14236         .ndo_start_xmit         = bnxt_start_xmit,
14237         .ndo_stop               = bnxt_close,
14238         .ndo_get_stats64        = bnxt_get_stats64,
14239         .ndo_set_rx_mode        = bnxt_set_rx_mode,
14240         .ndo_eth_ioctl          = bnxt_ioctl,
14241         .ndo_validate_addr      = eth_validate_addr,
14242         .ndo_set_mac_address    = bnxt_change_mac_addr,
14243         .ndo_change_mtu         = bnxt_change_mtu,
14244         .ndo_fix_features       = bnxt_fix_features,
14245         .ndo_set_features       = bnxt_set_features,
14246         .ndo_features_check     = bnxt_features_check,
14247         .ndo_tx_timeout         = bnxt_tx_timeout,
14248 #ifdef CONFIG_BNXT_SRIOV
14249         .ndo_get_vf_config      = bnxt_get_vf_config,
14250         .ndo_set_vf_mac         = bnxt_set_vf_mac,
14251         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
14252         .ndo_set_vf_rate        = bnxt_set_vf_bw,
14253         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
14254         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
14255         .ndo_set_vf_trust       = bnxt_set_vf_trust,
14256 #endif
14257         .ndo_setup_tc           = bnxt_setup_tc,
14258 #ifdef CONFIG_RFS_ACCEL
14259         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
14260 #endif
14261         .ndo_bpf                = bnxt_xdp,
14262         .ndo_xdp_xmit           = bnxt_xdp_xmit,
14263         .ndo_bridge_getlink     = bnxt_bridge_getlink,
14264         .ndo_bridge_setlink     = bnxt_bridge_setlink,
14265 };
14266
14267 static void bnxt_remove_one(struct pci_dev *pdev)
14268 {
14269         struct net_device *dev = pci_get_drvdata(pdev);
14270         struct bnxt *bp = netdev_priv(dev);
14271
14272         if (BNXT_PF(bp))
14273                 bnxt_sriov_disable(bp);
14274
14275         bnxt_rdma_aux_device_uninit(bp);
14276
14277         bnxt_ptp_clear(bp);
14278         unregister_netdev(dev);
14279         bnxt_free_l2_filters(bp, true);
14280         bnxt_free_ntp_fltrs(bp, true);
14281         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14282         /* Flush any pending tasks */
14283         cancel_work_sync(&bp->sp_task);
14284         cancel_delayed_work_sync(&bp->fw_reset_task);
14285         bp->sp_event = 0;
14286
14287         bnxt_dl_fw_reporters_destroy(bp);
14288         bnxt_dl_unregister(bp);
14289         bnxt_shutdown_tc(bp);
14290
14291         bnxt_clear_int_mode(bp);
14292         bnxt_hwrm_func_drv_unrgtr(bp);
14293         bnxt_free_hwrm_resources(bp);
14294         bnxt_hwmon_uninit(bp);
14295         bnxt_ethtool_free(bp);
14296         bnxt_dcb_free(bp);
14297         kfree(bp->ptp_cfg);
14298         bp->ptp_cfg = NULL;
14299         kfree(bp->fw_health);
14300         bp->fw_health = NULL;
14301         bnxt_cleanup_pci(bp);
14302         bnxt_free_ctx_mem(bp);
14303         kfree(bp->rss_indir_tbl);
14304         bp->rss_indir_tbl = NULL;
14305         bnxt_free_port_stats(bp);
14306         free_netdev(dev);
14307 }
14308
14309 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
14310 {
14311         int rc = 0;
14312         struct bnxt_link_info *link_info = &bp->link_info;
14313
14314         bp->phy_flags = 0;
14315         rc = bnxt_hwrm_phy_qcaps(bp);
14316         if (rc) {
14317                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
14318                            rc);
14319                 return rc;
14320         }
14321         if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
14322                 bp->dev->priv_flags |= IFF_SUPP_NOFCS;
14323         else
14324                 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
14325         if (!fw_dflt)
14326                 return 0;
14327
14328         mutex_lock(&bp->link_lock);
14329         rc = bnxt_update_link(bp, false);
14330         if (rc) {
14331                 mutex_unlock(&bp->link_lock);
14332                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
14333                            rc);
14334                 return rc;
14335         }
14336
14337         /* Older firmware does not have supported_auto_speeds, so assume
14338          * that all supported speeds can be autonegotiated.
14339          */
14340         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
14341                 link_info->support_auto_speeds = link_info->support_speeds;
14342
14343         bnxt_init_ethtool_link_settings(bp);
14344         mutex_unlock(&bp->link_lock);
14345         return 0;
14346 }
14347
14348 static int bnxt_get_max_irq(struct pci_dev *pdev)
14349 {
14350         u16 ctrl;
14351
14352         if (!pdev->msix_cap)
14353                 return 1;
14354
14355         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
14356         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
14357 }
14358
14359 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
14360                                 int *max_cp)
14361 {
14362         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
14363         int max_ring_grps = 0, max_irq;
14364
14365         *max_tx = hw_resc->max_tx_rings;
14366         *max_rx = hw_resc->max_rx_rings;
14367         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
14368         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
14369                         bnxt_get_ulp_msix_num(bp),
14370                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
14371         if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
14372                 *max_cp = min_t(int, *max_cp, max_irq);
14373         max_ring_grps = hw_resc->max_hw_ring_grps;
14374         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
14375                 *max_cp -= 1;
14376                 *max_rx -= 2;
14377         }
14378         if (bp->flags & BNXT_FLAG_AGG_RINGS)
14379                 *max_rx >>= 1;
14380         if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
14381                 int rc;
14382
14383                 rc = __bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
14384                 if (rc) {
14385                         *max_rx = 0;
14386                         *max_tx = 0;
14387                 }
14388                 /* On P5 chips, max_cp output param should be available NQs */
14389                 *max_cp = max_irq;
14390         }
14391         *max_rx = min_t(int, *max_rx, max_ring_grps);
14392 }
14393
14394 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
14395 {
14396         int rx, tx, cp;
14397
14398         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
14399         *max_rx = rx;
14400         *max_tx = tx;
14401         if (!rx || !tx || !cp)
14402                 return -ENOMEM;
14403
14404         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
14405 }
14406
14407 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
14408                                bool shared)
14409 {
14410         int rc;
14411
14412         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
14413         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
14414                 /* Not enough rings, try disabling agg rings. */
14415                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
14416                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
14417                 if (rc) {
14418                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
14419                         bp->flags |= BNXT_FLAG_AGG_RINGS;
14420                         return rc;
14421                 }
14422                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
14423                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
14424                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
14425                 bnxt_set_ring_params(bp);
14426         }
14427
14428         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
14429                 int max_cp, max_stat, max_irq;
14430
14431                 /* Reserve minimum resources for RoCE */
14432                 max_cp = bnxt_get_max_func_cp_rings(bp);
14433                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
14434                 max_irq = bnxt_get_max_func_irqs(bp);
14435                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
14436                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
14437                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
14438                         return 0;
14439
14440                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
14441                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
14442                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
14443                 max_cp = min_t(int, max_cp, max_irq);
14444                 max_cp = min_t(int, max_cp, max_stat);
14445                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
14446                 if (rc)
14447                         rc = 0;
14448         }
14449         return rc;
14450 }
14451
14452 /* In initial default shared ring setting, each shared ring must have a
14453  * RX/TX ring pair.
14454  */
14455 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
14456 {
14457         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
14458         bp->rx_nr_rings = bp->cp_nr_rings;
14459         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
14460         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
14461 }
14462
14463 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
14464 {
14465         int dflt_rings, max_rx_rings, max_tx_rings, rc;
14466
14467         if (!bnxt_can_reserve_rings(bp))
14468                 return 0;
14469
14470         if (sh)
14471                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
14472         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
14473         /* Reduce default rings on multi-port cards so that total default
14474          * rings do not exceed CPU count.
14475          */
14476         if (bp->port_count > 1) {
14477                 int max_rings =
14478                         max_t(int, num_online_cpus() / bp->port_count, 1);
14479
14480                 dflt_rings = min_t(int, dflt_rings, max_rings);
14481         }
14482         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
14483         if (rc)
14484                 return rc;
14485         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
14486         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
14487         if (sh)
14488                 bnxt_trim_dflt_sh_rings(bp);
14489         else
14490                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
14491         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
14492
14493         rc = __bnxt_reserve_rings(bp);
14494         if (rc && rc != -ENODEV)
14495                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
14496         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
14497         if (sh)
14498                 bnxt_trim_dflt_sh_rings(bp);
14499
14500         /* Rings may have been trimmed, re-reserve the trimmed rings. */
14501         if (bnxt_need_reserve_rings(bp)) {
14502                 rc = __bnxt_reserve_rings(bp);
14503                 if (rc && rc != -ENODEV)
14504                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
14505                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
14506         }
14507         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
14508                 bp->rx_nr_rings++;
14509                 bp->cp_nr_rings++;
14510         }
14511         if (rc) {
14512                 bp->tx_nr_rings = 0;
14513                 bp->rx_nr_rings = 0;
14514         }
14515         return rc;
14516 }
14517
14518 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
14519 {
14520         int rc;
14521
14522         if (bp->tx_nr_rings)
14523                 return 0;
14524
14525         bnxt_ulp_irq_stop(bp);
14526         bnxt_clear_int_mode(bp);
14527         rc = bnxt_set_dflt_rings(bp, true);
14528         if (rc) {
14529                 if (BNXT_VF(bp) && rc == -ENODEV)
14530                         netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
14531                 else
14532                         netdev_err(bp->dev, "Not enough rings available.\n");
14533                 goto init_dflt_ring_err;
14534         }
14535         rc = bnxt_init_int_mode(bp);
14536         if (rc)
14537                 goto init_dflt_ring_err;
14538
14539         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
14540
14541         bnxt_set_dflt_rfs(bp);
14542
14543 init_dflt_ring_err:
14544         bnxt_ulp_irq_restart(bp, rc);
14545         return rc;
14546 }
14547
14548 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
14549 {
14550         int rc;
14551
14552         ASSERT_RTNL();
14553         bnxt_hwrm_func_qcaps(bp);
14554
14555         if (netif_running(bp->dev))
14556                 __bnxt_close_nic(bp, true, false);
14557
14558         bnxt_ulp_irq_stop(bp);
14559         bnxt_clear_int_mode(bp);
14560         rc = bnxt_init_int_mode(bp);
14561         bnxt_ulp_irq_restart(bp, rc);
14562
14563         if (netif_running(bp->dev)) {
14564                 if (rc)
14565                         dev_close(bp->dev);
14566                 else
14567                         rc = bnxt_open_nic(bp, true, false);
14568         }
14569
14570         return rc;
14571 }
14572
14573 static int bnxt_init_mac_addr(struct bnxt *bp)
14574 {
14575         int rc = 0;
14576
14577         if (BNXT_PF(bp)) {
14578                 eth_hw_addr_set(bp->dev, bp->pf.mac_addr);
14579         } else {
14580 #ifdef CONFIG_BNXT_SRIOV
14581                 struct bnxt_vf_info *vf = &bp->vf;
14582                 bool strict_approval = true;
14583
14584                 if (is_valid_ether_addr(vf->mac_addr)) {
14585                         /* overwrite netdev dev_addr with admin VF MAC */
14586                         eth_hw_addr_set(bp->dev, vf->mac_addr);
14587                         /* Older PF driver or firmware may not approve this
14588                          * correctly.
14589                          */
14590                         strict_approval = false;
14591                 } else {
14592                         eth_hw_addr_random(bp->dev);
14593                 }
14594                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
14595 #endif
14596         }
14597         return rc;
14598 }
14599
14600 static void bnxt_vpd_read_info(struct bnxt *bp)
14601 {
14602         struct pci_dev *pdev = bp->pdev;
14603         unsigned int vpd_size, kw_len;
14604         int pos, size;
14605         u8 *vpd_data;
14606
14607         vpd_data = pci_vpd_alloc(pdev, &vpd_size);
14608         if (IS_ERR(vpd_data)) {
14609                 pci_warn(pdev, "Unable to read VPD\n");
14610                 return;
14611         }
14612
14613         pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
14614                                            PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
14615         if (pos < 0)
14616                 goto read_sn;
14617
14618         size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
14619         memcpy(bp->board_partno, &vpd_data[pos], size);
14620
14621 read_sn:
14622         pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
14623                                            PCI_VPD_RO_KEYWORD_SERIALNO,
14624                                            &kw_len);
14625         if (pos < 0)
14626                 goto exit;
14627
14628         size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
14629         memcpy(bp->board_serialno, &vpd_data[pos], size);
14630 exit:
14631         kfree(vpd_data);
14632 }
14633
14634 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
14635 {
14636         struct pci_dev *pdev = bp->pdev;
14637         u64 qword;
14638
14639         qword = pci_get_dsn(pdev);
14640         if (!qword) {
14641                 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
14642                 return -EOPNOTSUPP;
14643         }
14644
14645         put_unaligned_le64(qword, dsn);
14646
14647         bp->flags |= BNXT_FLAG_DSN_VALID;
14648         return 0;
14649 }
14650
14651 static int bnxt_map_db_bar(struct bnxt *bp)
14652 {
14653         if (!bp->db_size)
14654                 return -ENODEV;
14655         bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
14656         if (!bp->bar1)
14657                 return -ENOMEM;
14658         return 0;
14659 }
14660
14661 void bnxt_print_device_info(struct bnxt *bp)
14662 {
14663         netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n",
14664                     board_info[bp->board_idx].name,
14665                     (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr);
14666
14667         pcie_print_link_status(bp->pdev);
14668 }
14669
14670 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
14671 {
14672         struct net_device *dev;
14673         struct bnxt *bp;
14674         int rc, max_irqs;
14675
14676         if (pci_is_bridge(pdev))
14677                 return -ENODEV;
14678
14679         /* Clear any pending DMA transactions from crash kernel
14680          * while loading driver in capture kernel.
14681          */
14682         if (is_kdump_kernel()) {
14683                 pci_clear_master(pdev);
14684                 pcie_flr(pdev);
14685         }
14686
14687         max_irqs = bnxt_get_max_irq(pdev);
14688         dev = alloc_etherdev_mqs(sizeof(*bp), max_irqs * BNXT_MAX_QUEUE,
14689                                  max_irqs);
14690         if (!dev)
14691                 return -ENOMEM;
14692
14693         bp = netdev_priv(dev);
14694         bp->board_idx = ent->driver_data;
14695         bp->msg_enable = BNXT_DEF_MSG_ENABLE;
14696         bnxt_set_max_func_irqs(bp, max_irqs);
14697
14698         if (bnxt_vf_pciid(bp->board_idx))
14699                 bp->flags |= BNXT_FLAG_VF;
14700
14701         /* No devlink port registration in case of a VF */
14702         if (BNXT_PF(bp))
14703                 SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port);
14704
14705         if (pdev->msix_cap)
14706                 bp->flags |= BNXT_FLAG_MSIX_CAP;
14707
14708         rc = bnxt_init_board(pdev, dev);
14709         if (rc < 0)
14710                 goto init_err_free;
14711
14712         dev->netdev_ops = &bnxt_netdev_ops;
14713         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
14714         dev->ethtool_ops = &bnxt_ethtool_ops;
14715         pci_set_drvdata(pdev, dev);
14716
14717         rc = bnxt_alloc_hwrm_resources(bp);
14718         if (rc)
14719                 goto init_err_pci_clean;
14720
14721         mutex_init(&bp->hwrm_cmd_lock);
14722         mutex_init(&bp->link_lock);
14723
14724         rc = bnxt_fw_init_one_p1(bp);
14725         if (rc)
14726                 goto init_err_pci_clean;
14727
14728         if (BNXT_PF(bp))
14729                 bnxt_vpd_read_info(bp);
14730
14731         if (BNXT_CHIP_P5_PLUS(bp)) {
14732                 bp->flags |= BNXT_FLAG_CHIP_P5_PLUS;
14733                 if (BNXT_CHIP_P7(bp))
14734                         bp->flags |= BNXT_FLAG_CHIP_P7;
14735         }
14736
14737         rc = bnxt_alloc_rss_indir_tbl(bp);
14738         if (rc)
14739                 goto init_err_pci_clean;
14740
14741         rc = bnxt_fw_init_one_p2(bp);
14742         if (rc)
14743                 goto init_err_pci_clean;
14744
14745         rc = bnxt_map_db_bar(bp);
14746         if (rc) {
14747                 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
14748                         rc);
14749                 goto init_err_pci_clean;
14750         }
14751
14752         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
14753                            NETIF_F_TSO | NETIF_F_TSO6 |
14754                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
14755                            NETIF_F_GSO_IPXIP4 |
14756                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
14757                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
14758                            NETIF_F_RXCSUM | NETIF_F_GRO;
14759         if (bp->flags & BNXT_FLAG_UDP_GSO_CAP)
14760                 dev->hw_features |= NETIF_F_GSO_UDP_L4;
14761
14762         if (BNXT_SUPPORTS_TPA(bp))
14763                 dev->hw_features |= NETIF_F_LRO;
14764
14765         dev->hw_enc_features =
14766                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
14767                         NETIF_F_TSO | NETIF_F_TSO6 |
14768                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
14769                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
14770                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
14771         if (bp->flags & BNXT_FLAG_UDP_GSO_CAP)
14772                 dev->hw_enc_features |= NETIF_F_GSO_UDP_L4;
14773         if (bp->flags & BNXT_FLAG_CHIP_P7)
14774                 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels_p7;
14775         else
14776                 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
14777
14778         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
14779                                     NETIF_F_GSO_GRE_CSUM;
14780         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
14781         if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
14782                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
14783         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
14784                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
14785         if (BNXT_SUPPORTS_TPA(bp))
14786                 dev->hw_features |= NETIF_F_GRO_HW;
14787         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
14788         if (dev->features & NETIF_F_GRO_HW)
14789                 dev->features &= ~NETIF_F_LRO;
14790         dev->priv_flags |= IFF_UNICAST_FLT;
14791
14792         netif_set_tso_max_size(dev, GSO_MAX_SIZE);
14793
14794         dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
14795                             NETDEV_XDP_ACT_RX_SG;
14796
14797 #ifdef CONFIG_BNXT_SRIOV
14798         init_waitqueue_head(&bp->sriov_cfg_wait);
14799 #endif
14800         if (BNXT_SUPPORTS_TPA(bp)) {
14801                 bp->gro_func = bnxt_gro_func_5730x;
14802                 if (BNXT_CHIP_P4(bp))
14803                         bp->gro_func = bnxt_gro_func_5731x;
14804                 else if (BNXT_CHIP_P5_PLUS(bp))
14805                         bp->gro_func = bnxt_gro_func_5750x;
14806         }
14807         if (!BNXT_CHIP_P4_PLUS(bp))
14808                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
14809
14810         rc = bnxt_init_mac_addr(bp);
14811         if (rc) {
14812                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
14813                 rc = -EADDRNOTAVAIL;
14814                 goto init_err_pci_clean;
14815         }
14816
14817         if (BNXT_PF(bp)) {
14818                 /* Read the adapter's DSN to use as the eswitch switch_id */
14819                 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
14820         }
14821
14822         /* MTU range: 60 - FW defined max */
14823         dev->min_mtu = ETH_ZLEN;
14824         dev->max_mtu = bp->max_mtu;
14825
14826         rc = bnxt_probe_phy(bp, true);
14827         if (rc)
14828                 goto init_err_pci_clean;
14829
14830         bnxt_init_l2_fltr_tbl(bp);
14831         bnxt_set_rx_skb_mode(bp, false);
14832         bnxt_set_tpa_flags(bp);
14833         bnxt_set_ring_params(bp);
14834         rc = bnxt_set_dflt_rings(bp, true);
14835         if (rc) {
14836                 if (BNXT_VF(bp) && rc == -ENODEV) {
14837                         netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
14838                 } else {
14839                         netdev_err(bp->dev, "Not enough rings available.\n");
14840                         rc = -ENOMEM;
14841                 }
14842                 goto init_err_pci_clean;
14843         }
14844
14845         bnxt_fw_init_one_p3(bp);
14846
14847         bnxt_init_dflt_coal(bp);
14848
14849         if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
14850                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
14851
14852         rc = bnxt_init_int_mode(bp);
14853         if (rc)
14854                 goto init_err_pci_clean;
14855
14856         /* No TC has been set yet and rings may have been trimmed due to
14857          * limited MSIX, so we re-initialize the TX rings per TC.
14858          */
14859         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
14860
14861         if (BNXT_PF(bp)) {
14862                 if (!bnxt_pf_wq) {
14863                         bnxt_pf_wq =
14864                                 create_singlethread_workqueue("bnxt_pf_wq");
14865                         if (!bnxt_pf_wq) {
14866                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
14867                                 rc = -ENOMEM;
14868                                 goto init_err_pci_clean;
14869                         }
14870                 }
14871                 rc = bnxt_init_tc(bp);
14872                 if (rc)
14873                         netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
14874                                    rc);
14875         }
14876
14877         bnxt_inv_fw_health_reg(bp);
14878         rc = bnxt_dl_register(bp);
14879         if (rc)
14880                 goto init_err_dl;
14881
14882         rc = register_netdev(dev);
14883         if (rc)
14884                 goto init_err_cleanup;
14885
14886         bnxt_dl_fw_reporters_create(bp);
14887
14888         bnxt_rdma_aux_device_init(bp);
14889
14890         bnxt_print_device_info(bp);
14891
14892         pci_save_state(pdev);
14893
14894         return 0;
14895 init_err_cleanup:
14896         bnxt_dl_unregister(bp);
14897 init_err_dl:
14898         bnxt_shutdown_tc(bp);
14899         bnxt_clear_int_mode(bp);
14900
14901 init_err_pci_clean:
14902         bnxt_hwrm_func_drv_unrgtr(bp);
14903         bnxt_free_hwrm_resources(bp);
14904         bnxt_hwmon_uninit(bp);
14905         bnxt_ethtool_free(bp);
14906         bnxt_ptp_clear(bp);
14907         kfree(bp->ptp_cfg);
14908         bp->ptp_cfg = NULL;
14909         kfree(bp->fw_health);
14910         bp->fw_health = NULL;
14911         bnxt_cleanup_pci(bp);
14912         bnxt_free_ctx_mem(bp);
14913         kfree(bp->rss_indir_tbl);
14914         bp->rss_indir_tbl = NULL;
14915
14916 init_err_free:
14917         free_netdev(dev);
14918         return rc;
14919 }
14920
14921 static void bnxt_shutdown(struct pci_dev *pdev)
14922 {
14923         struct net_device *dev = pci_get_drvdata(pdev);
14924         struct bnxt *bp;
14925
14926         if (!dev)
14927                 return;
14928
14929         rtnl_lock();
14930         bp = netdev_priv(dev);
14931         if (!bp)
14932                 goto shutdown_exit;
14933
14934         if (netif_running(dev))
14935                 dev_close(dev);
14936
14937         bnxt_clear_int_mode(bp);
14938         pci_disable_device(pdev);
14939
14940         if (system_state == SYSTEM_POWER_OFF) {
14941                 pci_wake_from_d3(pdev, bp->wol);
14942                 pci_set_power_state(pdev, PCI_D3hot);
14943         }
14944
14945 shutdown_exit:
14946         rtnl_unlock();
14947 }
14948
14949 #ifdef CONFIG_PM_SLEEP
14950 static int bnxt_suspend(struct device *device)
14951 {
14952         struct net_device *dev = dev_get_drvdata(device);
14953         struct bnxt *bp = netdev_priv(dev);
14954         int rc = 0;
14955
14956         rtnl_lock();
14957         bnxt_ulp_stop(bp);
14958         if (netif_running(dev)) {
14959                 netif_device_detach(dev);
14960                 rc = bnxt_close(dev);
14961         }
14962         bnxt_hwrm_func_drv_unrgtr(bp);
14963         pci_disable_device(bp->pdev);
14964         bnxt_free_ctx_mem(bp);
14965         rtnl_unlock();
14966         return rc;
14967 }
14968
14969 static int bnxt_resume(struct device *device)
14970 {
14971         struct net_device *dev = dev_get_drvdata(device);
14972         struct bnxt *bp = netdev_priv(dev);
14973         int rc = 0;
14974
14975         rtnl_lock();
14976         rc = pci_enable_device(bp->pdev);
14977         if (rc) {
14978                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
14979                            rc);
14980                 goto resume_exit;
14981         }
14982         pci_set_master(bp->pdev);
14983         if (bnxt_hwrm_ver_get(bp)) {
14984                 rc = -ENODEV;
14985                 goto resume_exit;
14986         }
14987         rc = bnxt_hwrm_func_reset(bp);
14988         if (rc) {
14989                 rc = -EBUSY;
14990                 goto resume_exit;
14991         }
14992
14993         rc = bnxt_hwrm_func_qcaps(bp);
14994         if (rc)
14995                 goto resume_exit;
14996
14997         bnxt_clear_reservations(bp, true);
14998
14999         if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
15000                 rc = -ENODEV;
15001                 goto resume_exit;
15002         }
15003
15004         bnxt_get_wol_settings(bp);
15005         if (netif_running(dev)) {
15006                 rc = bnxt_open(dev);
15007                 if (!rc)
15008                         netif_device_attach(dev);
15009         }
15010
15011 resume_exit:
15012         bnxt_ulp_start(bp, rc);
15013         if (!rc)
15014                 bnxt_reenable_sriov(bp);
15015         rtnl_unlock();
15016         return rc;
15017 }
15018
15019 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
15020 #define BNXT_PM_OPS (&bnxt_pm_ops)
15021
15022 #else
15023
15024 #define BNXT_PM_OPS NULL
15025
15026 #endif /* CONFIG_PM_SLEEP */
15027
15028 /**
15029  * bnxt_io_error_detected - called when PCI error is detected
15030  * @pdev: Pointer to PCI device
15031  * @state: The current pci connection state
15032  *
15033  * This function is called after a PCI bus error affecting
15034  * this device has been detected.
15035  */
15036 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
15037                                                pci_channel_state_t state)
15038 {
15039         struct net_device *netdev = pci_get_drvdata(pdev);
15040         struct bnxt *bp = netdev_priv(netdev);
15041
15042         netdev_info(netdev, "PCI I/O error detected\n");
15043
15044         rtnl_lock();
15045         netif_device_detach(netdev);
15046
15047         bnxt_ulp_stop(bp);
15048
15049         if (state == pci_channel_io_perm_failure) {
15050                 rtnl_unlock();
15051                 return PCI_ERS_RESULT_DISCONNECT;
15052         }
15053
15054         if (state == pci_channel_io_frozen)
15055                 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
15056
15057         if (netif_running(netdev))
15058                 bnxt_close(netdev);
15059
15060         if (pci_is_enabled(pdev))
15061                 pci_disable_device(pdev);
15062         bnxt_free_ctx_mem(bp);
15063         rtnl_unlock();
15064
15065         /* Request a slot slot reset. */
15066         return PCI_ERS_RESULT_NEED_RESET;
15067 }
15068
15069 /**
15070  * bnxt_io_slot_reset - called after the pci bus has been reset.
15071  * @pdev: Pointer to PCI device
15072  *
15073  * Restart the card from scratch, as if from a cold-boot.
15074  * At this point, the card has exprienced a hard reset,
15075  * followed by fixups by BIOS, and has its config space
15076  * set up identically to what it was at cold boot.
15077  */
15078 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
15079 {
15080         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
15081         struct net_device *netdev = pci_get_drvdata(pdev);
15082         struct bnxt *bp = netdev_priv(netdev);
15083         int retry = 0;
15084         int err = 0;
15085         int off;
15086
15087         netdev_info(bp->dev, "PCI Slot Reset\n");
15088
15089         rtnl_lock();
15090
15091         if (pci_enable_device(pdev)) {
15092                 dev_err(&pdev->dev,
15093                         "Cannot re-enable PCI device after reset.\n");
15094         } else {
15095                 pci_set_master(pdev);
15096                 /* Upon fatal error, our device internal logic that latches to
15097                  * BAR value is getting reset and will restore only upon
15098                  * rewritting the BARs.
15099                  *
15100                  * As pci_restore_state() does not re-write the BARs if the
15101                  * value is same as saved value earlier, driver needs to
15102                  * write the BARs to 0 to force restore, in case of fatal error.
15103                  */
15104                 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
15105                                        &bp->state)) {
15106                         for (off = PCI_BASE_ADDRESS_0;
15107                              off <= PCI_BASE_ADDRESS_5; off += 4)
15108                                 pci_write_config_dword(bp->pdev, off, 0);
15109                 }
15110                 pci_restore_state(pdev);
15111                 pci_save_state(pdev);
15112
15113                 bnxt_inv_fw_health_reg(bp);
15114                 bnxt_try_map_fw_health_reg(bp);
15115
15116                 /* In some PCIe AER scenarios, firmware may take up to
15117                  * 10 seconds to become ready in the worst case.
15118                  */
15119                 do {
15120                         err = bnxt_try_recover_fw(bp);
15121                         if (!err)
15122                                 break;
15123                         retry++;
15124                 } while (retry < BNXT_FW_SLOT_RESET_RETRY);
15125
15126                 if (err) {
15127                         dev_err(&pdev->dev, "Firmware not ready\n");
15128                         goto reset_exit;
15129                 }
15130
15131                 err = bnxt_hwrm_func_reset(bp);
15132                 if (!err)
15133                         result = PCI_ERS_RESULT_RECOVERED;
15134
15135                 bnxt_ulp_irq_stop(bp);
15136                 bnxt_clear_int_mode(bp);
15137                 err = bnxt_init_int_mode(bp);
15138                 bnxt_ulp_irq_restart(bp, err);
15139         }
15140
15141 reset_exit:
15142         bnxt_clear_reservations(bp, true);
15143         rtnl_unlock();
15144
15145         return result;
15146 }
15147
15148 /**
15149  * bnxt_io_resume - called when traffic can start flowing again.
15150  * @pdev: Pointer to PCI device
15151  *
15152  * This callback is called when the error recovery driver tells
15153  * us that its OK to resume normal operation.
15154  */
15155 static void bnxt_io_resume(struct pci_dev *pdev)
15156 {
15157         struct net_device *netdev = pci_get_drvdata(pdev);
15158         struct bnxt *bp = netdev_priv(netdev);
15159         int err;
15160
15161         netdev_info(bp->dev, "PCI Slot Resume\n");
15162         rtnl_lock();
15163
15164         err = bnxt_hwrm_func_qcaps(bp);
15165         if (!err && netif_running(netdev))
15166                 err = bnxt_open(netdev);
15167
15168         bnxt_ulp_start(bp, err);
15169         if (!err) {
15170                 bnxt_reenable_sriov(bp);
15171                 netif_device_attach(netdev);
15172         }
15173
15174         rtnl_unlock();
15175 }
15176
15177 static const struct pci_error_handlers bnxt_err_handler = {
15178         .error_detected = bnxt_io_error_detected,
15179         .slot_reset     = bnxt_io_slot_reset,
15180         .resume         = bnxt_io_resume
15181 };
15182
15183 static struct pci_driver bnxt_pci_driver = {
15184         .name           = DRV_MODULE_NAME,
15185         .id_table       = bnxt_pci_tbl,
15186         .probe          = bnxt_init_one,
15187         .remove         = bnxt_remove_one,
15188         .shutdown       = bnxt_shutdown,
15189         .driver.pm      = BNXT_PM_OPS,
15190         .err_handler    = &bnxt_err_handler,
15191 #if defined(CONFIG_BNXT_SRIOV)
15192         .sriov_configure = bnxt_sriov_configure,
15193 #endif
15194 };
15195
15196 static int __init bnxt_init(void)
15197 {
15198         int err;
15199
15200         bnxt_debug_init();
15201         err = pci_register_driver(&bnxt_pci_driver);
15202         if (err) {
15203                 bnxt_debug_exit();
15204                 return err;
15205         }
15206
15207         return 0;
15208 }
15209
15210 static void __exit bnxt_exit(void)
15211 {
15212         pci_unregister_driver(&bnxt_pci_driver);
15213         if (bnxt_pf_wq)
15214                 destroy_workqueue(bnxt_pf_wq);
15215         bnxt_debug_exit();
15216 }
15217
15218 module_init(bnxt_init);
15219 module_exit(bnxt_exit);