drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2019 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/mdio.h>
  35 #include <linux/if.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/if_bridge.h>
  38 #include <linux/rtc.h>
  39 #include <linux/bpf.h>
  40 #include <net/ip.h>
  41 #include <net/tcp.h>
  42 #include <net/udp.h>
  43 #include <net/checksum.h>
  44 #include <net/ip6_checksum.h>
  45 #include <net/udp_tunnel.h>
  46 #include <linux/workqueue.h>
  47 #include <linux/prefetch.h>
  48 #include <linux/cache.h>
  49 #include <linux/log2.h>
  50 #include <linux/aer.h>
  51 #include <linux/bitmap.h>
  52 #include <linux/ptp_clock_kernel.h>
  53 #include <linux/timecounter.h>
  54 #include <linux/cpu_rmap.h>
  55 #include <linux/cpumask.h>
  56 #include <net/pkt_cls.h>
  57 #include <linux/hwmon.h>
  58 #include <linux/hwmon-sysfs.h>
  59 #include <net/page_pool.h>
  60
  61 #include "bnxt_hsi.h"
  62 #include "bnxt.h"
  63 #include "bnxt_ulp.h"
  64 #include "bnxt_sriov.h"
  65 #include "bnxt_ethtool.h"
  66 #include "bnxt_dcb.h"
  67 #include "bnxt_xdp.h"
  68 #include "bnxt_ptp.h"
  69 #include "bnxt_vfr.h"
  70 #include "bnxt_tc.h"
  71 #include "bnxt_devlink.h"
  72 #include "bnxt_debugfs.h"
  73
  74 #define BNXT_TX_TIMEOUT         (5 * HZ)
  75 #define BNXT_DEF_MSG_ENABLE     (NETIF_MSG_DRV | NETIF_MSG_HW)
  76
  77 MODULE_LICENSE("GPL");
  78 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  79
  80 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  81 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  82 #define BNXT_RX_COPY_THRESH 256
  83
  84 #define BNXT_TX_PUSH_THRESH 164
  85
  86 enum board_idx {
  87         BCM57301,
  88         BCM57302,
  89         BCM57304,
  90         BCM57417_NPAR,
  91         BCM58700,
  92         BCM57311,
  93         BCM57312,
  94         BCM57402,
  95         BCM57404,
  96         BCM57406,
  97         BCM57402_NPAR,
  98         BCM57407,
  99         BCM57412,
 100         BCM57414,
 101         BCM57416,
 102         BCM57417,
 103         BCM57412_NPAR,
 104         BCM57314,
 105         BCM57417_SFP,
 106         BCM57416_SFP,
 107         BCM57404_NPAR,
 108         BCM57406_NPAR,
 109         BCM57407_SFP,
 110         BCM57407_NPAR,
 111         BCM57414_NPAR,
 112         BCM57416_NPAR,
 113         BCM57452,
 114         BCM57454,
 115         BCM5745x_NPAR,
 116         BCM57508,
 117         BCM57504,
 118         BCM57502,
 119         BCM57508_NPAR,
 120         BCM57504_NPAR,
 121         BCM57502_NPAR,
 122         BCM58802,
 123         BCM58804,
 124         BCM58808,
 125         NETXTREME_E_VF,
 126         NETXTREME_C_VF,
 127         NETXTREME_S_VF,
 128         NETXTREME_C_VF_HV,
 129         NETXTREME_E_VF_HV,
 130         NETXTREME_E_P5_VF,
 131         NETXTREME_E_P5_VF_HV,
 132 };
 133
 134 /* indexed by enum above */
 135 static const struct {
 136         char *name;
 137 } board_info[] = {
 138         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 139         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 140         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 141         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 142         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 143         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 144         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 145         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 146         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 147         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 148         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 149         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 150         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 151         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 152         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 153         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 154         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 155         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 156         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 157         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 158         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 159         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 160         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 161         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 162         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 163         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 164         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 165         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 166         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 167         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 168         [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 169         [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
 170         [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
 171         [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
 172         [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
 173         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 174         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 175         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 176         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 177         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 178         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 179         [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
 180         [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
 181         [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
 182         [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
 183 };
 184
 185 static const struct pci_device_id bnxt_pci_tbl[] = {
 186         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 187         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 188         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 189         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 190         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 191         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 192         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 193         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 194         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 195         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 196         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 197         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 198         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 199         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 200         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 201         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 202         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 203         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 204         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 205         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 206         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 207         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 208         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 209         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 210         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 211         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 212         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 213         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 214         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 215         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 216         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 217         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 218         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 219         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 220         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 221         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 222         { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
 223         { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
 224         { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
 225         { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
 226         { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
 227         { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
 228         { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
 229         { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
 230         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 231         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 232 #ifdef CONFIG_BNXT_SRIOV
 233         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 234         { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
 235         { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
 236         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 237         { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
 238         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 239         { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
 240         { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
 241         { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
 242         { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
 243         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 244         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 245         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 246         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 247         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 248         { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
 249         { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
 250         { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
 251         { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
 252         { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
 253         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 254 #endif
 255         { 0 }
 256 };
 257
 258 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 259
 260 static const u16 bnxt_vf_req_snif[] = {
 261         HWRM_FUNC_CFG,
 262         HWRM_FUNC_VF_CFG,
 263         HWRM_PORT_PHY_QCFG,
 264         HWRM_CFA_L2_FILTER_ALLOC,
 265 };
 266
 267 static const u16 bnxt_async_events_arr[] = {
 268         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 269         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
 270         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 271         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 272         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 273         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 274         ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
 275         ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
 276         ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
 277         ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
 278         ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
 279         ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
 280 };
 281
 282 static struct workqueue_struct *bnxt_pf_wq;
 283
 284 static bool bnxt_vf_pciid(enum board_idx idx)
 285 {
 286         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 287                 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
 288                 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
 289                 idx == NETXTREME_E_P5_VF_HV);
 290 }
 291
 292 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 293 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 294 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 295
 296 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 297                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 298
 299 #define BNXT_DB_CQ(db, idx)                                             \
 300         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 301
 302 #define BNXT_DB_NQ_P5(db, idx)                                          \
 303         writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
 304
 305 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 306         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 307
 308 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 309         writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
 310
 311 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 312 {
 313         if (bp->flags & BNXT_FLAG_CHIP_P5)
 314                 BNXT_DB_NQ_P5(db, idx);
 315         else
 316                 BNXT_DB_CQ(db, idx);
 317 }
 318
 319 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 320 {
 321         if (bp->flags & BNXT_FLAG_CHIP_P5)
 322                 BNXT_DB_NQ_ARM_P5(db, idx);
 323         else
 324                 BNXT_DB_CQ_ARM(db, idx);
 325 }
 326
 327 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 328 {
 329         if (bp->flags & BNXT_FLAG_CHIP_P5)
 330                 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
 331                        db->doorbell);
 332         else
 333                 BNXT_DB_CQ(db, idx);
 334 }
 335
 336 const u16 bnxt_lhint_arr[] = {
 337         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 338         TX_BD_FLAGS_LHINT_512_TO_1023,
 339         TX_BD_FLAGS_LHINT_1024_TO_2047,
 340         TX_BD_FLAGS_LHINT_1024_TO_2047,
 341         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 342         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 343         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 344         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 345         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 346         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 347         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 348         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 349         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 350         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 351         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 352         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 353         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 354         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 355         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 356 };
 357
 358 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 359 {
 360         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 361
 362         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 363                 return 0;
 364
 365         return md_dst->u.port_info.port_id;
 366 }
 367
 368 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 369 {
 370         struct bnxt *bp = netdev_priv(dev);
 371         struct tx_bd *txbd;
 372         struct tx_bd_ext *txbd1;
 373         struct netdev_queue *txq;
 374         int i;
 375         dma_addr_t mapping;
 376         unsigned int length, pad = 0;
 377         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 378         u16 prod, last_frag;
 379         struct pci_dev *pdev = bp->pdev;
 380         struct bnxt_tx_ring_info *txr;
 381         struct bnxt_sw_tx_bd *tx_buf;
 382         __le32 lflags = 0;
 383
 384         i = skb_get_queue_mapping(skb);
 385         if (unlikely(i >= bp->tx_nr_rings)) {
 386                 dev_kfree_skb_any(skb);
 387                 return NETDEV_TX_OK;
 388         }
 389
 390         txq = netdev_get_tx_queue(dev, i);
 391         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 392         prod = txr->tx_prod;
 393
 394         free_size = bnxt_tx_avail(bp, txr);
 395         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 396                 netif_tx_stop_queue(txq);
 397                 return NETDEV_TX_BUSY;
 398         }
 399
 400         length = skb->len;
 401         len = skb_headlen(skb);
 402         last_frag = skb_shinfo(skb)->nr_frags;
 403
 404         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 405
 406         txbd->tx_bd_opaque = prod;
 407
 408         tx_buf = &txr->tx_buf_ring[prod];
 409         tx_buf->skb = skb;
 410         tx_buf->nr_frags = last_frag;
 411
 412         vlan_tag_flags = 0;
 413         cfa_action = bnxt_xmit_get_cfa_action(skb);
 414         if (skb_vlan_tag_present(skb)) {
 415                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 416                                  skb_vlan_tag_get(skb);
 417                 /* Currently supports 8021Q, 8021AD vlan offloads
 418                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 419                  */
 420                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 421                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 422         }
 423
 424         if (unlikely(skb->no_fcs)) {
 425                 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
 426                 goto normal_tx;
 427         }
 428
 429         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 430                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 431                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 432                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 433                 void __iomem *db = txr->tx_db.doorbell;
 434                 void *pdata = tx_push_buf->data;
 435                 u64 *end;
 436                 int j, push_len;
 437
 438                 /* Set COAL_NOW to be ready quickly for the next push */
 439                 tx_push->tx_bd_len_flags_type =
 440                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 441                                         TX_BD_TYPE_LONG_TX_BD |
 442                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 443                                         TX_BD_FLAGS_COAL_NOW |
 444                                         TX_BD_FLAGS_PACKET_END |
 445                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 446
 447                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 448                         tx_push1->tx_bd_hsize_lflags =
 449                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 450                 else
 451                         tx_push1->tx_bd_hsize_lflags = 0;
 452
 453                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 454                 tx_push1->tx_bd_cfa_action =
 455                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 456
 457                 end = pdata + length;
 458                 end = PTR_ALIGN(end, 8) - 1;
 459                 *end = 0;
 460
 461                 skb_copy_from_linear_data(skb, pdata, len);
 462                 pdata += len;
 463                 for (j = 0; j < last_frag; j++) {
 464                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 465                         void *fptr;
 466
 467                         fptr = skb_frag_address_safe(frag);
 468                         if (!fptr)
 469                                 goto normal_tx;
 470
 471                         memcpy(pdata, fptr, skb_frag_size(frag));
 472                         pdata += skb_frag_size(frag);
 473                 }
 474
 475                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 476                 txbd->tx_bd_haddr = txr->data_mapping;
 477                 prod = NEXT_TX(prod);
 478                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 479                 memcpy(txbd, tx_push1, sizeof(*txbd));
 480                 prod = NEXT_TX(prod);
 481                 tx_push->doorbell =
 482                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 483                 txr->tx_prod = prod;
 484
 485                 tx_buf->is_push = 1;
 486                 netdev_tx_sent_queue(txq, skb->len);
 487                 wmb();  /* Sync is_push and byte queue before pushing data */
 488
 489                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 490                 if (push_len > 16) {
 491                         __iowrite64_copy(db, tx_push_buf, 16);
 492                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 493                                          (push_len - 16) << 1);
 494                 } else {
 495                         __iowrite64_copy(db, tx_push_buf, push_len);
 496                 }
 497
 498                 goto tx_done;
 499         }
 500
 501 normal_tx:
 502         if (length < BNXT_MIN_PKT_SIZE) {
 503                 pad = BNXT_MIN_PKT_SIZE - length;
 504                 if (skb_pad(skb, pad)) {
 505                         /* SKB already freed. */
 506                         tx_buf->skb = NULL;
 507                         return NETDEV_TX_OK;
 508                 }
 509                 length = BNXT_MIN_PKT_SIZE;
 510         }
 511
 512         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 513
 514         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 515                 dev_kfree_skb_any(skb);
 516                 tx_buf->skb = NULL;
 517                 return NETDEV_TX_OK;
 518         }
 519
 520         dma_unmap_addr_set(tx_buf, mapping, mapping);
 521         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 522                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 523
 524         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 525
 526         prod = NEXT_TX(prod);
 527         txbd1 = (struct tx_bd_ext *)
 528                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 529
 530         txbd1->tx_bd_hsize_lflags = lflags;
 531         if (skb_is_gso(skb)) {
 532                 u32 hdr_len;
 533
 534                 if (skb->encapsulation)
 535                         hdr_len = skb_inner_network_offset(skb) +
 536                                 skb_inner_network_header_len(skb) +
 537                                 inner_tcp_hdrlen(skb);
 538                 else
 539                         hdr_len = skb_transport_offset(skb) +
 540                                 tcp_hdrlen(skb);
 541
 542                 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
 543                                         TX_BD_FLAGS_T_IPID |
 544                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 545                 length = skb_shinfo(skb)->gso_size;
 546                 txbd1->tx_bd_mss = cpu_to_le32(length);
 547                 length += hdr_len;
 548         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 549                 txbd1->tx_bd_hsize_lflags |=
 550                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 551                 txbd1->tx_bd_mss = 0;
 552         }
 553
 554         length >>= 9;
 555         if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
 556                 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
 557                                      skb->len);
 558                 i = 0;
 559                 goto tx_dma_error;
 560         }
 561         flags |= bnxt_lhint_arr[length];
 562         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 563
 564         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 565         txbd1->tx_bd_cfa_action =
 566                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 567         for (i = 0; i < last_frag; i++) {
 568                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 569
 570                 prod = NEXT_TX(prod);
 571                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 572
 573                 len = skb_frag_size(frag);
 574                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 575                                            DMA_TO_DEVICE);
 576
 577                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 578                         goto tx_dma_error;
 579
 580                 tx_buf = &txr->tx_buf_ring[prod];
 581                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 582
 583                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 584
 585                 flags = len << TX_BD_LEN_SHIFT;
 586                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 587         }
 588
 589         flags &= ~TX_BD_LEN;
 590         txbd->tx_bd_len_flags_type =
 591                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 592                             TX_BD_FLAGS_PACKET_END);
 593
 594         netdev_tx_sent_queue(txq, skb->len);
 595
 596         /* Sync BD data before updating doorbell */
 597         wmb();
 598
 599         prod = NEXT_TX(prod);
 600         txr->tx_prod = prod;
 601
 602         if (!netdev_xmit_more() || netif_xmit_stopped(txq))
 603                 bnxt_db_write(bp, &txr->tx_db, prod);
 604
 605 tx_done:
 606
 607         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 608                 if (netdev_xmit_more() && !tx_buf->is_push)
 609                         bnxt_db_write(bp, &txr->tx_db, prod);
 610
 611                 netif_tx_stop_queue(txq);
 612
 613                 /* netif_tx_stop_queue() must be done before checking
 614                  * tx index in bnxt_tx_avail() below, because in
 615                  * bnxt_tx_int(), we update tx index before checking for
 616                  * netif_tx_queue_stopped().
 617                  */
 618                 smp_mb();
 619                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 620                         netif_tx_wake_queue(txq);
 621         }
 622         return NETDEV_TX_OK;
 623
 624 tx_dma_error:
 625         last_frag = i;
 626
 627         /* start back at beginning and unmap skb */
 628         prod = txr->tx_prod;
 629         tx_buf = &txr->tx_buf_ring[prod];
 630         tx_buf->skb = NULL;
 631         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 632                          skb_headlen(skb), PCI_DMA_TODEVICE);
 633         prod = NEXT_TX(prod);
 634
 635         /* unmap remaining mapped pages */
 636         for (i = 0; i < last_frag; i++) {
 637                 prod = NEXT_TX(prod);
 638                 tx_buf = &txr->tx_buf_ring[prod];
 639                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 640                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 641                                PCI_DMA_TODEVICE);
 642         }
 643
 644         dev_kfree_skb_any(skb);
 645         return NETDEV_TX_OK;
 646 }
 647
 648 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 649 {
 650         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 651         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 652         u16 cons = txr->tx_cons;
 653         struct pci_dev *pdev = bp->pdev;
 654         int i;
 655         unsigned int tx_bytes = 0;
 656
 657         for (i = 0; i < nr_pkts; i++) {
 658                 struct bnxt_sw_tx_bd *tx_buf;
 659                 struct sk_buff *skb;
 660                 int j, last;
 661
 662                 tx_buf = &txr->tx_buf_ring[cons];
 663                 cons = NEXT_TX(cons);
 664                 skb = tx_buf->skb;
 665                 tx_buf->skb = NULL;
 666
 667                 if (tx_buf->is_push) {
 668                         tx_buf->is_push = 0;
 669                         goto next_tx_int;
 670                 }
 671
 672                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 673                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 674                 last = tx_buf->nr_frags;
 675
 676                 for (j = 0; j < last; j++) {
 677                         cons = NEXT_TX(cons);
 678                         tx_buf = &txr->tx_buf_ring[cons];
 679                         dma_unmap_page(
 680                                 &pdev->dev,
 681                                 dma_unmap_addr(tx_buf, mapping),
 682                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 683                                 PCI_DMA_TODEVICE);
 684                 }
 685
 686 next_tx_int:
 687                 cons = NEXT_TX(cons);
 688
 689                 tx_bytes += skb->len;
 690                 dev_kfree_skb_any(skb);
 691         }
 692
 693         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 694         txr->tx_cons = cons;
 695
 696         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 697          * before checking for netif_tx_queue_stopped().  Without the
 698          * memory barrier, there is a small possibility that bnxt_start_xmit()
 699          * will miss it and cause the queue to be stopped forever.
 700          */
 701         smp_mb();
 702
 703         if (unlikely(netif_tx_queue_stopped(txq)) &&
 704             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 705                 __netif_tx_lock(txq, smp_processor_id());
 706                 if (netif_tx_queue_stopped(txq) &&
 707                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 708                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 709                         netif_tx_wake_queue(txq);
 710                 __netif_tx_unlock(txq);
 711         }
 712 }
 713
 714 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 715                                          struct bnxt_rx_ring_info *rxr,
 716                                          gfp_t gfp)
 717 {
 718         struct device *dev = &bp->pdev->dev;
 719         struct page *page;
 720
 721         page = page_pool_dev_alloc_pages(rxr->page_pool);
 722         if (!page)
 723                 return NULL;
 724
 725         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 726                                       DMA_ATTR_WEAK_ORDERING);
 727         if (dma_mapping_error(dev, *mapping)) {
 728                 page_pool_recycle_direct(rxr->page_pool, page);
 729                 return NULL;
 730         }
 731         *mapping += bp->rx_dma_offset;
 732         return page;
 733 }
 734
 735 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 736                                        gfp_t gfp)
 737 {
 738         u8 *data;
 739         struct pci_dev *pdev = bp->pdev;
 740
 741         data = kmalloc(bp->rx_buf_size, gfp);
 742         if (!data)
 743                 return NULL;
 744
 745         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 746                                         bp->rx_buf_use_size, bp->rx_dir,
 747                                         DMA_ATTR_WEAK_ORDERING);
 748
 749         if (dma_mapping_error(&pdev->dev, *mapping)) {
 750                 kfree(data);
 751                 data = NULL;
 752         }
 753         return data;
 754 }
 755
 756 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 757                        u16 prod, gfp_t gfp)
 758 {
 759         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 760         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 761         dma_addr_t mapping;
 762
 763         if (BNXT_RX_PAGE_MODE(bp)) {
 764                 struct page *page =
 765                         __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
 766
 767                 if (!page)
 768                         return -ENOMEM;
 769
 770                 rx_buf->data = page;
 771                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 772         } else {
 773                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 774
 775                 if (!data)
 776                         return -ENOMEM;
 777
 778                 rx_buf->data = data;
 779                 rx_buf->data_ptr = data + bp->rx_offset;
 780         }
 781         rx_buf->mapping = mapping;
 782
 783         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 784         return 0;
 785 }
 786
 787 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 788 {
 789         u16 prod = rxr->rx_prod;
 790         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 791         struct rx_bd *cons_bd, *prod_bd;
 792
 793         prod_rx_buf = &rxr->rx_buf_ring[prod];
 794         cons_rx_buf = &rxr->rx_buf_ring[cons];
 795
 796         prod_rx_buf->data = data;
 797         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 798
 799         prod_rx_buf->mapping = cons_rx_buf->mapping;
 800
 801         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 802         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 803
 804         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 805 }
 806
 807 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 808 {
 809         u16 next, max = rxr->rx_agg_bmap_size;
 810
 811         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 812         if (next >= max)
 813                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 814         return next;
 815 }
 816
 817 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 818                                      struct bnxt_rx_ring_info *rxr,
 819                                      u16 prod, gfp_t gfp)
 820 {
 821         struct rx_bd *rxbd =
 822                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 823         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 824         struct pci_dev *pdev = bp->pdev;
 825         struct page *page;
 826         dma_addr_t mapping;
 827         u16 sw_prod = rxr->rx_sw_agg_prod;
 828         unsigned int offset = 0;
 829
 830         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 831                 page = rxr->rx_page;
 832                 if (!page) {
 833                         page = alloc_page(gfp);
 834                         if (!page)
 835                                 return -ENOMEM;
 836                         rxr->rx_page = page;
 837                         rxr->rx_page_offset = 0;
 838                 }
 839                 offset = rxr->rx_page_offset;
 840                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 841                 if (rxr->rx_page_offset == PAGE_SIZE)
 842                         rxr->rx_page = NULL;
 843                 else
 844                         get_page(page);
 845         } else {
 846                 page = alloc_page(gfp);
 847                 if (!page)
 848                         return -ENOMEM;
 849         }
 850
 851         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 852                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 853                                      DMA_ATTR_WEAK_ORDERING);
 854         if (dma_mapping_error(&pdev->dev, mapping)) {
 855                 __free_page(page);
 856                 return -EIO;
 857         }
 858
 859         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 860                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 861
 862         __set_bit(sw_prod, rxr->rx_agg_bmap);
 863         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 864         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 865
 866         rx_agg_buf->page = page;
 867         rx_agg_buf->offset = offset;
 868         rx_agg_buf->mapping = mapping;
 869         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 870         rxbd->rx_bd_opaque = sw_prod;
 871         return 0;
 872 }
 873
 874 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
 875                                        struct bnxt_cp_ring_info *cpr,
 876                                        u16 cp_cons, u16 curr)
 877 {
 878         struct rx_agg_cmp *agg;
 879
 880         cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
 881         agg = (struct rx_agg_cmp *)
 882                 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 883         return agg;
 884 }
 885
 886 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
 887                                               struct bnxt_rx_ring_info *rxr,
 888                                               u16 agg_id, u16 curr)
 889 {
 890         struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
 891
 892         return &tpa_info->agg_arr[curr];
 893 }
 894
 895 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
 896                                    u16 start, u32 agg_bufs, bool tpa)
 897 {
 898         struct bnxt_napi *bnapi = cpr->bnapi;
 899         struct bnxt *bp = bnapi->bp;
 900         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 901         u16 prod = rxr->rx_agg_prod;
 902         u16 sw_prod = rxr->rx_sw_agg_prod;
 903         bool p5_tpa = false;
 904         u32 i;
 905
 906         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
 907                 p5_tpa = true;
 908
 909         for (i = 0; i < agg_bufs; i++) {
 910                 u16 cons;
 911                 struct rx_agg_cmp *agg;
 912                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 913                 struct rx_bd *prod_bd;
 914                 struct page *page;
 915
 916                 if (p5_tpa)
 917                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
 918                 else
 919                         agg = bnxt_get_agg(bp, cpr, idx, start + i);
 920                 cons = agg->rx_agg_cmp_opaque;
 921                 __clear_bit(cons, rxr->rx_agg_bmap);
 922
 923                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 924                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 925
 926                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 927                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 928                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 929
 930                 /* It is possible for sw_prod to be equal to cons, so
 931                  * set cons_rx_buf->page to NULL first.
 932                  */
 933                 page = cons_rx_buf->page;
 934                 cons_rx_buf->page = NULL;
 935                 prod_rx_buf->page = page;
 936                 prod_rx_buf->offset = cons_rx_buf->offset;
 937
 938                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 939
 940                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 941
 942                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 943                 prod_bd->rx_bd_opaque = sw_prod;
 944
 945                 prod = NEXT_RX_AGG(prod);
 946                 sw_prod = NEXT_RX_AGG(sw_prod);
 947         }
 948         rxr->rx_agg_prod = prod;
 949         rxr->rx_sw_agg_prod = sw_prod;
 950 }
 951
 952 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 953                                         struct bnxt_rx_ring_info *rxr,
 954                                         u16 cons, void *data, u8 *data_ptr,
 955                                         dma_addr_t dma_addr,
 956                                         unsigned int offset_and_len)
 957 {
 958         unsigned int payload = offset_and_len >> 16;
 959         unsigned int len = offset_and_len & 0xffff;
 960         skb_frag_t *frag;
 961         struct page *page = data;
 962         u16 prod = rxr->rx_prod;
 963         struct sk_buff *skb;
 964         int off, err;
 965
 966         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 967         if (unlikely(err)) {
 968                 bnxt_reuse_rx_data(rxr, cons, data);
 969                 return NULL;
 970         }
 971         dma_addr -= bp->rx_dma_offset;
 972         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 973                              DMA_ATTR_WEAK_ORDERING);
 974         page_pool_release_page(rxr->page_pool, page);
 975
 976         if (unlikely(!payload))
 977                 payload = eth_get_headlen(bp->dev, data_ptr, len);
 978
 979         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 980         if (!skb) {
 981                 __free_page(page);
 982                 return NULL;
 983         }
 984
 985         off = (void *)data_ptr - page_address(page);
 986         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 987         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 988                payload + NET_IP_ALIGN);
 989
 990         frag = &skb_shinfo(skb)->frags[0];
 991         skb_frag_size_sub(frag, payload);
 992         skb_frag_off_add(frag, payload);
 993         skb->data_len -= payload;
 994         skb->tail += payload;
 995
 996         return skb;
 997 }
 998
 999 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1000                                    struct bnxt_rx_ring_info *rxr, u16 cons,
1001                                    void *data, u8 *data_ptr,
1002                                    dma_addr_t dma_addr,
1003                                    unsigned int offset_and_len)
1004 {
1005         u16 prod = rxr->rx_prod;
1006         struct sk_buff *skb;
1007         int err;
1008
1009         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1010         if (unlikely(err)) {
1011                 bnxt_reuse_rx_data(rxr, cons, data);
1012                 return NULL;
1013         }
1014
1015         skb = build_skb(data, 0);
1016         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1017                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1018         if (!skb) {
1019                 kfree(data);
1020                 return NULL;
1021         }
1022
1023         skb_reserve(skb, bp->rx_offset);
1024         skb_put(skb, offset_and_len & 0xffff);
1025         return skb;
1026 }
1027
1028 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
1029                                      struct bnxt_cp_ring_info *cpr,
1030                                      struct sk_buff *skb, u16 idx,
1031                                      u32 agg_bufs, bool tpa)
1032 {
1033         struct bnxt_napi *bnapi = cpr->bnapi;
1034         struct pci_dev *pdev = bp->pdev;
1035         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1036         u16 prod = rxr->rx_agg_prod;
1037         bool p5_tpa = false;
1038         u32 i;
1039
1040         if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1041                 p5_tpa = true;
1042
1043         for (i = 0; i < agg_bufs; i++) {
1044                 u16 cons, frag_len;
1045                 struct rx_agg_cmp *agg;
1046                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1047                 struct page *page;
1048                 dma_addr_t mapping;
1049
1050                 if (p5_tpa)
1051                         agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1052                 else
1053                         agg = bnxt_get_agg(bp, cpr, idx, i);
1054                 cons = agg->rx_agg_cmp_opaque;
1055                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1056                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1057
1058                 cons_rx_buf = &rxr->rx_agg_ring[cons];
1059                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1060                                    cons_rx_buf->offset, frag_len);
1061                 __clear_bit(cons, rxr->rx_agg_bmap);
1062
1063                 /* It is possible for bnxt_alloc_rx_page() to allocate
1064                  * a sw_prod index that equals the cons index, so we
1065                  * need to clear the cons entry now.
1066                  */
1067                 mapping = cons_rx_buf->mapping;
1068                 page = cons_rx_buf->page;
1069                 cons_rx_buf->page = NULL;
1070
1071                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1072                         struct skb_shared_info *shinfo;
1073                         unsigned int nr_frags;
1074
1075                         shinfo = skb_shinfo(skb);
1076                         nr_frags = --shinfo->nr_frags;
1077                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1078
1079                         dev_kfree_skb(skb);
1080
1081                         cons_rx_buf->page = page;
1082
1083                         /* Update prod since possibly some pages have been
1084                          * allocated already.
1085                          */
1086                         rxr->rx_agg_prod = prod;
1087                         bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1088                         return NULL;
1089                 }
1090
1091                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1092                                      PCI_DMA_FROMDEVICE,
1093                                      DMA_ATTR_WEAK_ORDERING);
1094
1095                 skb->data_len += frag_len;
1096                 skb->len += frag_len;
1097                 skb->truesize += PAGE_SIZE;
1098
1099                 prod = NEXT_RX_AGG(prod);
1100         }
1101         rxr->rx_agg_prod = prod;
1102         return skb;
1103 }
1104
1105 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1106                                u8 agg_bufs, u32 *raw_cons)
1107 {
1108         u16 last;
1109         struct rx_agg_cmp *agg;
1110
1111         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1112         last = RING_CMP(*raw_cons);
1113         agg = (struct rx_agg_cmp *)
1114                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1115         return RX_AGG_CMP_VALID(agg, *raw_cons);
1116 }
1117
1118 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1119                                             unsigned int len,
1120                                             dma_addr_t mapping)
1121 {
1122         struct bnxt *bp = bnapi->bp;
1123         struct pci_dev *pdev = bp->pdev;
1124         struct sk_buff *skb;
1125
1126         skb = napi_alloc_skb(&bnapi->napi, len);
1127         if (!skb)
1128                 return NULL;
1129
1130         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1131                                 bp->rx_dir);
1132
1133         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1134                len + NET_IP_ALIGN);
1135
1136         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1137                                    bp->rx_dir);
1138
1139         skb_put(skb, len);
1140         return skb;
1141 }
1142
1143 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1144                            u32 *raw_cons, void *cmp)
1145 {
1146         struct rx_cmp *rxcmp = cmp;
1147         u32 tmp_raw_cons = *raw_cons;
1148         u8 cmp_type, agg_bufs = 0;
1149
1150         cmp_type = RX_CMP_TYPE(rxcmp);
1151
1152         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1153                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1154                             RX_CMP_AGG_BUFS) >>
1155                            RX_CMP_AGG_BUFS_SHIFT;
1156         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1157                 struct rx_tpa_end_cmp *tpa_end = cmp;
1158
1159                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1160                         return 0;
1161
1162                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1163         }
1164
1165         if (agg_bufs) {
1166                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1167                         return -EBUSY;
1168         }
1169         *raw_cons = tmp_raw_cons;
1170         return 0;
1171 }
1172
1173 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1174 {
1175         if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1176                 return;
1177
1178         if (BNXT_PF(bp))
1179                 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1180         else
1181                 schedule_delayed_work(&bp->fw_reset_task, delay);
1182 }
1183
1184 static void bnxt_queue_sp_work(struct bnxt *bp)
1185 {
1186         if (BNXT_PF(bp))
1187                 queue_work(bnxt_pf_wq, &bp->sp_task);
1188         else
1189                 schedule_work(&bp->sp_task);
1190 }
1191
1192 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1193 {
1194         if (!rxr->bnapi->in_reset) {
1195                 rxr->bnapi->in_reset = true;
1196                 if (bp->flags & BNXT_FLAG_CHIP_P5)
1197                         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1198                 else
1199                         set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
1200                 bnxt_queue_sp_work(bp);
1201         }
1202         rxr->rx_next_cons = 0xffff;
1203 }
1204
1205 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1206 {
1207         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1208         u16 idx = agg_id & MAX_TPA_P5_MASK;
1209
1210         if (test_bit(idx, map->agg_idx_bmap))
1211                 idx = find_first_zero_bit(map->agg_idx_bmap,
1212                                           BNXT_AGG_IDX_BMAP_SIZE);
1213         __set_bit(idx, map->agg_idx_bmap);
1214         map->agg_id_tbl[agg_id] = idx;
1215         return idx;
1216 }
1217
1218 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1219 {
1220         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1221
1222         __clear_bit(idx, map->agg_idx_bmap);
1223 }
1224
1225 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1226 {
1227         struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1228
1229         return map->agg_id_tbl[agg_id];
1230 }
1231
1232 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1233                            struct rx_tpa_start_cmp *tpa_start,
1234                            struct rx_tpa_start_cmp_ext *tpa_start1)
1235 {
1236         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1237         struct bnxt_tpa_info *tpa_info;
1238         u16 cons, prod, agg_id;
1239         struct rx_bd *prod_bd;
1240         dma_addr_t mapping;
1241
1242         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1243                 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1244                 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1245         } else {
1246                 agg_id = TPA_START_AGG_ID(tpa_start);
1247         }
1248         cons = tpa_start->rx_tpa_start_cmp_opaque;
1249         prod = rxr->rx_prod;
1250         cons_rx_buf = &rxr->rx_buf_ring[cons];
1251         prod_rx_buf = &rxr->rx_buf_ring[prod];
1252         tpa_info = &rxr->rx_tpa[agg_id];
1253
1254         if (unlikely(cons != rxr->rx_next_cons ||
1255                      TPA_START_ERROR(tpa_start))) {
1256                 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1257                             cons, rxr->rx_next_cons,
1258                             TPA_START_ERROR_CODE(tpa_start1));
1259                 bnxt_sched_reset(bp, rxr);
1260                 return;
1261         }
1262         /* Store cfa_code in tpa_info to use in tpa_end
1263          * completion processing.
1264          */
1265         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1266         prod_rx_buf->data = tpa_info->data;
1267         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1268
1269         mapping = tpa_info->mapping;
1270         prod_rx_buf->mapping = mapping;
1271
1272         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1273
1274         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1275
1276         tpa_info->data = cons_rx_buf->data;
1277         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1278         cons_rx_buf->data = NULL;
1279         tpa_info->mapping = cons_rx_buf->mapping;
1280
1281         tpa_info->len =
1282                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1283                                 RX_TPA_START_CMP_LEN_SHIFT;
1284         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1285                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1286
1287                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1288                 tpa_info->gso_type = SKB_GSO_TCPV4;
1289                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1290                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1291                         tpa_info->gso_type = SKB_GSO_TCPV6;
1292                 tpa_info->rss_hash =
1293                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1294         } else {
1295                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1296                 tpa_info->gso_type = 0;
1297                 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1298         }
1299         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1300         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1301         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1302         tpa_info->agg_count = 0;
1303
1304         rxr->rx_prod = NEXT_RX(prod);
1305         cons = NEXT_RX(cons);
1306         rxr->rx_next_cons = NEXT_RX(cons);
1307         cons_rx_buf = &rxr->rx_buf_ring[cons];
1308
1309         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1310         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1311         cons_rx_buf->data = NULL;
1312 }
1313
1314 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1315 {
1316         if (agg_bufs)
1317                 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1318 }
1319
1320 #ifdef CONFIG_INET
1321 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1322 {
1323         struct udphdr *uh = NULL;
1324
1325         if (ip_proto == htons(ETH_P_IP)) {
1326                 struct iphdr *iph = (struct iphdr *)skb->data;
1327
1328                 if (iph->protocol == IPPROTO_UDP)
1329                         uh = (struct udphdr *)(iph + 1);
1330         } else {
1331                 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1332
1333                 if (iph->nexthdr == IPPROTO_UDP)
1334                         uh = (struct udphdr *)(iph + 1);
1335         }
1336         if (uh) {
1337                 if (uh->check)
1338                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1339                 else
1340                         skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1341         }
1342 }
1343 #endif
1344
1345 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1346                                            int payload_off, int tcp_ts,
1347                                            struct sk_buff *skb)
1348 {
1349 #ifdef CONFIG_INET
1350         struct tcphdr *th;
1351         int len, nw_off;
1352         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1353         u32 hdr_info = tpa_info->hdr_info;
1354         bool loopback = false;
1355
1356         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1357         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1358         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1359
1360         /* If the packet is an internal loopback packet, the offsets will
1361          * have an extra 4 bytes.
1362          */
1363         if (inner_mac_off == 4) {
1364                 loopback = true;
1365         } else if (inner_mac_off > 4) {
1366                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1367                                             ETH_HLEN - 2));
1368
1369                 /* We only support inner iPv4/ipv6.  If we don't see the
1370                  * correct protocol ID, it must be a loopback packet where
1371                  * the offsets are off by 4.
1372                  */
1373                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1374                         loopback = true;
1375         }
1376         if (loopback) {
1377                 /* internal loopback packet, subtract all offsets by 4 */
1378                 inner_ip_off -= 4;
1379                 inner_mac_off -= 4;
1380                 outer_ip_off -= 4;
1381         }
1382
1383         nw_off = inner_ip_off - ETH_HLEN;
1384         skb_set_network_header(skb, nw_off);
1385         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1386                 struct ipv6hdr *iph = ipv6_hdr(skb);
1387
1388                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1389                 len = skb->len - skb_transport_offset(skb);
1390                 th = tcp_hdr(skb);
1391                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1392         } else {
1393                 struct iphdr *iph = ip_hdr(skb);
1394
1395                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1396                 len = skb->len - skb_transport_offset(skb);
1397                 th = tcp_hdr(skb);
1398                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1399         }
1400
1401         if (inner_mac_off) { /* tunnel */
1402                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1403                                             ETH_HLEN - 2));
1404
1405                 bnxt_gro_tunnel(skb, proto);
1406         }
1407 #endif
1408         return skb;
1409 }
1410
1411 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1412                                            int payload_off, int tcp_ts,
1413                                            struct sk_buff *skb)
1414 {
1415 #ifdef CONFIG_INET
1416         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1417         u32 hdr_info = tpa_info->hdr_info;
1418         int iphdr_len, nw_off;
1419
1420         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1421         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1422         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1423
1424         nw_off = inner_ip_off - ETH_HLEN;
1425         skb_set_network_header(skb, nw_off);
1426         iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1427                      sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1428         skb_set_transport_header(skb, nw_off + iphdr_len);
1429
1430         if (inner_mac_off) { /* tunnel */
1431                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1432                                             ETH_HLEN - 2));
1433
1434                 bnxt_gro_tunnel(skb, proto);
1435         }
1436 #endif
1437         return skb;
1438 }
1439
1440 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1441 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1442
1443 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1444                                            int payload_off, int tcp_ts,
1445                                            struct sk_buff *skb)
1446 {
1447 #ifdef CONFIG_INET
1448         struct tcphdr *th;
1449         int len, nw_off, tcp_opt_len = 0;
1450
1451         if (tcp_ts)
1452                 tcp_opt_len = 12;
1453
1454         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1455                 struct iphdr *iph;
1456
1457                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1458                          ETH_HLEN;
1459                 skb_set_network_header(skb, nw_off);
1460                 iph = ip_hdr(skb);
1461                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1462                 len = skb->len - skb_transport_offset(skb);
1463                 th = tcp_hdr(skb);
1464                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1465         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1466                 struct ipv6hdr *iph;
1467
1468                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1469                          ETH_HLEN;
1470                 skb_set_network_header(skb, nw_off);
1471                 iph = ipv6_hdr(skb);
1472                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1473                 len = skb->len - skb_transport_offset(skb);
1474                 th = tcp_hdr(skb);
1475                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1476         } else {
1477                 dev_kfree_skb_any(skb);
1478                 return NULL;
1479         }
1480
1481         if (nw_off) /* tunnel */
1482                 bnxt_gro_tunnel(skb, skb->protocol);
1483 #endif
1484         return skb;
1485 }
1486
1487 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1488                                            struct bnxt_tpa_info *tpa_info,
1489                                            struct rx_tpa_end_cmp *tpa_end,
1490                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1491                                            struct sk_buff *skb)
1492 {
1493 #ifdef CONFIG_INET
1494         int payload_off;
1495         u16 segs;
1496
1497         segs = TPA_END_TPA_SEGS(tpa_end);
1498         if (segs == 1)
1499                 return skb;
1500
1501         NAPI_GRO_CB(skb)->count = segs;
1502         skb_shinfo(skb)->gso_size =
1503                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1504         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1505         if (bp->flags & BNXT_FLAG_CHIP_P5)
1506                 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1507         else
1508                 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1509         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1510         if (likely(skb))
1511                 tcp_gro_complete(skb);
1512 #endif
1513         return skb;
1514 }
1515
1516 /* Given the cfa_code of a received packet determine which
1517  * netdev (vf-rep or PF) the packet is destined to.
1518  */
1519 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1520 {
1521         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1522
1523         /* if vf-rep dev is NULL, the must belongs to the PF */
1524         return dev ? dev : bp->dev;
1525 }
1526
1527 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1528                                            struct bnxt_cp_ring_info *cpr,
1529                                            u32 *raw_cons,
1530                                            struct rx_tpa_end_cmp *tpa_end,
1531                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1532                                            u8 *event)
1533 {
1534         struct bnxt_napi *bnapi = cpr->bnapi;
1535         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1536         u8 *data_ptr, agg_bufs;
1537         unsigned int len;
1538         struct bnxt_tpa_info *tpa_info;
1539         dma_addr_t mapping;
1540         struct sk_buff *skb;
1541         u16 idx = 0, agg_id;
1542         void *data;
1543         bool gro;
1544
1545         if (unlikely(bnapi->in_reset)) {
1546                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1547
1548                 if (rc < 0)
1549                         return ERR_PTR(-EBUSY);
1550                 return NULL;
1551         }
1552
1553         if (bp->flags & BNXT_FLAG_CHIP_P5) {
1554                 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1555                 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1556                 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1557                 tpa_info = &rxr->rx_tpa[agg_id];
1558                 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1559                         netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1560                                     agg_bufs, tpa_info->agg_count);
1561                         agg_bufs = tpa_info->agg_count;
1562                 }
1563                 tpa_info->agg_count = 0;
1564                 *event |= BNXT_AGG_EVENT;
1565                 bnxt_free_agg_idx(rxr, agg_id);
1566                 idx = agg_id;
1567                 gro = !!(bp->flags & BNXT_FLAG_GRO);
1568         } else {
1569                 agg_id = TPA_END_AGG_ID(tpa_end);
1570                 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1571                 tpa_info = &rxr->rx_tpa[agg_id];
1572                 idx = RING_CMP(*raw_cons);
1573                 if (agg_bufs) {
1574                         if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1575                                 return ERR_PTR(-EBUSY);
1576
1577                         *event |= BNXT_AGG_EVENT;
1578                         idx = NEXT_CMP(idx);
1579                 }
1580                 gro = !!TPA_END_GRO(tpa_end);
1581         }
1582         data = tpa_info->data;
1583         data_ptr = tpa_info->data_ptr;
1584         prefetch(data_ptr);
1585         len = tpa_info->len;
1586         mapping = tpa_info->mapping;
1587
1588         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1589                 bnxt_abort_tpa(cpr, idx, agg_bufs);
1590                 if (agg_bufs > MAX_SKB_FRAGS)
1591                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1592                                     agg_bufs, (int)MAX_SKB_FRAGS);
1593                 return NULL;
1594         }
1595
1596         if (len <= bp->rx_copy_thresh) {
1597                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1598                 if (!skb) {
1599                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1600                         return NULL;
1601                 }
1602         } else {
1603                 u8 *new_data;
1604                 dma_addr_t new_mapping;
1605
1606                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1607                 if (!new_data) {
1608                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1609                         return NULL;
1610                 }
1611
1612                 tpa_info->data = new_data;
1613                 tpa_info->data_ptr = new_data + bp->rx_offset;
1614                 tpa_info->mapping = new_mapping;
1615
1616                 skb = build_skb(data, 0);
1617                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1618                                        bp->rx_buf_use_size, bp->rx_dir,
1619                                        DMA_ATTR_WEAK_ORDERING);
1620
1621                 if (!skb) {
1622                         kfree(data);
1623                         bnxt_abort_tpa(cpr, idx, agg_bufs);
1624                         return NULL;
1625                 }
1626                 skb_reserve(skb, bp->rx_offset);
1627                 skb_put(skb, len);
1628         }
1629
1630         if (agg_bufs) {
1631                 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1632                 if (!skb) {
1633                         /* Page reuse already handled by bnxt_rx_pages(). */
1634                         return NULL;
1635                 }
1636         }
1637
1638         skb->protocol =
1639                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1640
1641         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1642                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1643
1644         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1645             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1646                 u16 vlan_proto = tpa_info->metadata >>
1647                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1648                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1649
1650                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1651         }
1652
1653         skb_checksum_none_assert(skb);
1654         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1655                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1656                 skb->csum_level =
1657                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1658         }
1659
1660         if (gro)
1661                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1662
1663         return skb;
1664 }
1665
1666 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1667                          struct rx_agg_cmp *rx_agg)
1668 {
1669         u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1670         struct bnxt_tpa_info *tpa_info;
1671
1672         agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1673         tpa_info = &rxr->rx_tpa[agg_id];
1674         BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1675         tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1676 }
1677
1678 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1679                              struct sk_buff *skb)
1680 {
1681         if (skb->dev != bp->dev) {
1682                 /* this packet belongs to a vf-rep */
1683                 bnxt_vf_rep_rx(bp, skb);
1684                 return;
1685         }
1686         skb_record_rx_queue(skb, bnapi->index);
1687         napi_gro_receive(&bnapi->napi, skb);
1688 }
1689
1690 /* returns the following:
1691  * 1       - 1 packet successfully received
1692  * 0       - successful TPA_START, packet not completed yet
1693  * -EBUSY  - completion ring does not have all the agg buffers yet
1694  * -ENOMEM - packet aborted due to out of memory
1695  * -EIO    - packet aborted due to hw error indicated in BD
1696  */
1697 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1698                        u32 *raw_cons, u8 *event)
1699 {
1700         struct bnxt_napi *bnapi = cpr->bnapi;
1701         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1702         struct net_device *dev = bp->dev;
1703         struct rx_cmp *rxcmp;
1704         struct rx_cmp_ext *rxcmp1;
1705         u32 tmp_raw_cons = *raw_cons;
1706         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1707         struct bnxt_sw_rx_bd *rx_buf;
1708         unsigned int len;
1709         u8 *data_ptr, agg_bufs, cmp_type;
1710         dma_addr_t dma_addr;
1711         struct sk_buff *skb;
1712         void *data;
1713         int rc = 0;
1714         u32 misc;
1715
1716         rxcmp = (struct rx_cmp *)
1717                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1718
1719         cmp_type = RX_CMP_TYPE(rxcmp);
1720
1721         if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1722                 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1723                 goto next_rx_no_prod_no_len;
1724         }
1725
1726         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1727         cp_cons = RING_CMP(tmp_raw_cons);
1728         rxcmp1 = (struct rx_cmp_ext *)
1729                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1730
1731         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1732                 return -EBUSY;
1733
1734         prod = rxr->rx_prod;
1735
1736         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1737                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1738                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1739
1740                 *event |= BNXT_RX_EVENT;
1741                 goto next_rx_no_prod_no_len;
1742
1743         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1744                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1745                                    (struct rx_tpa_end_cmp *)rxcmp,
1746                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1747
1748                 if (IS_ERR(skb))
1749                         return -EBUSY;
1750
1751                 rc = -ENOMEM;
1752                 if (likely(skb)) {
1753                         bnxt_deliver_skb(bp, bnapi, skb);
1754                         rc = 1;
1755                 }
1756                 *event |= BNXT_RX_EVENT;
1757                 goto next_rx_no_prod_no_len;
1758         }
1759
1760         cons = rxcmp->rx_cmp_opaque;
1761         if (unlikely(cons != rxr->rx_next_cons)) {
1762                 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
1763
1764                 /* 0xffff is forced error, don't print it */
1765                 if (rxr->rx_next_cons != 0xffff)
1766                         netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1767                                     cons, rxr->rx_next_cons);
1768                 bnxt_sched_reset(bp, rxr);
1769                 if (rc1)
1770                         return rc1;
1771                 goto next_rx_no_prod_no_len;
1772         }
1773         rx_buf = &rxr->rx_buf_ring[cons];
1774         data = rx_buf->data;
1775         data_ptr = rx_buf->data_ptr;
1776         prefetch(data_ptr);
1777
1778         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1779         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1780
1781         if (agg_bufs) {
1782                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1783                         return -EBUSY;
1784
1785                 cp_cons = NEXT_CMP(cp_cons);
1786                 *event |= BNXT_AGG_EVENT;
1787         }
1788         *event |= BNXT_RX_EVENT;
1789
1790         rx_buf->data = NULL;
1791         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1792                 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1793
1794                 bnxt_reuse_rx_data(rxr, cons, data);
1795                 if (agg_bufs)
1796                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1797                                                false);
1798
1799                 rc = -EIO;
1800                 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1801                         bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1802                         if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1803                             !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1804                                 netdev_warn_once(bp->dev, "RX buffer error %x\n",
1805                                                  rx_err);
1806                                 bnxt_sched_reset(bp, rxr);
1807                         }
1808                 }
1809                 goto next_rx_no_len;
1810         }
1811
1812         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1813         dma_addr = rx_buf->mapping;
1814
1815         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1816                 rc = 1;
1817                 goto next_rx;
1818         }
1819
1820         if (len <= bp->rx_copy_thresh) {
1821                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1822                 bnxt_reuse_rx_data(rxr, cons, data);
1823                 if (!skb) {
1824                         if (agg_bufs)
1825                                 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1826                                                        agg_bufs, false);
1827                         rc = -ENOMEM;
1828                         goto next_rx;
1829                 }
1830         } else {
1831                 u32 payload;
1832
1833                 if (rx_buf->data_ptr == data_ptr)
1834                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1835                 else
1836                         payload = 0;
1837                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1838                                       payload | len);
1839                 if (!skb) {
1840                         rc = -ENOMEM;
1841                         goto next_rx;
1842                 }
1843         }
1844
1845         if (agg_bufs) {
1846                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1847                 if (!skb) {
1848                         rc = -ENOMEM;
1849                         goto next_rx;
1850                 }
1851         }
1852
1853         if (RX_CMP_HASH_VALID(rxcmp)) {
1854                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1855                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1856
1857                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1858                 if (hash_type != 1 && hash_type != 3)
1859                         type = PKT_HASH_TYPE_L3;
1860                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1861         }
1862
1863         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1864         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1865
1866         if ((rxcmp1->rx_cmp_flags2 &
1867              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1868             (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1869                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1870                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1871                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1872
1873                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1874         }
1875
1876         skb_checksum_none_assert(skb);
1877         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1878                 if (dev->features & NETIF_F_RXCSUM) {
1879                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1880                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1881                 }
1882         } else {
1883                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1884                         if (dev->features & NETIF_F_RXCSUM)
1885                                 bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
1886                 }
1887         }
1888
1889         bnxt_deliver_skb(bp, bnapi, skb);
1890         rc = 1;
1891
1892 next_rx:
1893         cpr->rx_packets += 1;
1894         cpr->rx_bytes += len;
1895
1896 next_rx_no_len:
1897         rxr->rx_prod = NEXT_RX(prod);
1898         rxr->rx_next_cons = NEXT_RX(cons);
1899
1900 next_rx_no_prod_no_len:
1901         *raw_cons = tmp_raw_cons;
1902
1903         return rc;
1904 }
1905
1906 /* In netpoll mode, if we are using a combined completion ring, we need to
1907  * discard the rx packets and recycle the buffers.
1908  */
1909 static int bnxt_force_rx_discard(struct bnxt *bp,
1910                                  struct bnxt_cp_ring_info *cpr,
1911                                  u32 *raw_cons, u8 *event)
1912 {
1913         u32 tmp_raw_cons = *raw_cons;
1914         struct rx_cmp_ext *rxcmp1;
1915         struct rx_cmp *rxcmp;
1916         u16 cp_cons;
1917         u8 cmp_type;
1918
1919         cp_cons = RING_CMP(tmp_raw_cons);
1920         rxcmp = (struct rx_cmp *)
1921                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1922
1923         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1924         cp_cons = RING_CMP(tmp_raw_cons);
1925         rxcmp1 = (struct rx_cmp_ext *)
1926                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1927
1928         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1929                 return -EBUSY;
1930
1931         cmp_type = RX_CMP_TYPE(rxcmp);
1932         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1933                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1934                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1935         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1936                 struct rx_tpa_end_cmp_ext *tpa_end1;
1937
1938                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1939                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1940                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1941         }
1942         return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1943 }
1944
1945 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1946 {
1947         struct bnxt_fw_health *fw_health = bp->fw_health;
1948         u32 reg = fw_health->regs[reg_idx];
1949         u32 reg_type, reg_off, val = 0;
1950
1951         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1952         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1953         switch (reg_type) {
1954         case BNXT_FW_HEALTH_REG_TYPE_CFG:
1955                 pci_read_config_dword(bp->pdev, reg_off, &val);
1956                 break;
1957         case BNXT_FW_HEALTH_REG_TYPE_GRC:
1958                 reg_off = fw_health->mapped_regs[reg_idx];
1959                 fallthrough;
1960         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1961                 val = readl(bp->bar0 + reg_off);
1962                 break;
1963         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1964                 val = readl(bp->bar1 + reg_off);
1965                 break;
1966         }
1967         if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1968                 val &= fw_health->fw_reset_inprog_reg_mask;
1969         return val;
1970 }
1971
1972 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
1973 {
1974         int i;
1975
1976         for (i = 0; i < bp->rx_nr_rings; i++) {
1977                 u16 grp_idx = bp->rx_ring[i].bnapi->index;
1978                 struct bnxt_ring_grp_info *grp_info;
1979
1980                 grp_info = &bp->grp_info[grp_idx];
1981                 if (grp_info->agg_fw_ring_id == ring_id)
1982                         return grp_idx;
1983         }
1984         return INVALID_HW_RING_ID;
1985 }
1986
1987 #define BNXT_GET_EVENT_PORT(data)       \
1988         ((data) &                       \
1989          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1990
1991 #define BNXT_EVENT_RING_TYPE(data2)     \
1992         ((data2) &                      \
1993          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
1994
1995 #define BNXT_EVENT_RING_TYPE_RX(data2)  \
1996         (BNXT_EVENT_RING_TYPE(data2) == \
1997          ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
1998
1999 static int bnxt_async_event_process(struct bnxt *bp,
2000                                     struct hwrm_async_event_cmpl *cmpl)
2001 {
2002         u16 event_id = le16_to_cpu(cmpl->event_id);
2003         u32 data1 = le32_to_cpu(cmpl->event_data1);
2004         u32 data2 = le32_to_cpu(cmpl->event_data2);
2005
2006         /* TODO CHIMP_FW: Define event id's for link change, error etc */
2007         switch (event_id) {
2008         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2009                 struct bnxt_link_info *link_info = &bp->link_info;
2010
2011                 if (BNXT_VF(bp))
2012                         goto async_event_process_exit;
2013
2014                 /* print unsupported speed warning in forced speed mode only */
2015                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2016                     (data1 & 0x20000)) {
2017                         u16 fw_speed = link_info->force_link_speed;
2018                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2019
2020                         if (speed != SPEED_UNKNOWN)
2021                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2022                                             speed);
2023                 }
2024                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2025         }
2026                 fallthrough;
2027         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2028         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2029                 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2030                 fallthrough;
2031         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2032                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2033                 break;
2034         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2035                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2036                 break;
2037         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2038                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2039
2040                 if (BNXT_VF(bp))
2041                         break;
2042
2043                 if (bp->pf.port_id != port_id)
2044                         break;
2045
2046                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2047                 break;
2048         }
2049         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2050                 if (BNXT_PF(bp))
2051                         goto async_event_process_exit;
2052                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2053                 break;
2054         case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2055                 char *fatal_str = "non-fatal";
2056
2057                 if (!bp->fw_health)
2058                         goto async_event_process_exit;
2059
2060                 bp->fw_reset_timestamp = jiffies;
2061                 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2062                 if (!bp->fw_reset_min_dsecs)
2063                         bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2064                 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2065                 if (!bp->fw_reset_max_dsecs)
2066                         bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2067                 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2068                         fatal_str = "fatal";
2069                         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2070                 }
2071                 netif_warn(bp, hw, bp->dev,
2072                            "Firmware %s reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2073                            fatal_str, data1, data2,
2074                            bp->fw_reset_min_dsecs * 100,
2075                            bp->fw_reset_max_dsecs * 100);
2076                 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2077                 break;
2078         }
2079         case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2080                 struct bnxt_fw_health *fw_health = bp->fw_health;
2081
2082                 if (!fw_health)
2083                         goto async_event_process_exit;
2084
2085                 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2086                 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2087                 if (!fw_health->enabled) {
2088                         netif_info(bp, drv, bp->dev,
2089                                    "Error recovery info: error recovery[0]\n");
2090                         break;
2091                 }
2092                 fw_health->tmr_multiplier =
2093                         DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2094                                      bp->current_interval * 10);
2095                 fw_health->tmr_counter = fw_health->tmr_multiplier;
2096                 fw_health->last_fw_heartbeat =
2097                         bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2098                 fw_health->last_fw_reset_cnt =
2099                         bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2100                 netif_info(bp, drv, bp->dev,
2101                            "Error recovery info: error recovery[1], master[%d], reset count[%u], health status: 0x%x\n",
2102                            fw_health->master, fw_health->last_fw_reset_cnt,
2103                            bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG));
2104                 goto async_event_process_exit;
2105         }
2106         case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2107                 netif_notice(bp, hw, bp->dev,
2108                              "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2109                              data1, data2);
2110                 goto async_event_process_exit;
2111         case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2112                 struct bnxt_rx_ring_info *rxr;
2113                 u16 grp_idx;
2114
2115                 if (bp->flags & BNXT_FLAG_CHIP_P5)
2116                         goto async_event_process_exit;
2117
2118                 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2119                             BNXT_EVENT_RING_TYPE(data2), data1);
2120                 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2121                         goto async_event_process_exit;
2122
2123                 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2124                 if (grp_idx == INVALID_HW_RING_ID) {
2125                         netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2126                                     data1);
2127                         goto async_event_process_exit;
2128                 }
2129                 rxr = bp->bnapi[grp_idx]->rx_ring;
2130                 bnxt_sched_reset(bp, rxr);
2131                 goto async_event_process_exit;
2132         }
2133         case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2134                 struct bnxt_fw_health *fw_health = bp->fw_health;
2135
2136                 netif_notice(bp, hw, bp->dev,
2137                              "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2138                              data1, data2);
2139                 if (fw_health) {
2140                         fw_health->echo_req_data1 = data1;
2141                         fw_health->echo_req_data2 = data2;
2142                         set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2143                         break;
2144                 }
2145                 goto async_event_process_exit;
2146         }
2147         default:
2148                 goto async_event_process_exit;
2149         }
2150         bnxt_queue_sp_work(bp);
2151 async_event_process_exit:
2152         bnxt_ulp_async_events(bp, cmpl);
2153         return 0;
2154 }
2155
2156 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2157 {
2158         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2159         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2160         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2161                                 (struct hwrm_fwd_req_cmpl *)txcmp;
2162
2163         switch (cmpl_type) {
2164         case CMPL_BASE_TYPE_HWRM_DONE:
2165                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2166                 if (seq_id == bp->hwrm_intr_seq_id)
2167                         bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2168                 else
2169                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2170                 break;
2171
2172         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2173                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2174
2175                 if ((vf_id < bp->pf.first_vf_id) ||
2176                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2177                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2178                                    vf_id);
2179                         return -EINVAL;
2180                 }
2181
2182                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2183                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2184                 bnxt_queue_sp_work(bp);
2185                 break;
2186
2187         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2188                 bnxt_async_event_process(bp,
2189                                          (struct hwrm_async_event_cmpl *)txcmp);
2190
2191         default:
2192                 break;
2193         }
2194
2195         return 0;
2196 }
2197
2198 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2199 {
2200         struct bnxt_napi *bnapi = dev_instance;
2201         struct bnxt *bp = bnapi->bp;
2202         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2203         u32 cons = RING_CMP(cpr->cp_raw_cons);
2204
2205         cpr->event_ctr++;
2206         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2207         napi_schedule(&bnapi->napi);
2208         return IRQ_HANDLED;
2209 }
2210
2211 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2212 {
2213         u32 raw_cons = cpr->cp_raw_cons;
2214         u16 cons = RING_CMP(raw_cons);
2215         struct tx_cmp *txcmp;
2216
2217         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2218
2219         return TX_CMP_VALID(txcmp, raw_cons);
2220 }
2221
2222 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2223 {
2224         struct bnxt_napi *bnapi = dev_instance;
2225         struct bnxt *bp = bnapi->bp;
2226         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2227         u32 cons = RING_CMP(cpr->cp_raw_cons);
2228         u32 int_status;
2229
2230         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2231
2232         if (!bnxt_has_work(bp, cpr)) {
2233                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2234                 /* return if erroneous interrupt */
2235                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2236                         return IRQ_NONE;
2237         }
2238
2239         /* disable ring IRQ */
2240         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2241
2242         /* Return here if interrupt is shared and is disabled. */
2243         if (unlikely(atomic_read(&bp->intr_sem) != 0))
2244                 return IRQ_HANDLED;
2245
2246         napi_schedule(&bnapi->napi);
2247         return IRQ_HANDLED;
2248 }
2249
2250 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2251                             int budget)
2252 {
2253         struct bnxt_napi *bnapi = cpr->bnapi;
2254         u32 raw_cons = cpr->cp_raw_cons;
2255         u32 cons;
2256         int tx_pkts = 0;
2257         int rx_pkts = 0;
2258         u8 event = 0;
2259         struct tx_cmp *txcmp;
2260
2261         cpr->has_more_work = 0;
2262         cpr->had_work_done = 1;
2263         while (1) {
2264                 int rc;
2265
2266                 cons = RING_CMP(raw_cons);
2267                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2268
2269                 if (!TX_CMP_VALID(txcmp, raw_cons))
2270                         break;
2271
2272                 /* The valid test of the entry must be done first before
2273                  * reading any further.
2274                  */
2275                 dma_rmb();
2276                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2277                         tx_pkts++;
2278                         /* return full budget so NAPI will complete. */
2279                         if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2280                                 rx_pkts = budget;
2281                                 raw_cons = NEXT_RAW_CMP(raw_cons);
2282                                 if (budget)
2283                                         cpr->has_more_work = 1;
2284                                 break;
2285                         }
2286                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2287                         if (likely(budget))
2288                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2289                         else
2290                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2291                                                            &event);
2292                         if (likely(rc >= 0))
2293                                 rx_pkts += rc;
2294                         /* Increment rx_pkts when rc is -ENOMEM to count towards
2295                          * the NAPI budget.  Otherwise, we may potentially loop
2296                          * here forever if we consistently cannot allocate
2297                          * buffers.
2298                          */
2299                         else if (rc == -ENOMEM && budget)
2300                                 rx_pkts++;
2301                         else if (rc == -EBUSY)  /* partial completion */
2302                                 break;
2303                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2304                                      CMPL_BASE_TYPE_HWRM_DONE) ||
2305                                     (TX_CMP_TYPE(txcmp) ==
2306                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2307                                     (TX_CMP_TYPE(txcmp) ==
2308                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2309                         bnxt_hwrm_handler(bp, txcmp);
2310                 }
2311                 raw_cons = NEXT_RAW_CMP(raw_cons);
2312
2313                 if (rx_pkts && rx_pkts == budget) {
2314                         cpr->has_more_work = 1;
2315                         break;
2316                 }
2317         }
2318
2319         if (event & BNXT_REDIRECT_EVENT)
2320                 xdp_do_flush_map();
2321
2322         if (event & BNXT_TX_EVENT) {
2323                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2324                 u16 prod = txr->tx_prod;
2325
2326                 /* Sync BD data before updating doorbell */
2327                 wmb();
2328
2329                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2330         }
2331
2332         cpr->cp_raw_cons = raw_cons;
2333         bnapi->tx_pkts += tx_pkts;
2334         bnapi->events |= event;
2335         return rx_pkts;
2336 }
2337
2338 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2339 {
2340         if (bnapi->tx_pkts) {
2341                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2342                 bnapi->tx_pkts = 0;
2343         }
2344
2345         if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2346                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2347
2348                 if (bnapi->events & BNXT_AGG_EVENT)
2349                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2350                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2351         }
2352         bnapi->events = 0;
2353 }
2354
2355 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2356                           int budget)
2357 {
2358         struct bnxt_napi *bnapi = cpr->bnapi;
2359         int rx_pkts;
2360
2361         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2362
2363         /* ACK completion ring before freeing tx ring and producing new
2364          * buffers in rx/agg rings to prevent overflowing the completion
2365          * ring.
2366          */
2367         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2368
2369         __bnxt_poll_work_done(bp, bnapi);
2370         return rx_pkts;
2371 }
2372
2373 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2374 {
2375         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2376         struct bnxt *bp = bnapi->bp;
2377         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2378         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2379         struct tx_cmp *txcmp;
2380         struct rx_cmp_ext *rxcmp1;
2381         u32 cp_cons, tmp_raw_cons;
2382         u32 raw_cons = cpr->cp_raw_cons;
2383         u32 rx_pkts = 0;
2384         u8 event = 0;
2385
2386         while (1) {
2387                 int rc;
2388
2389                 cp_cons = RING_CMP(raw_cons);
2390                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2391
2392                 if (!TX_CMP_VALID(txcmp, raw_cons))
2393                         break;
2394
2395                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2396                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2397                         cp_cons = RING_CMP(tmp_raw_cons);
2398                         rxcmp1 = (struct rx_cmp_ext *)
2399                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2400
2401                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2402                                 break;
2403
2404                         /* force an error to recycle the buffer */
2405                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2406                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2407
2408                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2409                         if (likely(rc == -EIO) && budget)
2410                                 rx_pkts++;
2411                         else if (rc == -EBUSY)  /* partial completion */
2412                                 break;
2413                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2414                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2415                         bnxt_hwrm_handler(bp, txcmp);
2416                 } else {
2417                         netdev_err(bp->dev,
2418                                    "Invalid completion received on special ring\n");
2419                 }
2420                 raw_cons = NEXT_RAW_CMP(raw_cons);
2421
2422                 if (rx_pkts == budget)
2423                         break;
2424         }
2425
2426         cpr->cp_raw_cons = raw_cons;
2427         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2428         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2429
2430         if (event & BNXT_AGG_EVENT)
2431                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2432
2433         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2434                 napi_complete_done(napi, rx_pkts);
2435                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2436         }
2437         return rx_pkts;
2438 }
2439
2440 static int bnxt_poll(struct napi_struct *napi, int budget)
2441 {
2442         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2443         struct bnxt *bp = bnapi->bp;
2444         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2445         int work_done = 0;
2446
2447         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2448                 napi_complete(napi);
2449                 return 0;
2450         }
2451         while (1) {
2452                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2453
2454                 if (work_done >= budget) {
2455                         if (!budget)
2456                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2457                         break;
2458                 }
2459
2460                 if (!bnxt_has_work(bp, cpr)) {
2461                         if (napi_complete_done(napi, work_done))
2462                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2463                         break;
2464                 }
2465         }
2466         if (bp->flags & BNXT_FLAG_DIM) {
2467                 struct dim_sample dim_sample = {};
2468
2469                 dim_update_sample(cpr->event_ctr,
2470                                   cpr->rx_packets,
2471                                   cpr->rx_bytes,
2472                                   &dim_sample);
2473                 net_dim(&cpr->dim, dim_sample);
2474         }
2475         return work_done;
2476 }
2477
2478 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2479 {
2480         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2481         int i, work_done = 0;
2482
2483         for (i = 0; i < 2; i++) {
2484                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2485
2486                 if (cpr2) {
2487                         work_done += __bnxt_poll_work(bp, cpr2,
2488                                                       budget - work_done);
2489                         cpr->has_more_work |= cpr2->has_more_work;
2490                 }
2491         }
2492         return work_done;
2493 }
2494
2495 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2496                                  u64 dbr_type)
2497 {
2498         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2499         int i;
2500
2501         for (i = 0; i < 2; i++) {
2502                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2503                 struct bnxt_db_info *db;
2504
2505                 if (cpr2 && cpr2->had_work_done) {
2506                         db = &cpr2->cp_db;
2507                         writeq(db->db_key64 | dbr_type |
2508                                RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2509                         cpr2->had_work_done = 0;
2510                 }
2511         }
2512         __bnxt_poll_work_done(bp, bnapi);
2513 }
2514
2515 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2516 {
2517         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2518         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2519         u32 raw_cons = cpr->cp_raw_cons;
2520         struct bnxt *bp = bnapi->bp;
2521         struct nqe_cn *nqcmp;
2522         int work_done = 0;
2523         u32 cons;
2524
2525         if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2526                 napi_complete(napi);
2527                 return 0;
2528         }
2529         if (cpr->has_more_work) {
2530                 cpr->has_more_work = 0;
2531                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2532         }
2533         while (1) {
2534                 cons = RING_CMP(raw_cons);
2535                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2536
2537                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2538                         if (cpr->has_more_work)
2539                                 break;
2540
2541                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2542                         cpr->cp_raw_cons = raw_cons;
2543                         if (napi_complete_done(napi, work_done))
2544                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2545                                                   cpr->cp_raw_cons);
2546                         return work_done;
2547                 }
2548
2549                 /* The valid test of the entry must be done first before
2550                  * reading any further.
2551                  */
2552                 dma_rmb();
2553
2554                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2555                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2556                         struct bnxt_cp_ring_info *cpr2;
2557
2558                         cpr2 = cpr->cp_ring_arr[idx];
2559                         work_done += __bnxt_poll_work(bp, cpr2,
2560                                                       budget - work_done);
2561                         cpr->has_more_work |= cpr2->has_more_work;
2562                 } else {
2563                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2564                 }
2565                 raw_cons = NEXT_RAW_CMP(raw_cons);
2566         }
2567         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2568         if (raw_cons != cpr->cp_raw_cons) {
2569                 cpr->cp_raw_cons = raw_cons;
2570                 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2571         }
2572         return work_done;
2573 }
2574
2575 static void bnxt_free_tx_skbs(struct bnxt *bp)
2576 {
2577         int i, max_idx;
2578         struct pci_dev *pdev = bp->pdev;
2579
2580         if (!bp->tx_ring)
2581                 return;
2582
2583         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2584         for (i = 0; i < bp->tx_nr_rings; i++) {
2585                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2586                 int j;
2587
2588                 for (j = 0; j < max_idx;) {
2589                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2590                         struct sk_buff *skb;
2591                         int k, last;
2592
2593                         if (i < bp->tx_nr_rings_xdp &&
2594                             tx_buf->action == XDP_REDIRECT) {
2595                                 dma_unmap_single(&pdev->dev,
2596                                         dma_unmap_addr(tx_buf, mapping),
2597                                         dma_unmap_len(tx_buf, len),
2598                                         PCI_DMA_TODEVICE);
2599                                 xdp_return_frame(tx_buf->xdpf);
2600                                 tx_buf->action = 0;
2601                                 tx_buf->xdpf = NULL;
2602                                 j++;
2603                                 continue;
2604                         }
2605
2606                         skb = tx_buf->skb;
2607                         if (!skb) {
2608                                 j++;
2609                                 continue;
2610                         }
2611
2612                         tx_buf->skb = NULL;
2613
2614                         if (tx_buf->is_push) {
2615                                 dev_kfree_skb(skb);
2616                                 j += 2;
2617                                 continue;
2618                         }
2619
2620                         dma_unmap_single(&pdev->dev,
2621                                          dma_unmap_addr(tx_buf, mapping),
2622                                          skb_headlen(skb),
2623                                          PCI_DMA_TODEVICE);
2624
2625                         last = tx_buf->nr_frags;
2626                         j += 2;
2627                         for (k = 0; k < last; k++, j++) {
2628                                 int ring_idx = j & bp->tx_ring_mask;
2629                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2630
2631                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2632                                 dma_unmap_page(
2633                                         &pdev->dev,
2634                                         dma_unmap_addr(tx_buf, mapping),
2635                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2636                         }
2637                         dev_kfree_skb(skb);
2638                 }
2639                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2640         }
2641 }
2642
2643 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2644 {
2645         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2646         struct pci_dev *pdev = bp->pdev;
2647         struct bnxt_tpa_idx_map *map;
2648         int i, max_idx, max_agg_idx;
2649
2650         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2651         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2652         if (!rxr->rx_tpa)
2653                 goto skip_rx_tpa_free;
2654
2655         for (i = 0; i < bp->max_tpa; i++) {
2656                 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2657                 u8 *data = tpa_info->data;
2658
2659                 if (!data)
2660                         continue;
2661
2662                 dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2663                                        bp->rx_buf_use_size, bp->rx_dir,
2664                                        DMA_ATTR_WEAK_ORDERING);
2665
2666                 tpa_info->data = NULL;
2667
2668                 kfree(data);
2669         }
2670
2671 skip_rx_tpa_free:
2672         for (i = 0; i < max_idx; i++) {
2673                 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2674                 dma_addr_t mapping = rx_buf->mapping;
2675                 void *data = rx_buf->data;
2676
2677                 if (!data)
2678                         continue;
2679
2680                 rx_buf->data = NULL;
2681                 if (BNXT_RX_PAGE_MODE(bp)) {
2682                         mapping -= bp->rx_dma_offset;
2683                         dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE,
2684                                              bp->rx_dir,
2685                                              DMA_ATTR_WEAK_ORDERING);
2686                         page_pool_recycle_direct(rxr->page_pool, data);
2687                 } else {
2688                         dma_unmap_single_attrs(&pdev->dev, mapping,
2689                                                bp->rx_buf_use_size, bp->rx_dir,
2690                                                DMA_ATTR_WEAK_ORDERING);
2691                         kfree(data);
2692                 }
2693         }
2694         for (i = 0; i < max_agg_idx; i++) {
2695                 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2696                 struct page *page = rx_agg_buf->page;
2697
2698                 if (!page)
2699                         continue;
2700
2701                 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2702                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
2703                                      DMA_ATTR_WEAK_ORDERING);
2704
2705                 rx_agg_buf->page = NULL;
2706                 __clear_bit(i, rxr->rx_agg_bmap);
2707
2708                 __free_page(page);
2709         }
2710         if (rxr->rx_page) {
2711                 __free_page(rxr->rx_page);
2712                 rxr->rx_page = NULL;
2713         }
2714         map = rxr->rx_tpa_idx_map;
2715         if (map)
2716                 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2717 }
2718
2719 static void bnxt_free_rx_skbs(struct bnxt *bp)
2720 {
2721         int i;
2722
2723         if (!bp->rx_ring)
2724                 return;
2725
2726         for (i = 0; i < bp->rx_nr_rings; i++)
2727                 bnxt_free_one_rx_ring_skbs(bp, i);
2728 }
2729
2730 static void bnxt_free_skbs(struct bnxt *bp)
2731 {
2732         bnxt_free_tx_skbs(bp);
2733         bnxt_free_rx_skbs(bp);
2734 }
2735
2736 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len)
2737 {
2738         u8 init_val = mem_init->init_val;
2739         u16 offset = mem_init->offset;
2740         u8 *p2 = p;
2741         int i;
2742
2743         if (!init_val)
2744                 return;
2745         if (offset == BNXT_MEM_INVALID_OFFSET) {
2746                 memset(p, init_val, len);
2747                 return;
2748         }
2749         for (i = 0; i < len; i += mem_init->size)
2750                 *(p2 + i + offset) = init_val;
2751 }
2752
2753 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2754 {
2755         struct pci_dev *pdev = bp->pdev;
2756         int i;
2757
2758         for (i = 0; i < rmem->nr_pages; i++) {
2759                 if (!rmem->pg_arr[i])
2760                         continue;
2761
2762                 dma_free_coherent(&pdev->dev, rmem->page_size,
2763                                   rmem->pg_arr[i], rmem->dma_arr[i]);
2764
2765                 rmem->pg_arr[i] = NULL;
2766         }
2767         if (rmem->pg_tbl) {
2768                 size_t pg_tbl_size = rmem->nr_pages * 8;
2769
2770                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2771                         pg_tbl_size = rmem->page_size;
2772                 dma_free_coherent(&pdev->dev, pg_tbl_size,
2773                                   rmem->pg_tbl, rmem->pg_tbl_map);
2774                 rmem->pg_tbl = NULL;
2775         }
2776         if (rmem->vmem_size && *rmem->vmem) {
2777                 vfree(*rmem->vmem);
2778                 *rmem->vmem = NULL;
2779         }
2780 }
2781
2782 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2783 {
2784         struct pci_dev *pdev = bp->pdev;
2785         u64 valid_bit = 0;
2786         int i;
2787
2788         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2789                 valid_bit = PTU_PTE_VALID;
2790         if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2791                 size_t pg_tbl_size = rmem->nr_pages * 8;
2792
2793                 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2794                         pg_tbl_size = rmem->page_size;
2795                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2796                                                   &rmem->pg_tbl_map,
2797                                                   GFP_KERNEL);
2798                 if (!rmem->pg_tbl)
2799                         return -ENOMEM;
2800         }
2801
2802         for (i = 0; i < rmem->nr_pages; i++) {
2803                 u64 extra_bits = valid_bit;
2804
2805                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2806                                                      rmem->page_size,
2807                                                      &rmem->dma_arr[i],
2808                                                      GFP_KERNEL);
2809                 if (!rmem->pg_arr[i])
2810                         return -ENOMEM;
2811
2812                 if (rmem->mem_init)
2813                         bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i],
2814                                           rmem->page_size);
2815                 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2816                         if (i == rmem->nr_pages - 2 &&
2817                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2818                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2819                         else if (i == rmem->nr_pages - 1 &&
2820                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2821                                 extra_bits |= PTU_PTE_LAST;
2822                         rmem->pg_tbl[i] =
2823                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2824                 }
2825         }
2826
2827         if (rmem->vmem_size) {
2828                 *rmem->vmem = vzalloc(rmem->vmem_size);
2829                 if (!(*rmem->vmem))
2830                         return -ENOMEM;
2831         }
2832         return 0;
2833 }
2834
2835 static void bnxt_free_tpa_info(struct bnxt *bp)
2836 {
2837         int i;
2838
2839         for (i = 0; i < bp->rx_nr_rings; i++) {
2840                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2841
2842                 kfree(rxr->rx_tpa_idx_map);
2843                 rxr->rx_tpa_idx_map = NULL;
2844                 if (rxr->rx_tpa) {
2845                         kfree(rxr->rx_tpa[0].agg_arr);
2846                         rxr->rx_tpa[0].agg_arr = NULL;
2847                 }
2848                 kfree(rxr->rx_tpa);
2849                 rxr->rx_tpa = NULL;
2850         }
2851 }
2852
2853 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2854 {
2855         int i, j, total_aggs = 0;
2856
2857         bp->max_tpa = MAX_TPA;
2858         if (bp->flags & BNXT_FLAG_CHIP_P5) {
2859                 if (!bp->max_tpa_v2)
2860                         return 0;
2861                 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2862                 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2863         }
2864
2865         for (i = 0; i < bp->rx_nr_rings; i++) {
2866                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2867                 struct rx_agg_cmp *agg;
2868
2869                 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2870                                       GFP_KERNEL);
2871                 if (!rxr->rx_tpa)
2872                         return -ENOMEM;
2873
2874                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2875                         continue;
2876                 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2877                 rxr->rx_tpa[0].agg_arr = agg;
2878                 if (!agg)
2879                         return -ENOMEM;
2880                 for (j = 1; j < bp->max_tpa; j++)
2881                         rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2882                 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2883                                               GFP_KERNEL);
2884                 if (!rxr->rx_tpa_idx_map)
2885                         return -ENOMEM;
2886         }
2887         return 0;
2888 }
2889
2890 static void bnxt_free_rx_rings(struct bnxt *bp)
2891 {
2892         int i;
2893
2894         if (!bp->rx_ring)
2895                 return;
2896
2897         bnxt_free_tpa_info(bp);
2898         for (i = 0; i < bp->rx_nr_rings; i++) {
2899                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2900                 struct bnxt_ring_struct *ring;
2901
2902                 if (rxr->xdp_prog)
2903                         bpf_prog_put(rxr->xdp_prog);
2904
2905                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2906                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2907
2908                 page_pool_destroy(rxr->page_pool);
2909                 rxr->page_pool = NULL;
2910
2911                 kfree(rxr->rx_agg_bmap);
2912                 rxr->rx_agg_bmap = NULL;
2913
2914                 ring = &rxr->rx_ring_struct;
2915                 bnxt_free_ring(bp, &ring->ring_mem);
2916
2917                 ring = &rxr->rx_agg_ring_struct;
2918                 bnxt_free_ring(bp, &ring->ring_mem);
2919         }
2920 }
2921
2922 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2923                                    struct bnxt_rx_ring_info *rxr)
2924 {
2925         struct page_pool_params pp = { 0 };
2926
2927         pp.pool_size = bp->rx_ring_size;
2928         pp.nid = dev_to_node(&bp->pdev->dev);
2929         pp.dev = &bp->pdev->dev;
2930         pp.dma_dir = DMA_BIDIRECTIONAL;
2931
2932         rxr->page_pool = page_pool_create(&pp);
2933         if (IS_ERR(rxr->page_pool)) {
2934                 int err = PTR_ERR(rxr->page_pool);
2935
2936                 rxr->page_pool = NULL;
2937                 return err;
2938         }
2939         return 0;
2940 }
2941
2942 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2943 {
2944         int i, rc = 0, agg_rings = 0;
2945
2946         if (!bp->rx_ring)
2947                 return -ENOMEM;
2948
2949         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2950                 agg_rings = 1;
2951
2952         for (i = 0; i < bp->rx_nr_rings; i++) {
2953                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2954                 struct bnxt_ring_struct *ring;
2955
2956                 ring = &rxr->rx_ring_struct;
2957
2958                 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2959                 if (rc)
2960                         return rc;
2961
2962                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
2963                 if (rc < 0)
2964                         return rc;
2965
2966                 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2967                                                 MEM_TYPE_PAGE_POOL,
2968                                                 rxr->page_pool);
2969                 if (rc) {
2970                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2971                         return rc;
2972                 }
2973
2974                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2975                 if (rc)
2976                         return rc;
2977
2978                 ring->grp_idx = i;
2979                 if (agg_rings) {
2980                         u16 mem_size;
2981
2982                         ring = &rxr->rx_agg_ring_struct;
2983                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2984                         if (rc)
2985                                 return rc;
2986
2987                         ring->grp_idx = i;
2988                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2989                         mem_size = rxr->rx_agg_bmap_size / 8;
2990                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2991                         if (!rxr->rx_agg_bmap)
2992                                 return -ENOMEM;
2993                 }
2994         }
2995         if (bp->flags & BNXT_FLAG_TPA)
2996                 rc = bnxt_alloc_tpa_info(bp);
2997         return rc;
2998 }
2999
3000 static void bnxt_free_tx_rings(struct bnxt *bp)
3001 {
3002         int i;
3003         struct pci_dev *pdev = bp->pdev;
3004
3005         if (!bp->tx_ring)
3006                 return;
3007
3008         for (i = 0; i < bp->tx_nr_rings; i++) {
3009                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3010                 struct bnxt_ring_struct *ring;
3011
3012                 if (txr->tx_push) {
3013                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
3014                                           txr->tx_push, txr->tx_push_mapping);
3015                         txr->tx_push = NULL;
3016                 }
3017
3018                 ring = &txr->tx_ring_struct;
3019
3020                 bnxt_free_ring(bp, &ring->ring_mem);
3021         }
3022 }
3023
3024 static int bnxt_alloc_tx_rings(struct bnxt *bp)
3025 {
3026         int i, j, rc;
3027         struct pci_dev *pdev = bp->pdev;
3028
3029         bp->tx_push_size = 0;
3030         if (bp->tx_push_thresh) {
3031                 int push_size;
3032
3033                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3034                                         bp->tx_push_thresh);
3035
3036                 if (push_size > 256) {
3037                         push_size = 0;
3038                         bp->tx_push_thresh = 0;
3039                 }
3040
3041                 bp->tx_push_size = push_size;
3042         }
3043
3044         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3045                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3046                 struct bnxt_ring_struct *ring;
3047                 u8 qidx;
3048
3049                 ring = &txr->tx_ring_struct;
3050
3051                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3052                 if (rc)
3053                         return rc;
3054
3055                 ring->grp_idx = txr->bnapi->index;
3056                 if (bp->tx_push_size) {
3057                         dma_addr_t mapping;
3058
3059                         /* One pre-allocated DMA buffer to backup
3060                          * TX push operation
3061                          */
3062                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
3063                                                 bp->tx_push_size,
3064                                                 &txr->tx_push_mapping,
3065                                                 GFP_KERNEL);
3066
3067                         if (!txr->tx_push)
3068                                 return -ENOMEM;
3069
3070                         mapping = txr->tx_push_mapping +
3071                                 sizeof(struct tx_push_bd);
3072                         txr->data_mapping = cpu_to_le64(mapping);
3073                 }
3074                 qidx = bp->tc_to_qidx[j];
3075                 ring->queue_id = bp->q_info[qidx].queue_id;
3076                 if (i < bp->tx_nr_rings_xdp)
3077                         continue;
3078                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3079                         j++;
3080         }
3081         return 0;
3082 }
3083
3084 static void bnxt_free_cp_rings(struct bnxt *bp)
3085 {
3086         int i;
3087
3088         if (!bp->bnapi)
3089                 return;
3090
3091         for (i = 0; i < bp->cp_nr_rings; i++) {
3092                 struct bnxt_napi *bnapi = bp->bnapi[i];
3093                 struct bnxt_cp_ring_info *cpr;
3094                 struct bnxt_ring_struct *ring;
3095                 int j;
3096
3097                 if (!bnapi)
3098                         continue;
3099
3100                 cpr = &bnapi->cp_ring;
3101                 ring = &cpr->cp_ring_struct;
3102
3103                 bnxt_free_ring(bp, &ring->ring_mem);
3104
3105                 for (j = 0; j < 2; j++) {
3106                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3107
3108                         if (cpr2) {
3109                                 ring = &cpr2->cp_ring_struct;
3110                                 bnxt_free_ring(bp, &ring->ring_mem);
3111                                 kfree(cpr2);
3112                                 cpr->cp_ring_arr[j] = NULL;
3113                         }
3114                 }
3115         }
3116 }
3117
3118 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3119 {
3120         struct bnxt_ring_mem_info *rmem;
3121         struct bnxt_ring_struct *ring;
3122         struct bnxt_cp_ring_info *cpr;
3123         int rc;
3124
3125         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3126         if (!cpr)
3127                 return NULL;
3128
3129         ring = &cpr->cp_ring_struct;
3130         rmem = &ring->ring_mem;
3131         rmem->nr_pages = bp->cp_nr_pages;
3132         rmem->page_size = HW_CMPD_RING_SIZE;
3133         rmem->pg_arr = (void **)cpr->cp_desc_ring;
3134         rmem->dma_arr = cpr->cp_desc_mapping;
3135         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3136         rc = bnxt_alloc_ring(bp, rmem);
3137         if (rc) {
3138                 bnxt_free_ring(bp, rmem);
3139                 kfree(cpr);
3140                 cpr = NULL;
3141         }
3142         return cpr;
3143 }
3144
3145 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3146 {
3147         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3148         int i, rc, ulp_base_vec, ulp_msix;
3149
3150         ulp_msix = bnxt_get_ulp_msix_num(bp);
3151         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3152         for (i = 0; i < bp->cp_nr_rings; i++) {
3153                 struct bnxt_napi *bnapi = bp->bnapi[i];
3154                 struct bnxt_cp_ring_info *cpr;
3155                 struct bnxt_ring_struct *ring;
3156
3157                 if (!bnapi)
3158                         continue;
3159
3160                 cpr = &bnapi->cp_ring;
3161                 cpr->bnapi = bnapi;
3162                 ring = &cpr->cp_ring_struct;
3163
3164                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3165                 if (rc)
3166                         return rc;
3167
3168                 if (ulp_msix && i >= ulp_base_vec)
3169                         ring->map_idx = i + ulp_msix;
3170                 else
3171                         ring->map_idx = i;
3172
3173                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3174                         continue;
3175
3176                 if (i < bp->rx_nr_rings) {
3177                         struct bnxt_cp_ring_info *cpr2 =
3178                                 bnxt_alloc_cp_sub_ring(bp);
3179
3180                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3181                         if (!cpr2)
3182                                 return -ENOMEM;
3183                         cpr2->bnapi = bnapi;
3184                 }
3185                 if ((sh && i < bp->tx_nr_rings) ||
3186                     (!sh && i >= bp->rx_nr_rings)) {
3187                         struct bnxt_cp_ring_info *cpr2 =
3188                                 bnxt_alloc_cp_sub_ring(bp);
3189
3190                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3191                         if (!cpr2)
3192                                 return -ENOMEM;
3193                         cpr2->bnapi = bnapi;
3194                 }
3195         }
3196         return 0;
3197 }
3198
3199 static void bnxt_init_ring_struct(struct bnxt *bp)
3200 {
3201         int i;
3202
3203         for (i = 0; i < bp->cp_nr_rings; i++) {
3204                 struct bnxt_napi *bnapi = bp->bnapi[i];
3205                 struct bnxt_ring_mem_info *rmem;
3206                 struct bnxt_cp_ring_info *cpr;
3207                 struct bnxt_rx_ring_info *rxr;
3208                 struct bnxt_tx_ring_info *txr;
3209                 struct bnxt_ring_struct *ring;
3210
3211                 if (!bnapi)
3212                         continue;
3213
3214                 cpr = &bnapi->cp_ring;
3215                 ring = &cpr->cp_ring_struct;
3216                 rmem = &ring->ring_mem;
3217                 rmem->nr_pages = bp->cp_nr_pages;
3218                 rmem->page_size = HW_CMPD_RING_SIZE;
3219                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3220                 rmem->dma_arr = cpr->cp_desc_mapping;
3221                 rmem->vmem_size = 0;
3222
3223                 rxr = bnapi->rx_ring;
3224                 if (!rxr)
3225                         goto skip_rx;
3226
3227                 ring = &rxr->rx_ring_struct;
3228                 rmem = &ring->ring_mem;
3229                 rmem->nr_pages = bp->rx_nr_pages;
3230                 rmem->page_size = HW_RXBD_RING_SIZE;
3231                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3232                 rmem->dma_arr = rxr->rx_desc_mapping;
3233                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3234                 rmem->vmem = (void **)&rxr->rx_buf_ring;
3235
3236                 ring = &rxr->rx_agg_ring_struct;
3237                 rmem = &ring->ring_mem;
3238                 rmem->nr_pages = bp->rx_agg_nr_pages;
3239                 rmem->page_size = HW_RXBD_RING_SIZE;
3240                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3241                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3242                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3243                 rmem->vmem = (void **)&rxr->rx_agg_ring;
3244
3245 skip_rx:
3246                 txr = bnapi->tx_ring;
3247                 if (!txr)
3248                         continue;
3249
3250                 ring = &txr->tx_ring_struct;
3251                 rmem = &ring->ring_mem;
3252                 rmem->nr_pages = bp->tx_nr_pages;
3253                 rmem->page_size = HW_RXBD_RING_SIZE;
3254                 rmem->pg_arr = (void **)txr->tx_desc_ring;
3255                 rmem->dma_arr = txr->tx_desc_mapping;
3256                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3257                 rmem->vmem = (void **)&txr->tx_buf_ring;
3258         }
3259 }
3260
3261 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3262 {
3263         int i;
3264         u32 prod;
3265         struct rx_bd **rx_buf_ring;
3266
3267         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3268         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3269                 int j;
3270                 struct rx_bd *rxbd;
3271
3272                 rxbd = rx_buf_ring[i];
3273                 if (!rxbd)
3274                         continue;
3275
3276                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3277                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3278                         rxbd->rx_bd_opaque = prod;
3279                 }
3280         }
3281 }
3282
3283 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3284 {
3285         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3286         struct net_device *dev = bp->dev;
3287         u32 prod;
3288         int i;
3289
3290         prod = rxr->rx_prod;
3291         for (i = 0; i < bp->rx_ring_size; i++) {
3292                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3293                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3294                                     ring_nr, i, bp->rx_ring_size);
3295                         break;
3296                 }
3297                 prod = NEXT_RX(prod);
3298         }
3299         rxr->rx_prod = prod;
3300
3301         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3302                 return 0;
3303
3304         prod = rxr->rx_agg_prod;
3305         for (i = 0; i < bp->rx_agg_ring_size; i++) {
3306                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3307                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3308                                     ring_nr, i, bp->rx_ring_size);
3309                         break;
3310                 }
3311                 prod = NEXT_RX_AGG(prod);
3312         }
3313         rxr->rx_agg_prod = prod;
3314
3315         if (rxr->rx_tpa) {
3316                 dma_addr_t mapping;
3317                 u8 *data;
3318
3319                 for (i = 0; i < bp->max_tpa; i++) {
3320                         data = __bnxt_alloc_rx_data(bp, &mapping, GFP_KERNEL);
3321                         if (!data)
3322                                 return -ENOMEM;
3323
3324                         rxr->rx_tpa[i].data = data;
3325                         rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3326                         rxr->rx_tpa[i].mapping = mapping;
3327                 }
3328         }
3329         return 0;
3330 }
3331
3332 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3333 {
3334         struct bnxt_rx_ring_info *rxr;
3335         struct bnxt_ring_struct *ring;
3336         u32 type;
3337
3338         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3339                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3340
3341         if (NET_IP_ALIGN == 2)
3342                 type |= RX_BD_FLAGS_SOP;
3343
3344         rxr = &bp->rx_ring[ring_nr];
3345         ring = &rxr->rx_ring_struct;
3346         bnxt_init_rxbd_pages(ring, type);
3347
3348         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3349                 bpf_prog_add(bp->xdp_prog, 1);
3350                 rxr->xdp_prog = bp->xdp_prog;
3351         }
3352         ring->fw_ring_id = INVALID_HW_RING_ID;
3353
3354         ring = &rxr->rx_agg_ring_struct;
3355         ring->fw_ring_id = INVALID_HW_RING_ID;
3356
3357         if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3358                 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3359                         RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3360
3361                 bnxt_init_rxbd_pages(ring, type);
3362         }
3363
3364         return bnxt_alloc_one_rx_ring(bp, ring_nr);
3365 }
3366
3367 static void bnxt_init_cp_rings(struct bnxt *bp)
3368 {
3369         int i, j;
3370
3371         for (i = 0; i < bp->cp_nr_rings; i++) {
3372                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3373                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3374
3375                 ring->fw_ring_id = INVALID_HW_RING_ID;
3376                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3377                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3378                 for (j = 0; j < 2; j++) {
3379                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3380
3381                         if (!cpr2)
3382                                 continue;
3383
3384                         ring = &cpr2->cp_ring_struct;
3385                         ring->fw_ring_id = INVALID_HW_RING_ID;
3386                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3387                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3388                 }
3389         }
3390 }
3391
3392 static int bnxt_init_rx_rings(struct bnxt *bp)
3393 {
3394         int i, rc = 0;
3395
3396         if (BNXT_RX_PAGE_MODE(bp)) {
3397                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3398                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3399         } else {
3400                 bp->rx_offset = BNXT_RX_OFFSET;
3401                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3402         }
3403
3404         for (i = 0; i < bp->rx_nr_rings; i++) {
3405                 rc = bnxt_init_one_rx_ring(bp, i);
3406                 if (rc)
3407                         break;
3408         }
3409
3410         return rc;
3411 }
3412
3413 static int bnxt_init_tx_rings(struct bnxt *bp)
3414 {
3415         u16 i;
3416
3417         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3418                                    MAX_SKB_FRAGS + 1);
3419
3420         for (i = 0; i < bp->tx_nr_rings; i++) {
3421                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3422                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3423
3424                 ring->fw_ring_id = INVALID_HW_RING_ID;
3425         }
3426
3427         return 0;
3428 }
3429
3430 static void bnxt_free_ring_grps(struct bnxt *bp)
3431 {
3432         kfree(bp->grp_info);
3433         bp->grp_info = NULL;
3434 }
3435
3436 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3437 {
3438         int i;
3439
3440         if (irq_re_init) {
3441                 bp->grp_info = kcalloc(bp->cp_nr_rings,
3442                                        sizeof(struct bnxt_ring_grp_info),
3443                                        GFP_KERNEL);
3444                 if (!bp->grp_info)
3445                         return -ENOMEM;
3446         }
3447         for (i = 0; i < bp->cp_nr_rings; i++) {
3448                 if (irq_re_init)
3449                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3450                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3451                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3452                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3453                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3454         }
3455         return 0;
3456 }
3457
3458 static void bnxt_free_vnics(struct bnxt *bp)
3459 {
3460         kfree(bp->vnic_info);
3461         bp->vnic_info = NULL;
3462         bp->nr_vnics = 0;
3463 }
3464
3465 static int bnxt_alloc_vnics(struct bnxt *bp)
3466 {
3467         int num_vnics = 1;
3468
3469 #ifdef CONFIG_RFS_ACCEL
3470         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3471                 num_vnics += bp->rx_nr_rings;
3472 #endif
3473
3474         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3475                 num_vnics++;
3476
3477         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3478                                 GFP_KERNEL);
3479         if (!bp->vnic_info)
3480                 return -ENOMEM;
3481
3482         bp->nr_vnics = num_vnics;
3483         return 0;
3484 }
3485
3486 static void bnxt_init_vnics(struct bnxt *bp)
3487 {
3488         int i;
3489
3490         for (i = 0; i < bp->nr_vnics; i++) {
3491                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3492                 int j;
3493
3494                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3495                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3496                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3497
3498                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3499
3500                 if (bp->vnic_info[i].rss_hash_key) {
3501                         if (i == 0)
3502                                 prandom_bytes(vnic->rss_hash_key,
3503                                               HW_HASH_KEY_SIZE);
3504                         else
3505                                 memcpy(vnic->rss_hash_key,
3506                                        bp->vnic_info[0].rss_hash_key,
3507                                        HW_HASH_KEY_SIZE);
3508                 }
3509         }
3510 }
3511
3512 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3513 {
3514         int pages;
3515
3516         pages = ring_size / desc_per_pg;
3517
3518         if (!pages)
3519                 return 1;
3520
3521         pages++;
3522
3523         while (pages & (pages - 1))
3524                 pages++;
3525
3526         return pages;
3527 }
3528
3529 void bnxt_set_tpa_flags(struct bnxt *bp)
3530 {
3531         bp->flags &= ~BNXT_FLAG_TPA;
3532         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3533                 return;
3534         if (bp->dev->features & NETIF_F_LRO)
3535                 bp->flags |= BNXT_FLAG_LRO;
3536         else if (bp->dev->features & NETIF_F_GRO_HW)
3537                 bp->flags |= BNXT_FLAG_GRO;
3538 }
3539
3540 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3541  * be set on entry.
3542  */
3543 void bnxt_set_ring_params(struct bnxt *bp)
3544 {
3545         u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3546         u32 agg_factor = 0, agg_ring_size = 0;
3547
3548         /* 8 for CRC and VLAN */
3549         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3550
3551         rx_space = rx_size + NET_SKB_PAD +
3552                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3553
3554         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3555         ring_size = bp->rx_ring_size;
3556         bp->rx_agg_ring_size = 0;
3557         bp->rx_agg_nr_pages = 0;
3558
3559         if (bp->flags & BNXT_FLAG_TPA)
3560                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3561
3562         bp->flags &= ~BNXT_FLAG_JUMBO;
3563         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3564                 u32 jumbo_factor;
3565
3566                 bp->flags |= BNXT_FLAG_JUMBO;
3567                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3568                 if (jumbo_factor > agg_factor)
3569                         agg_factor = jumbo_factor;
3570         }
3571         agg_ring_size = ring_size * agg_factor;
3572
3573         if (agg_ring_size) {
3574                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3575                                                         RX_DESC_CNT);
3576                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3577                         u32 tmp = agg_ring_size;
3578
3579                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3580                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3581                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3582                                     tmp, agg_ring_size);
3583                 }
3584                 bp->rx_agg_ring_size = agg_ring_size;
3585                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3586                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3587                 rx_space = rx_size + NET_SKB_PAD +
3588                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3589         }
3590
3591         bp->rx_buf_use_size = rx_size;
3592         bp->rx_buf_size = rx_space;
3593
3594         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3595         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3596
3597         ring_size = bp->tx_ring_size;
3598         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3599         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3600
3601         max_rx_cmpl = bp->rx_ring_size;
3602         /* MAX TPA needs to be added because TPA_START completions are
3603          * immediately recycled, so the TPA completions are not bound by
3604          * the RX ring size.
3605          */
3606         if (bp->flags & BNXT_FLAG_TPA)
3607                 max_rx_cmpl += bp->max_tpa;
3608         /* RX and TPA completions are 32-byte, all others are 16-byte */
3609         ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3610         bp->cp_ring_size = ring_size;
3611
3612         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3613         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3614                 bp->cp_nr_pages = MAX_CP_PAGES;
3615                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3616                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3617                             ring_size, bp->cp_ring_size);
3618         }
3619         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3620         bp->cp_ring_mask = bp->cp_bit - 1;
3621 }
3622
3623 /* Changing allocation mode of RX rings.
3624  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3625  */
3626 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3627 {
3628         if (page_mode) {
3629                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3630                         return -EOPNOTSUPP;
3631                 bp->dev->max_mtu =
3632                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3633                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3634                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3635                 bp->rx_dir = DMA_BIDIRECTIONAL;
3636                 bp->rx_skb_func = bnxt_rx_page_skb;
3637                 /* Disable LRO or GRO_HW */
3638                 netdev_update_features(bp->dev);
3639         } else {
3640                 bp->dev->max_mtu = bp->max_mtu;
3641                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3642                 bp->rx_dir = DMA_FROM_DEVICE;
3643                 bp->rx_skb_func = bnxt_rx_skb;
3644         }
3645         return 0;
3646 }
3647
3648 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3649 {
3650         int i;
3651         struct bnxt_vnic_info *vnic;
3652         struct pci_dev *pdev = bp->pdev;
3653
3654         if (!bp->vnic_info)
3655                 return;
3656
3657         for (i = 0; i < bp->nr_vnics; i++) {
3658                 vnic = &bp->vnic_info[i];
3659
3660                 kfree(vnic->fw_grp_ids);
3661                 vnic->fw_grp_ids = NULL;
3662
3663                 kfree(vnic->uc_list);
3664                 vnic->uc_list = NULL;
3665
3666                 if (vnic->mc_list) {
3667                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3668                                           vnic->mc_list, vnic->mc_list_mapping);
3669                         vnic->mc_list = NULL;
3670                 }
3671
3672                 if (vnic->rss_table) {
3673                         dma_free_coherent(&pdev->dev, vnic->rss_table_size,
3674                                           vnic->rss_table,
3675                                           vnic->rss_table_dma_addr);
3676                         vnic->rss_table = NULL;
3677                 }
3678
3679                 vnic->rss_hash_key = NULL;
3680                 vnic->flags = 0;
3681         }
3682 }
3683
3684 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3685 {
3686         int i, rc = 0, size;
3687         struct bnxt_vnic_info *vnic;
3688         struct pci_dev *pdev = bp->pdev;
3689         int max_rings;
3690
3691         for (i = 0; i < bp->nr_vnics; i++) {
3692                 vnic = &bp->vnic_info[i];
3693
3694                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3695                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3696
3697                         if (mem_size > 0) {
3698                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3699                                 if (!vnic->uc_list) {
3700                                         rc = -ENOMEM;
3701                                         goto out;
3702                                 }
3703                         }
3704                 }
3705
3706                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3707                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3708                         vnic->mc_list =
3709                                 dma_alloc_coherent(&pdev->dev,
3710                                                    vnic->mc_list_size,
3711                                                    &vnic->mc_list_mapping,
3712                                                    GFP_KERNEL);
3713                         if (!vnic->mc_list) {
3714                                 rc = -ENOMEM;
3715                                 goto out;
3716                         }
3717                 }
3718
3719                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3720                         goto vnic_skip_grps;
3721
3722                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3723                         max_rings = bp->rx_nr_rings;
3724                 else
3725                         max_rings = 1;
3726
3727                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3728                 if (!vnic->fw_grp_ids) {
3729                         rc = -ENOMEM;
3730                         goto out;
3731                 }
3732 vnic_skip_grps:
3733                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3734                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3735                         continue;
3736
3737                 /* Allocate rss table and hash key */
3738                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3739                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3740                         size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
3741
3742                 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
3743                 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
3744                                                      vnic->rss_table_size,
3745                                                      &vnic->rss_table_dma_addr,
3746                                                      GFP_KERNEL);
3747                 if (!vnic->rss_table) {
3748                         rc = -ENOMEM;
3749                         goto out;
3750                 }
3751
3752                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3753                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3754         }
3755         return 0;
3756
3757 out:
3758         return rc;
3759 }
3760
3761 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3762 {
3763         struct pci_dev *pdev = bp->pdev;
3764
3765         if (bp->hwrm_cmd_resp_addr) {
3766                 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3767                                   bp->hwrm_cmd_resp_dma_addr);
3768                 bp->hwrm_cmd_resp_addr = NULL;
3769         }
3770
3771         if (bp->hwrm_cmd_kong_resp_addr) {
3772                 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3773                                   bp->hwrm_cmd_kong_resp_addr,
3774                                   bp->hwrm_cmd_kong_resp_dma_addr);
3775                 bp->hwrm_cmd_kong_resp_addr = NULL;
3776         }
3777 }
3778
3779 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3780 {
3781         struct pci_dev *pdev = bp->pdev;
3782
3783         if (bp->hwrm_cmd_kong_resp_addr)
3784                 return 0;
3785
3786         bp->hwrm_cmd_kong_resp_addr =
3787                 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3788                                    &bp->hwrm_cmd_kong_resp_dma_addr,
3789                                    GFP_KERNEL);
3790         if (!bp->hwrm_cmd_kong_resp_addr)
3791                 return -ENOMEM;
3792
3793         return 0;
3794 }
3795
3796 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3797 {
3798         struct pci_dev *pdev = bp->pdev;
3799
3800         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3801                                                    &bp->hwrm_cmd_resp_dma_addr,
3802                                                    GFP_KERNEL);
3803         if (!bp->hwrm_cmd_resp_addr)
3804                 return -ENOMEM;
3805
3806         return 0;
3807 }
3808
3809 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3810 {
3811         if (bp->hwrm_short_cmd_req_addr) {
3812                 struct pci_dev *pdev = bp->pdev;
3813
3814                 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3815                                   bp->hwrm_short_cmd_req_addr,
3816                                   bp->hwrm_short_cmd_req_dma_addr);
3817                 bp->hwrm_short_cmd_req_addr = NULL;
3818         }
3819 }
3820
3821 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3822 {
3823         struct pci_dev *pdev = bp->pdev;
3824
3825         if (bp->hwrm_short_cmd_req_addr)
3826                 return 0;
3827
3828         bp->hwrm_short_cmd_req_addr =
3829                 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3830                                    &bp->hwrm_short_cmd_req_dma_addr,
3831                                    GFP_KERNEL);
3832         if (!bp->hwrm_short_cmd_req_addr)
3833                 return -ENOMEM;
3834
3835         return 0;
3836 }
3837
3838 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
3839 {
3840         kfree(stats->hw_masks);
3841         stats->hw_masks = NULL;
3842         kfree(stats->sw_stats);
3843         stats->sw_stats = NULL;
3844         if (stats->hw_stats) {
3845                 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
3846                                   stats->hw_stats_map);
3847                 stats->hw_stats = NULL;
3848         }
3849 }
3850
3851 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
3852                                 bool alloc_masks)
3853 {
3854         stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
3855                                              &stats->hw_stats_map, GFP_KERNEL);
3856         if (!stats->hw_stats)
3857                 return -ENOMEM;
3858
3859         stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
3860         if (!stats->sw_stats)
3861                 goto stats_mem_err;
3862
3863         if (alloc_masks) {
3864                 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
3865                 if (!stats->hw_masks)
3866                         goto stats_mem_err;
3867         }
3868         return 0;
3869
3870 stats_mem_err:
3871         bnxt_free_stats_mem(bp, stats);
3872         return -ENOMEM;
3873 }
3874
3875 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
3876 {
3877         int i;
3878
3879         for (i = 0; i < count; i++)
3880                 mask_arr[i] = mask;
3881 }
3882
3883 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
3884 {
3885         int i;
3886
3887         for (i = 0; i < count; i++)
3888                 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
3889 }
3890
3891 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
3892                                     struct bnxt_stats_mem *stats)
3893 {
3894         struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
3895         struct hwrm_func_qstats_ext_input req = {0};
3896         __le64 *hw_masks;
3897         int rc;
3898
3899         if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
3900             !(bp->flags & BNXT_FLAG_CHIP_P5))
3901                 return -EOPNOTSUPP;
3902
3903         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1);
3904         req.fid = cpu_to_le16(0xffff);
3905         req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3906         mutex_lock(&bp->hwrm_cmd_lock);
3907         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3908         if (rc)
3909                 goto qstat_exit;
3910
3911         hw_masks = &resp->rx_ucast_pkts;
3912         bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
3913
3914 qstat_exit:
3915         mutex_unlock(&bp->hwrm_cmd_lock);
3916         return rc;
3917 }
3918
3919 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
3920 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
3921
3922 static void bnxt_init_stats(struct bnxt *bp)
3923 {
3924         struct bnxt_napi *bnapi = bp->bnapi[0];
3925         struct bnxt_cp_ring_info *cpr;
3926         struct bnxt_stats_mem *stats;
3927         __le64 *rx_stats, *tx_stats;
3928         int rc, rx_count, tx_count;
3929         u64 *rx_masks, *tx_masks;
3930         u64 mask;
3931         u8 flags;
3932
3933         cpr = &bnapi->cp_ring;
3934         stats = &cpr->stats;
3935         rc = bnxt_hwrm_func_qstat_ext(bp, stats);
3936         if (rc) {
3937                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3938                         mask = (1ULL << 48) - 1;
3939                 else
3940                         mask = -1ULL;
3941                 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
3942         }
3943         if (bp->flags & BNXT_FLAG_PORT_STATS) {
3944                 stats = &bp->port_stats;
3945                 rx_stats = stats->hw_stats;
3946                 rx_masks = stats->hw_masks;
3947                 rx_count = sizeof(struct rx_port_stats) / 8;
3948                 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3949                 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3950                 tx_count = sizeof(struct tx_port_stats) / 8;
3951
3952                 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
3953                 rc = bnxt_hwrm_port_qstats(bp, flags);
3954                 if (rc) {
3955                         mask = (1ULL << 40) - 1;
3956
3957                         bnxt_fill_masks(rx_masks, mask, rx_count);
3958                         bnxt_fill_masks(tx_masks, mask, tx_count);
3959                 } else {
3960                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3961                         bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
3962                         bnxt_hwrm_port_qstats(bp, 0);
3963                 }
3964         }
3965         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
3966                 stats = &bp->rx_port_stats_ext;
3967                 rx_stats = stats->hw_stats;
3968                 rx_masks = stats->hw_masks;
3969                 rx_count = sizeof(struct rx_port_stats_ext) / 8;
3970                 stats = &bp->tx_port_stats_ext;
3971                 tx_stats = stats->hw_stats;
3972                 tx_masks = stats->hw_masks;
3973                 tx_count = sizeof(struct tx_port_stats_ext) / 8;
3974
3975                 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3976                 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
3977                 if (rc) {
3978                         mask = (1ULL << 40) - 1;
3979
3980                         bnxt_fill_masks(rx_masks, mask, rx_count);
3981                         if (tx_stats)
3982                                 bnxt_fill_masks(tx_masks, mask, tx_count);
3983                 } else {
3984                         bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3985                         if (tx_stats)
3986                                 bnxt_copy_hw_masks(tx_masks, tx_stats,
3987                                                    tx_count);
3988                         bnxt_hwrm_port_qstats_ext(bp, 0);
3989                 }
3990         }
3991 }
3992
3993 static void bnxt_free_port_stats(struct bnxt *bp)
3994 {
3995         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3996         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3997
3998         bnxt_free_stats_mem(bp, &bp->port_stats);
3999         bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4000         bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4001 }
4002
4003 static void bnxt_free_ring_stats(struct bnxt *bp)
4004 {
4005         int i;
4006
4007         if (!bp->bnapi)
4008                 return;
4009
4010         for (i = 0; i < bp->cp_nr_rings; i++) {
4011                 struct bnxt_napi *bnapi = bp->bnapi[i];
4012                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4013
4014                 bnxt_free_stats_mem(bp, &cpr->stats);
4015         }
4016 }
4017
4018 static int bnxt_alloc_stats(struct bnxt *bp)
4019 {
4020         u32 size, i;
4021         int rc;
4022
4023         size = bp->hw_ring_stats_size;
4024
4025         for (i = 0; i < bp->cp_nr_rings; i++) {
4026                 struct bnxt_napi *bnapi = bp->bnapi[i];
4027                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4028
4029                 cpr->stats.len = size;
4030                 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4031                 if (rc)
4032                         return rc;
4033
4034                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4035         }
4036
4037         if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4038                 return 0;
4039
4040         if (bp->port_stats.hw_stats)
4041                 goto alloc_ext_stats;
4042
4043         bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4044         rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4045         if (rc)
4046                 return rc;
4047
4048         bp->flags |= BNXT_FLAG_PORT_STATS;
4049
4050 alloc_ext_stats:
4051         /* Display extended statistics only if FW supports it */
4052         if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4053                 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4054                         return 0;
4055
4056         if (bp->rx_port_stats_ext.hw_stats)
4057                 goto alloc_tx_ext_stats;
4058
4059         bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4060         rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4061         /* Extended stats are optional */
4062         if (rc)
4063                 return 0;
4064
4065 alloc_tx_ext_stats:
4066         if (bp->tx_port_stats_ext.hw_stats)
4067                 return 0;
4068
4069         if (bp->hwrm_spec_code >= 0x10902 ||
4070             (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4071                 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4072                 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4073                 /* Extended stats are optional */
4074                 if (rc)
4075                         return 0;
4076         }
4077         bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4078         return 0;
4079 }
4080
4081 static void bnxt_clear_ring_indices(struct bnxt *bp)
4082 {
4083         int i;
4084
4085         if (!bp->bnapi)
4086                 return;
4087
4088         for (i = 0; i < bp->cp_nr_rings; i++) {
4089                 struct bnxt_napi *bnapi = bp->bnapi[i];
4090                 struct bnxt_cp_ring_info *cpr;
4091                 struct bnxt_rx_ring_info *rxr;
4092                 struct bnxt_tx_ring_info *txr;
4093
4094                 if (!bnapi)
4095                         continue;
4096
4097                 cpr = &bnapi->cp_ring;
4098                 cpr->cp_raw_cons = 0;
4099
4100                 txr = bnapi->tx_ring;
4101                 if (txr) {
4102                         txr->tx_prod = 0;
4103                         txr->tx_cons = 0;
4104                 }
4105
4106                 rxr = bnapi->rx_ring;
4107                 if (rxr) {
4108                         rxr->rx_prod = 0;
4109                         rxr->rx_agg_prod = 0;
4110                         rxr->rx_sw_agg_prod = 0;
4111                         rxr->rx_next_cons = 0;
4112                 }
4113         }
4114 }
4115
4116 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4117 {
4118 #ifdef CONFIG_RFS_ACCEL
4119         int i;
4120
4121         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
4122          * safe to delete the hash table.
4123          */
4124         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4125                 struct hlist_head *head;
4126                 struct hlist_node *tmp;
4127                 struct bnxt_ntuple_filter *fltr;
4128
4129                 head = &bp->ntp_fltr_hash_tbl[i];
4130                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4131                         hlist_del(&fltr->hash);
4132                         kfree(fltr);
4133                 }
4134         }
4135         if (irq_reinit) {
4136                 kfree(bp->ntp_fltr_bmap);
4137                 bp->ntp_fltr_bmap = NULL;
4138         }
4139         bp->ntp_fltr_count = 0;
4140 #endif
4141 }
4142
4143 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4144 {
4145 #ifdef CONFIG_RFS_ACCEL
4146         int i, rc = 0;
4147
4148         if (!(bp->flags & BNXT_FLAG_RFS))
4149                 return 0;
4150
4151         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4152                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4153
4154         bp->ntp_fltr_count = 0;
4155         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
4156                                     sizeof(long),
4157                                     GFP_KERNEL);
4158
4159         if (!bp->ntp_fltr_bmap)
4160                 rc = -ENOMEM;
4161
4162         return rc;
4163 #else
4164         return 0;
4165 #endif
4166 }
4167
4168 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4169 {
4170         bnxt_free_vnic_attributes(bp);
4171         bnxt_free_tx_rings(bp);
4172         bnxt_free_rx_rings(bp);
4173         bnxt_free_cp_rings(bp);
4174         bnxt_free_ntp_fltrs(bp, irq_re_init);
4175         if (irq_re_init) {
4176                 bnxt_free_ring_stats(bp);
4177                 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
4178                     test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4179                         bnxt_free_port_stats(bp);
4180                 bnxt_free_ring_grps(bp);
4181                 bnxt_free_vnics(bp);
4182                 kfree(bp->tx_ring_map);
4183                 bp->tx_ring_map = NULL;
4184                 kfree(bp->tx_ring);
4185                 bp->tx_ring = NULL;
4186                 kfree(bp->rx_ring);
4187                 bp->rx_ring = NULL;
4188                 kfree(bp->bnapi);
4189                 bp->bnapi = NULL;
4190         } else {
4191                 bnxt_clear_ring_indices(bp);
4192         }
4193 }
4194
4195 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4196 {
4197         int i, j, rc, size, arr_size;
4198         void *bnapi;
4199
4200         if (irq_re_init) {
4201                 /* Allocate bnapi mem pointer array and mem block for
4202                  * all queues
4203                  */
4204                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4205                                 bp->cp_nr_rings);
4206                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4207                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4208                 if (!bnapi)
4209                         return -ENOMEM;
4210
4211                 bp->bnapi = bnapi;
4212                 bnapi += arr_size;
4213                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4214                         bp->bnapi[i] = bnapi;
4215                         bp->bnapi[i]->index = i;
4216                         bp->bnapi[i]->bp = bp;
4217                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4218                                 struct bnxt_cp_ring_info *cpr =
4219                                         &bp->bnapi[i]->cp_ring;
4220
4221                                 cpr->cp_ring_struct.ring_mem.flags =
4222                                         BNXT_RMEM_RING_PTE_FLAG;
4223                         }
4224                 }
4225
4226                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
4227                                       sizeof(struct bnxt_rx_ring_info),
4228                                       GFP_KERNEL);
4229                 if (!bp->rx_ring)
4230                         return -ENOMEM;
4231
4232                 for (i = 0; i < bp->rx_nr_rings; i++) {
4233                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4234
4235                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4236                                 rxr->rx_ring_struct.ring_mem.flags =
4237                                         BNXT_RMEM_RING_PTE_FLAG;
4238                                 rxr->rx_agg_ring_struct.ring_mem.flags =
4239                                         BNXT_RMEM_RING_PTE_FLAG;
4240                         }
4241                         rxr->bnapi = bp->bnapi[i];
4242                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4243                 }
4244
4245                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4246                                       sizeof(struct bnxt_tx_ring_info),
4247                                       GFP_KERNEL);
4248                 if (!bp->tx_ring)
4249                         return -ENOMEM;
4250
4251                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4252                                           GFP_KERNEL);
4253
4254                 if (!bp->tx_ring_map)
4255                         return -ENOMEM;
4256
4257                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4258                         j = 0;
4259                 else
4260                         j = bp->rx_nr_rings;
4261
4262                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4263                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4264
4265                         if (bp->flags & BNXT_FLAG_CHIP_P5)
4266                                 txr->tx_ring_struct.ring_mem.flags =
4267                                         BNXT_RMEM_RING_PTE_FLAG;
4268                         txr->bnapi = bp->bnapi[j];
4269                         bp->bnapi[j]->tx_ring = txr;
4270                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4271                         if (i >= bp->tx_nr_rings_xdp) {
4272                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
4273                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
4274                         } else {
4275                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4276                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4277                         }
4278                 }
4279
4280                 rc = bnxt_alloc_stats(bp);
4281                 if (rc)
4282                         goto alloc_mem_err;
4283                 bnxt_init_stats(bp);
4284
4285                 rc = bnxt_alloc_ntp_fltrs(bp);
4286                 if (rc)
4287                         goto alloc_mem_err;
4288
4289                 rc = bnxt_alloc_vnics(bp);
4290                 if (rc)
4291                         goto alloc_mem_err;
4292         }
4293
4294         bnxt_init_ring_struct(bp);
4295
4296         rc = bnxt_alloc_rx_rings(bp);
4297         if (rc)
4298                 goto alloc_mem_err;
4299
4300         rc = bnxt_alloc_tx_rings(bp);
4301         if (rc)
4302                 goto alloc_mem_err;
4303
4304         rc = bnxt_alloc_cp_rings(bp);
4305         if (rc)
4306                 goto alloc_mem_err;
4307
4308         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4309                                   BNXT_VNIC_UCAST_FLAG;
4310         rc = bnxt_alloc_vnic_attributes(bp);
4311         if (rc)
4312                 goto alloc_mem_err;
4313         return 0;
4314
4315 alloc_mem_err:
4316         bnxt_free_mem(bp, true);
4317         return rc;
4318 }
4319
4320 static void bnxt_disable_int(struct bnxt *bp)
4321 {
4322         int i;
4323
4324         if (!bp->bnapi)
4325                 return;
4326
4327         for (i = 0; i < bp->cp_nr_rings; i++) {
4328                 struct bnxt_napi *bnapi = bp->bnapi[i];
4329                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4330                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4331
4332                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4333                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4334         }
4335 }
4336
4337 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4338 {
4339         struct bnxt_napi *bnapi = bp->bnapi[n];
4340         struct bnxt_cp_ring_info *cpr;
4341
4342         cpr = &bnapi->cp_ring;
4343         return cpr->cp_ring_struct.map_idx;
4344 }
4345
4346 static void bnxt_disable_int_sync(struct bnxt *bp)
4347 {
4348         int i;
4349
4350         if (!bp->irq_tbl)
4351                 return;
4352
4353         atomic_inc(&bp->intr_sem);
4354
4355         bnxt_disable_int(bp);
4356         for (i = 0; i < bp->cp_nr_rings; i++) {
4357                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4358
4359                 synchronize_irq(bp->irq_tbl[map_idx].vector);
4360         }
4361 }
4362
4363 static void bnxt_enable_int(struct bnxt *bp)
4364 {
4365         int i;
4366
4367         atomic_set(&bp->intr_sem, 0);
4368         for (i = 0; i < bp->cp_nr_rings; i++) {
4369                 struct bnxt_napi *bnapi = bp->bnapi[i];
4370                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4371
4372                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4373         }
4374 }
4375
4376 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4377                             u16 cmpl_ring, u16 target_id)
4378 {
4379         struct input *req = request;
4380
4381         req->req_type = cpu_to_le16(req_type);
4382         req->cmpl_ring = cpu_to_le16(cmpl_ring);
4383         req->target_id = cpu_to_le16(target_id);
4384         if (bnxt_kong_hwrm_message(bp, req))
4385                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4386         else
4387                 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4388 }
4389
4390 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4391 {
4392         switch (hwrm_err) {
4393         case HWRM_ERR_CODE_SUCCESS:
4394                 return 0;
4395         case HWRM_ERR_CODE_RESOURCE_LOCKED:
4396                 return -EROFS;
4397         case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4398                 return -EACCES;
4399         case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4400                 return -ENOSPC;
4401         case HWRM_ERR_CODE_INVALID_PARAMS:
4402         case HWRM_ERR_CODE_INVALID_FLAGS:
4403         case HWRM_ERR_CODE_INVALID_ENABLES:
4404         case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4405         case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4406                 return -EINVAL;
4407         case HWRM_ERR_CODE_NO_BUFFER:
4408                 return -ENOMEM;
4409         case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4410         case HWRM_ERR_CODE_BUSY:
4411                 return -EAGAIN;
4412         case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4413                 return -EOPNOTSUPP;
4414         default:
4415                 return -EIO;
4416         }
4417 }
4418
4419 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4420                                  int timeout, bool silent)
4421 {
4422         int i, intr_process, rc, tmo_count;
4423         struct input *req = msg;
4424         u32 *data = msg;
4425         u8 *valid;
4426         u16 cp_ring_id, len = 0;
4427         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4428         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4429         struct hwrm_short_input short_input = {0};
4430         u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4431         u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4432         u16 dst = BNXT_HWRM_CHNL_CHIMP;
4433
4434         if (BNXT_NO_FW_ACCESS(bp) &&
4435             le16_to_cpu(req->req_type) != HWRM_FUNC_RESET)
4436                 return -EBUSY;
4437
4438         if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4439                 if (msg_len > bp->hwrm_max_ext_req_len ||
4440                     !bp->hwrm_short_cmd_req_addr)
4441                         return -EINVAL;
4442         }
4443
4444         if (bnxt_hwrm_kong_chnl(bp, req)) {
4445                 dst = BNXT_HWRM_CHNL_KONG;
4446                 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4447                 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4448                 resp = bp->hwrm_cmd_kong_resp_addr;
4449         }
4450
4451         memset(resp, 0, PAGE_SIZE);
4452         cp_ring_id = le16_to_cpu(req->cmpl_ring);
4453         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4454
4455         req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4456         /* currently supports only one outstanding message */
4457         if (intr_process)
4458                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4459
4460         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4461             msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4462                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4463                 u16 max_msg_len;
4464
4465                 /* Set boundary for maximum extended request length for short
4466                  * cmd format. If passed up from device use the max supported
4467                  * internal req length.
4468                  */
4469                 max_msg_len = bp->hwrm_max_ext_req_len;
4470
4471                 memcpy(short_cmd_req, req, msg_len);
4472                 if (msg_len < max_msg_len)
4473                         memset(short_cmd_req + msg_len, 0,
4474                                max_msg_len - msg_len);
4475
4476                 short_input.req_type = req->req_type;
4477                 short_input.signature =
4478                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4479                 short_input.size = cpu_to_le16(msg_len);
4480                 short_input.req_addr =
4481                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4482
4483                 data = (u32 *)&short_input;
4484                 msg_len = sizeof(short_input);
4485
4486                 /* Sync memory write before updating doorbell */
4487                 wmb();
4488
4489                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4490         }
4491
4492         /* Write request msg to hwrm channel */
4493         __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4494
4495         for (i = msg_len; i < max_req_len; i += 4)
4496                 writel(0, bp->bar0 + bar_offset + i);
4497
4498         /* Ring channel doorbell */
4499         writel(1, bp->bar0 + doorbell_offset);
4500
4501         if (!pci_is_enabled(bp->pdev))
4502                 return -ENODEV;
4503
4504         if (!timeout)
4505                 timeout = DFLT_HWRM_CMD_TIMEOUT;
4506         /* Limit timeout to an upper limit */
4507         timeout = min(timeout, HWRM_CMD_MAX_TIMEOUT);
4508         /* convert timeout to usec */
4509         timeout *= 1000;
4510
4511         i = 0;
4512         /* Short timeout for the first few iterations:
4513          * number of loops = number of loops for short timeout +
4514          * number of loops for standard timeout.
4515          */
4516         tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4517         timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4518         tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4519
4520         if (intr_process) {
4521                 u16 seq_id = bp->hwrm_intr_seq_id;
4522
4523                 /* Wait until hwrm response cmpl interrupt is processed */
4524                 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4525                        i++ < tmo_count) {
4526                         /* Abort the wait for completion if the FW health
4527                          * check has failed.
4528                          */
4529                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4530                                 return -EBUSY;
4531                         /* on first few passes, just barely sleep */
4532                         if (i < HWRM_SHORT_TIMEOUT_COUNTER) {
4533                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4534                                              HWRM_SHORT_MAX_TIMEOUT);
4535                         } else {
4536                                 if (HWRM_WAIT_MUST_ABORT(bp, req))
4537                                         break;
4538                                 usleep_range(HWRM_MIN_TIMEOUT,
4539                                              HWRM_MAX_TIMEOUT);
4540                         }
4541                 }
4542
4543                 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4544                         if (!silent)
4545                                 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4546                                            le16_to_cpu(req->req_type));
4547                         return -EBUSY;
4548                 }
4549                 len = le16_to_cpu(resp->resp_len);
4550                 valid = ((u8 *)resp) + len - 1;
4551         } else {
4552                 int j;
4553
4554                 /* Check if response len is updated */
4555                 for (i = 0; i < tmo_count; i++) {
4556                         /* Abort the wait for completion if the FW health
4557                          * check has failed.
4558                          */
4559                         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4560                                 return -EBUSY;
4561                         len = le16_to_cpu(resp->resp_len);
4562                         if (len)
4563                                 break;
4564                         /* on first few passes, just barely sleep */
4565                         if (i < HWRM_SHORT_TIMEOUT_COUNTER) {
4566                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4567                                              HWRM_SHORT_MAX_TIMEOUT);
4568                         } else {
4569                                 if (HWRM_WAIT_MUST_ABORT(bp, req))
4570                                         goto timeout_abort;
4571                                 usleep_range(HWRM_MIN_TIMEOUT,
4572                                              HWRM_MAX_TIMEOUT);
4573                         }
4574                 }
4575
4576                 if (i >= tmo_count) {
4577 timeout_abort:
4578                         if (!silent)
4579                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4580                                            HWRM_TOTAL_TIMEOUT(i),
4581                                            le16_to_cpu(req->req_type),
4582                                            le16_to_cpu(req->seq_id), len);
4583                         return -EBUSY;
4584                 }
4585
4586                 /* Last byte of resp contains valid bit */
4587                 valid = ((u8 *)resp) + len - 1;
4588                 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4589                         /* make sure we read from updated DMA memory */
4590                         dma_rmb();
4591                         if (*valid)
4592                                 break;
4593                         usleep_range(1, 5);
4594                 }
4595
4596                 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4597                         if (!silent)
4598                                 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4599                                            HWRM_TOTAL_TIMEOUT(i),
4600                                            le16_to_cpu(req->req_type),
4601                                            le16_to_cpu(req->seq_id), len,
4602                                            *valid);
4603                         return -EBUSY;
4604                 }
4605         }
4606
4607         /* Zero valid bit for compatibility.  Valid bit in an older spec
4608          * may become a new field in a newer spec.  We must make sure that
4609          * a new field not implemented by old spec will read zero.
4610          */
4611         *valid = 0;
4612         rc = le16_to_cpu(resp->error_code);
4613         if (rc && !silent)
4614                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4615                            le16_to_cpu(resp->req_type),
4616                            le16_to_cpu(resp->seq_id), rc);
4617         return bnxt_hwrm_to_stderr(rc);
4618 }
4619
4620 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4621 {
4622         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4623 }
4624
4625 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4626                               int timeout)
4627 {
4628         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4629 }
4630
4631 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4632 {
4633         int rc;
4634
4635         mutex_lock(&bp->hwrm_cmd_lock);
4636         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4637         mutex_unlock(&bp->hwrm_cmd_lock);
4638         return rc;
4639 }
4640
4641 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4642                              int timeout)
4643 {
4644         int rc;
4645
4646         mutex_lock(&bp->hwrm_cmd_lock);
4647         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4648         mutex_unlock(&bp->hwrm_cmd_lock);
4649         return rc;
4650 }
4651
4652 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4653                             bool async_only)
4654 {
4655         struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4656         struct hwrm_func_drv_rgtr_input req = {0};
4657         DECLARE_BITMAP(async_events_bmap, 256);
4658         u32 *events = (u32 *)async_events_bmap;
4659         u32 flags;
4660         int rc, i;
4661
4662         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4663
4664         req.enables =
4665                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4666                             FUNC_DRV_RGTR_REQ_ENABLES_VER |
4667                             FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4668
4669         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4670         flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4671         if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4672                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4673         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4674                 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4675                          FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4676         req.flags = cpu_to_le32(flags);
4677         req.ver_maj_8b = DRV_VER_MAJ;
4678         req.ver_min_8b = DRV_VER_MIN;
4679         req.ver_upd_8b = DRV_VER_UPD;
4680         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4681         req.ver_min = cpu_to_le16(DRV_VER_MIN);
4682         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4683
4684         if (BNXT_PF(bp)) {
4685                 u32 data[8];
4686                 int i;
4687
4688                 memset(data, 0, sizeof(data));
4689                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4690                         u16 cmd = bnxt_vf_req_snif[i];
4691                         unsigned int bit, idx;
4692
4693                         idx = cmd / 32;
4694                         bit = cmd % 32;
4695                         data[idx] |= 1 << bit;
4696                 }
4697
4698                 for (i = 0; i < 8; i++)
4699                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4700
4701                 req.enables |=
4702                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4703         }
4704
4705         if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4706                 req.flags |= cpu_to_le32(
4707                         FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4708
4709         memset(async_events_bmap, 0, sizeof(async_events_bmap));
4710         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4711                 u16 event_id = bnxt_async_events_arr[i];
4712
4713                 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4714                     !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4715                         continue;
4716                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4717         }
4718         if (bmap && bmap_size) {
4719                 for (i = 0; i < bmap_size; i++) {
4720                         if (test_bit(i, bmap))
4721                                 __set_bit(i, async_events_bmap);
4722                 }
4723         }
4724         for (i = 0; i < 8; i++)
4725                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4726
4727         if (async_only)
4728                 req.enables =
4729                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4730
4731         mutex_lock(&bp->hwrm_cmd_lock);
4732         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4733         if (!rc) {
4734                 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4735                 if (resp->flags &
4736                     cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4737                         bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4738         }
4739         mutex_unlock(&bp->hwrm_cmd_lock);
4740         return rc;
4741 }
4742
4743 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4744 {
4745         struct hwrm_func_drv_unrgtr_input req = {0};
4746
4747         if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4748                 return 0;
4749
4750         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4751         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4752 }
4753
4754 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4755 {
4756         u32 rc = 0;
4757         struct hwrm_tunnel_dst_port_free_input req = {0};
4758
4759         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4760         req.tunnel_type = tunnel_type;
4761
4762         switch (tunnel_type) {
4763         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4764                 req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4765                 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4766                 break;
4767         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4768                 req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4769                 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4770                 break;
4771         default:
4772                 break;
4773         }
4774
4775         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4776         if (rc)
4777                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4778                            rc);
4779         return rc;
4780 }
4781
4782 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4783                                            u8 tunnel_type)
4784 {
4785         u32 rc = 0;
4786         struct hwrm_tunnel_dst_port_alloc_input req = {0};
4787         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4788
4789         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4790
4791         req.tunnel_type = tunnel_type;
4792         req.tunnel_dst_port_val = port;
4793
4794         mutex_lock(&bp->hwrm_cmd_lock);
4795         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4796         if (rc) {
4797                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4798                            rc);
4799                 goto err_out;
4800         }
4801
4802         switch (tunnel_type) {
4803         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4804                 bp->vxlan_fw_dst_port_id =
4805                         le16_to_cpu(resp->tunnel_dst_port_id);
4806                 break;
4807         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4808                 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4809                 break;
4810         default:
4811                 break;
4812         }
4813
4814 err_out:
4815         mutex_unlock(&bp->hwrm_cmd_lock);
4816         return rc;
4817 }
4818
4819 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4820 {
4821         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4822         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4823
4824         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4825         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4826
4827         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4828         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4829         req.mask = cpu_to_le32(vnic->rx_mask);
4830         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4831 }
4832
4833 #ifdef CONFIG_RFS_ACCEL
4834 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4835                                             struct bnxt_ntuple_filter *fltr)
4836 {
4837         struct hwrm_cfa_ntuple_filter_free_input req = {0};
4838
4839         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4840         req.ntuple_filter_id = fltr->filter_id;
4841         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4842 }
4843
4844 #define BNXT_NTP_FLTR_FLAGS                                     \
4845         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4846          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4847          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4848          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4849          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4850          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4851          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4852          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4853          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4854          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4855          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4856          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4857          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4858          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4859
4860 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4861                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4862
4863 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4864                                              struct bnxt_ntuple_filter *fltr)
4865 {
4866         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4867         struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4868         struct flow_keys *keys = &fltr->fkeys;
4869         struct bnxt_vnic_info *vnic;
4870         u32 flags = 0;
4871         int rc = 0;
4872
4873         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4874         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4875
4876         if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4877                 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4878                 req.dst_id = cpu_to_le16(fltr->rxq);
4879         } else {
4880                 vnic = &bp->vnic_info[fltr->rxq + 1];
4881                 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4882         }
4883         req.flags = cpu_to_le32(flags);
4884         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4885
4886         req.ethertype = htons(ETH_P_IP);
4887         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4888         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4889         req.ip_protocol = keys->basic.ip_proto;
4890
4891         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4892                 int i;
4893
4894                 req.ethertype = htons(ETH_P_IPV6);
4895                 req.ip_addr_type =
4896                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4897                 *(struct in6_addr *)&req.src_ipaddr[0] =
4898                         keys->addrs.v6addrs.src;
4899                 *(struct in6_addr *)&req.dst_ipaddr[0] =
4900                         keys->addrs.v6addrs.dst;
4901                 for (i = 0; i < 4; i++) {
4902                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4903                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4904                 }
4905         } else {
4906                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4907                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4908                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4909                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4910         }
4911         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4912                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4913                 req.tunnel_type =
4914                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4915         }
4916
4917         req.src_port = keys->ports.src;
4918         req.src_port_mask = cpu_to_be16(0xffff);
4919         req.dst_port = keys->ports.dst;
4920         req.dst_port_mask = cpu_to_be16(0xffff);
4921
4922         mutex_lock(&bp->hwrm_cmd_lock);
4923         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4924         if (!rc) {
4925                 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4926                 fltr->filter_id = resp->ntuple_filter_id;
4927         }
4928         mutex_unlock(&bp->hwrm_cmd_lock);
4929         return rc;
4930 }
4931 #endif
4932
4933 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4934                                      u8 *mac_addr)
4935 {
4936         u32 rc = 0;
4937         struct hwrm_cfa_l2_filter_alloc_input req = {0};
4938         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4939
4940         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4941         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4942         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4943                 req.flags |=
4944                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4945         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4946         req.enables =
4947                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4948                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4949                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4950         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4951         req.l2_addr_mask[0] = 0xff;
4952         req.l2_addr_mask[1] = 0xff;
4953         req.l2_addr_mask[2] = 0xff;
4954         req.l2_addr_mask[3] = 0xff;
4955         req.l2_addr_mask[4] = 0xff;
4956         req.l2_addr_mask[5] = 0xff;
4957
4958         mutex_lock(&bp->hwrm_cmd_lock);
4959         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4960         if (!rc)
4961                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4962                                                         resp->l2_filter_id;
4963         mutex_unlock(&bp->hwrm_cmd_lock);
4964         return rc;
4965 }
4966
4967 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4968 {
4969         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4970         int rc = 0;
4971
4972         /* Any associated ntuple filters will also be cleared by firmware. */
4973         mutex_lock(&bp->hwrm_cmd_lock);
4974         for (i = 0; i < num_of_vnics; i++) {
4975                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4976
4977                 for (j = 0; j < vnic->uc_filter_count; j++) {
4978                         struct hwrm_cfa_l2_filter_free_input req = {0};
4979
4980                         bnxt_hwrm_cmd_hdr_init(bp, &req,
4981                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
4982
4983                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
4984
4985                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4986                                                 HWRM_CMD_TIMEOUT);
4987                 }
4988                 vnic->uc_filter_count = 0;
4989         }
4990         mutex_unlock(&bp->hwrm_cmd_lock);
4991
4992         return rc;
4993 }
4994
4995 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4996 {
4997         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4998         u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4999         struct hwrm_vnic_tpa_cfg_input req = {0};
5000
5001         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
5002                 return 0;
5003
5004         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
5005
5006         if (tpa_flags) {
5007                 u16 mss = bp->dev->mtu - 40;
5008                 u32 nsegs, n, segs = 0, flags;
5009
5010                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
5011                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
5012                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
5013                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
5014                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
5015                 if (tpa_flags & BNXT_FLAG_GRO)
5016                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
5017
5018                 req.flags = cpu_to_le32(flags);
5019
5020                 req.enables =
5021                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
5022                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
5023                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
5024
5025                 /* Number of segs are log2 units, and first packet is not
5026                  * included as part of this units.
5027                  */
5028                 if (mss <= BNXT_RX_PAGE_SIZE) {
5029                         n = BNXT_RX_PAGE_SIZE / mss;
5030                         nsegs = (MAX_SKB_FRAGS - 1) * n;
5031                 } else {
5032                         n = mss / BNXT_RX_PAGE_SIZE;
5033                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
5034                                 n++;
5035                         nsegs = (MAX_SKB_FRAGS - n) / n;
5036                 }
5037
5038                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5039                         segs = MAX_TPA_SEGS_P5;
5040                         max_aggs = bp->max_tpa;
5041                 } else {
5042                         segs = ilog2(nsegs);
5043                 }
5044                 req.max_agg_segs = cpu_to_le16(segs);
5045                 req.max_aggs = cpu_to_le16(max_aggs);
5046
5047                 req.min_agg_len = cpu_to_le32(512);
5048         }
5049         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5050
5051         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5052 }
5053
5054 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
5055 {
5056         struct bnxt_ring_grp_info *grp_info;
5057
5058         grp_info = &bp->grp_info[ring->grp_idx];
5059         return grp_info->cp_fw_ring_id;
5060 }
5061
5062 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
5063 {
5064         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5065                 struct bnxt_napi *bnapi = rxr->bnapi;
5066                 struct bnxt_cp_ring_info *cpr;
5067
5068                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
5069                 return cpr->cp_ring_struct.fw_ring_id;
5070         } else {
5071                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
5072         }
5073 }
5074
5075 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
5076 {
5077         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5078                 struct bnxt_napi *bnapi = txr->bnapi;
5079                 struct bnxt_cp_ring_info *cpr;
5080
5081                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
5082                 return cpr->cp_ring_struct.fw_ring_id;
5083         } else {
5084                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
5085         }
5086 }
5087
5088 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5089 {
5090         int entries;
5091
5092         if (bp->flags & BNXT_FLAG_CHIP_P5)
5093                 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5094         else
5095                 entries = HW_HASH_INDEX_SIZE;
5096
5097         bp->rss_indir_tbl_entries = entries;
5098         bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5099                                           GFP_KERNEL);
5100         if (!bp->rss_indir_tbl)
5101                 return -ENOMEM;
5102         return 0;
5103 }
5104
5105 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5106 {
5107         u16 max_rings, max_entries, pad, i;
5108
5109         if (!bp->rx_nr_rings)
5110                 return;
5111
5112         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5113                 max_rings = bp->rx_nr_rings - 1;
5114         else
5115                 max_rings = bp->rx_nr_rings;
5116
5117         max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5118
5119         for (i = 0; i < max_entries; i++)
5120                 bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5121
5122         pad = bp->rss_indir_tbl_entries - max_entries;
5123         if (pad)
5124                 memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5125 }
5126
5127 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5128 {
5129         u16 i, tbl_size, max_ring = 0;
5130
5131         if (!bp->rss_indir_tbl)
5132                 return 0;
5133
5134         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5135         for (i = 0; i < tbl_size; i++)
5136                 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5137         return max_ring;
5138 }
5139
5140 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5141 {
5142         if (bp->flags & BNXT_FLAG_CHIP_P5)
5143                 return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5144         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5145                 return 2;
5146         return 1;
5147 }
5148
5149 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5150 {
5151         bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5152         u16 i, j;
5153
5154         /* Fill the RSS indirection table with ring group ids */
5155         for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5156                 if (!no_rss)
5157                         j = bp->rss_indir_tbl[i];
5158                 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5159         }
5160 }
5161
5162 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5163                                       struct bnxt_vnic_info *vnic)
5164 {
5165         __le16 *ring_tbl = vnic->rss_table;
5166         struct bnxt_rx_ring_info *rxr;
5167         u16 tbl_size, i;
5168
5169         tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5170
5171         for (i = 0; i < tbl_size; i++) {
5172                 u16 ring_id, j;
5173
5174                 j = bp->rss_indir_tbl[i];
5175                 rxr = &bp->rx_ring[j];
5176
5177                 ring_id = rxr->rx_ring_struct.fw_ring_id;
5178                 *ring_tbl++ = cpu_to_le16(ring_id);
5179                 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5180                 *ring_tbl++ = cpu_to_le16(ring_id);
5181         }
5182 }
5183
5184 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5185 {
5186         if (bp->flags & BNXT_FLAG_CHIP_P5)
5187                 __bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5188         else
5189                 __bnxt_fill_hw_rss_tbl(bp, vnic);
5190 }
5191
5192 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5193 {
5194         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5195         struct hwrm_vnic_rss_cfg_input req = {0};
5196
5197         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5198             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5199                 return 0;
5200
5201         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5202         if (set_rss) {
5203                 bnxt_fill_hw_rss_tbl(bp, vnic);
5204                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5205                 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5206                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5207                 req.hash_key_tbl_addr =
5208                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
5209         }
5210         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5211         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5212 }
5213
5214 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5215 {
5216         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5217         struct hwrm_vnic_rss_cfg_input req = {0};
5218         dma_addr_t ring_tbl_map;
5219         u32 i, nr_ctxs;
5220
5221         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5222         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5223         if (!set_rss) {
5224                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5225                 return 0;
5226         }
5227         bnxt_fill_hw_rss_tbl(bp, vnic);
5228         req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5229         req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5230         req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5231         ring_tbl_map = vnic->rss_table_dma_addr;
5232         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5233         for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5234                 int rc;
5235
5236                 req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5237                 req.ring_table_pair_index = i;
5238                 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5239                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5240                 if (rc)
5241                         return rc;
5242         }
5243         return 0;
5244 }
5245
5246 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5247 {
5248         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5249         struct hwrm_vnic_plcmodes_cfg_input req = {0};
5250
5251         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
5252         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
5253                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5254                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5255         req.enables =
5256                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
5257                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5258         /* thresholds not implemented in firmware yet */
5259         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5260         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5261         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5262         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5263 }
5264
5265 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5266                                         u16 ctx_idx)
5267 {
5268         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
5269
5270         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
5271         req.rss_cos_lb_ctx_id =
5272                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5273
5274         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5275         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5276 }
5277
5278 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5279 {
5280         int i, j;
5281
5282         for (i = 0; i < bp->nr_vnics; i++) {
5283                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5284
5285                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5286                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5287                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5288                 }
5289         }
5290         bp->rsscos_nr_ctxs = 0;
5291 }
5292
5293 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5294 {
5295         int rc;
5296         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
5297         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
5298                                                 bp->hwrm_cmd_resp_addr;
5299
5300         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
5301                                -1);
5302
5303         mutex_lock(&bp->hwrm_cmd_lock);
5304         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5305         if (!rc)
5306                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5307                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
5308         mutex_unlock(&bp->hwrm_cmd_lock);
5309
5310         return rc;
5311 }
5312
5313 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5314 {
5315         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5316                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5317         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5318 }
5319
5320 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5321 {
5322         unsigned int ring = 0, grp_idx;
5323         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5324         struct hwrm_vnic_cfg_input req = {0};
5325         u16 def_vlan = 0;
5326
5327         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
5328
5329         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5330                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5331
5332                 req.default_rx_ring_id =
5333                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5334                 req.default_cmpl_ring_id =
5335                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5336                 req.enables =
5337                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5338                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5339                 goto vnic_mru;
5340         }
5341         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5342         /* Only RSS support for now TBD: COS & LB */
5343         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5344                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5345                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5346                                            VNIC_CFG_REQ_ENABLES_MRU);
5347         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5348                 req.rss_rule =
5349                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5350                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5351                                            VNIC_CFG_REQ_ENABLES_MRU);
5352                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5353         } else {
5354                 req.rss_rule = cpu_to_le16(0xffff);
5355         }
5356
5357         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5358             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5359                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5360                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5361         } else {
5362                 req.cos_rule = cpu_to_le16(0xffff);
5363         }
5364
5365         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5366                 ring = 0;
5367         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5368                 ring = vnic_id - 1;
5369         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5370                 ring = bp->rx_nr_rings - 1;
5371
5372         grp_idx = bp->rx_ring[ring].bnapi->index;
5373         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5374         req.lb_rule = cpu_to_le16(0xffff);
5375 vnic_mru:
5376         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5377
5378         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5379 #ifdef CONFIG_BNXT_SRIOV
5380         if (BNXT_VF(bp))
5381                 def_vlan = bp->vf.vlan;
5382 #endif
5383         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5384                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5385         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5386                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5387
5388         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5389 }
5390
5391 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5392 {
5393         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5394                 struct hwrm_vnic_free_input req = {0};
5395
5396                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5397                 req.vnic_id =
5398                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5399
5400                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5401                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5402         }
5403 }
5404
5405 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5406 {
5407         u16 i;
5408
5409         for (i = 0; i < bp->nr_vnics; i++)
5410                 bnxt_hwrm_vnic_free_one(bp, i);
5411 }
5412
5413 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5414                                 unsigned int start_rx_ring_idx,
5415                                 unsigned int nr_rings)
5416 {
5417         int rc = 0;
5418         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5419         struct hwrm_vnic_alloc_input req = {0};
5420         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5421         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5422
5423         if (bp->flags & BNXT_FLAG_CHIP_P5)
5424                 goto vnic_no_ring_grps;
5425
5426         /* map ring groups to this vnic */
5427         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5428                 grp_idx = bp->rx_ring[i].bnapi->index;
5429                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5430                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5431                                    j, nr_rings);
5432                         break;
5433                 }
5434                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5435         }
5436
5437 vnic_no_ring_grps:
5438         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5439                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5440         if (vnic_id == 0)
5441                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5442
5443         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5444
5445         mutex_lock(&bp->hwrm_cmd_lock);
5446         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5447         if (!rc)
5448                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5449         mutex_unlock(&bp->hwrm_cmd_lock);
5450         return rc;
5451 }
5452
5453 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5454 {
5455         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5456         struct hwrm_vnic_qcaps_input req = {0};
5457         int rc;
5458
5459         bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5460         bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5461         if (bp->hwrm_spec_code < 0x10600)
5462                 return 0;
5463
5464         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5465         mutex_lock(&bp->hwrm_cmd_lock);
5466         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5467         if (!rc) {
5468                 u32 flags = le32_to_cpu(resp->flags);
5469
5470                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5471                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5472                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5473                 if (flags &
5474                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5475                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5476
5477                 /* Older P5 fw before EXT_HW_STATS support did not set
5478                  * VLAN_STRIP_CAP properly.
5479                  */
5480                 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5481                     (BNXT_CHIP_P5_THOR(bp) &&
5482                      !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5483                         bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5484                 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5485                 if (bp->max_tpa_v2) {
5486                         if (BNXT_CHIP_P5_THOR(bp))
5487                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5488                         else
5489                                 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5490                 }
5491         }
5492         mutex_unlock(&bp->hwrm_cmd_lock);
5493         return rc;
5494 }
5495
5496 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5497 {
5498         u16 i;
5499         u32 rc = 0;
5500
5501         if (bp->flags & BNXT_FLAG_CHIP_P5)
5502                 return 0;
5503
5504         mutex_lock(&bp->hwrm_cmd_lock);
5505         for (i = 0; i < bp->rx_nr_rings; i++) {
5506                 struct hwrm_ring_grp_alloc_input req = {0};
5507                 struct hwrm_ring_grp_alloc_output *resp =
5508                                         bp->hwrm_cmd_resp_addr;
5509                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5510
5511                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5512
5513                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5514                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5515                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5516                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5517
5518                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5519                                         HWRM_CMD_TIMEOUT);
5520                 if (rc)
5521                         break;
5522
5523                 bp->grp_info[grp_idx].fw_grp_id =
5524                         le32_to_cpu(resp->ring_group_id);
5525         }
5526         mutex_unlock(&bp->hwrm_cmd_lock);
5527         return rc;
5528 }
5529
5530 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5531 {
5532         u16 i;
5533         struct hwrm_ring_grp_free_input req = {0};
5534
5535         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5536                 return;
5537
5538         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5539
5540         mutex_lock(&bp->hwrm_cmd_lock);
5541         for (i = 0; i < bp->cp_nr_rings; i++) {
5542                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5543                         continue;
5544                 req.ring_group_id =
5545                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
5546
5547                 _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5548                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5549         }
5550         mutex_unlock(&bp->hwrm_cmd_lock);
5551 }
5552
5553 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5554                                     struct bnxt_ring_struct *ring,
5555                                     u32 ring_type, u32 map_index)
5556 {
5557         int rc = 0, err = 0;
5558         struct hwrm_ring_alloc_input req = {0};
5559         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5560         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5561         struct bnxt_ring_grp_info *grp_info;
5562         u16 ring_id;
5563
5564         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5565
5566         req.enables = 0;
5567         if (rmem->nr_pages > 1) {
5568                 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5569                 /* Page size is in log2 units */
5570                 req.page_size = BNXT_PAGE_SHIFT;
5571                 req.page_tbl_depth = 1;
5572         } else {
5573                 req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5574         }
5575         req.fbo = 0;
5576         /* Association of ring index with doorbell index and MSIX number */
5577         req.logical_id = cpu_to_le16(map_index);
5578
5579         switch (ring_type) {
5580         case HWRM_RING_ALLOC_TX: {
5581                 struct bnxt_tx_ring_info *txr;
5582
5583                 txr = container_of(ring, struct bnxt_tx_ring_info,
5584                                    tx_ring_struct);
5585                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5586                 /* Association of transmit ring with completion ring */
5587                 grp_info = &bp->grp_info[ring->grp_idx];
5588                 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5589                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5590                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5591                 req.queue_id = cpu_to_le16(ring->queue_id);
5592                 break;
5593         }
5594         case HWRM_RING_ALLOC_RX:
5595                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5596                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5597                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5598                         u16 flags = 0;
5599
5600                         /* Association of rx ring with stats context */
5601                         grp_info = &bp->grp_info[ring->grp_idx];
5602                         req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5603                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5604                         req.enables |= cpu_to_le32(
5605                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5606                         if (NET_IP_ALIGN == 2)
5607                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5608                         req.flags = cpu_to_le16(flags);
5609                 }
5610                 break;
5611         case HWRM_RING_ALLOC_AGG:
5612                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5613                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5614                         /* Association of agg ring with rx ring */
5615                         grp_info = &bp->grp_info[ring->grp_idx];
5616                         req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5617                         req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5618                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5619                         req.enables |= cpu_to_le32(
5620                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5621                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5622                 } else {
5623                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5624                 }
5625                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5626                 break;
5627         case HWRM_RING_ALLOC_CMPL:
5628                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5629                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5630                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5631                         /* Association of cp ring with nq */
5632                         grp_info = &bp->grp_info[map_index];
5633                         req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5634                         req.cq_handle = cpu_to_le64(ring->handle);
5635                         req.enables |= cpu_to_le32(
5636                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5637                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5638                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5639                 }
5640                 break;
5641         case HWRM_RING_ALLOC_NQ:
5642                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5643                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5644                 if (bp->flags & BNXT_FLAG_USING_MSIX)
5645                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5646                 break;
5647         default:
5648                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5649                            ring_type);
5650                 return -1;
5651         }
5652
5653         mutex_lock(&bp->hwrm_cmd_lock);
5654         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5655         err = le16_to_cpu(resp->error_code);
5656         ring_id = le16_to_cpu(resp->ring_id);
5657         mutex_unlock(&bp->hwrm_cmd_lock);
5658
5659         if (rc || err) {
5660                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5661                            ring_type, rc, err);
5662                 return -EIO;
5663         }
5664         ring->fw_ring_id = ring_id;
5665         return rc;
5666 }
5667
5668 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5669 {
5670         int rc;
5671
5672         if (BNXT_PF(bp)) {
5673                 struct hwrm_func_cfg_input req = {0};
5674
5675                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5676                 req.fid = cpu_to_le16(0xffff);
5677                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5678                 req.async_event_cr = cpu_to_le16(idx);
5679                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5680         } else {
5681                 struct hwrm_func_vf_cfg_input req = {0};
5682
5683                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5684                 req.enables =
5685                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5686                 req.async_event_cr = cpu_to_le16(idx);
5687                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5688         }
5689         return rc;
5690 }
5691
5692 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5693                         u32 map_idx, u32 xid)
5694 {
5695         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5696                 if (BNXT_PF(bp))
5697                         db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5698                 else
5699                         db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5700                 switch (ring_type) {
5701                 case HWRM_RING_ALLOC_TX:
5702                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5703                         break;
5704                 case HWRM_RING_ALLOC_RX:
5705                 case HWRM_RING_ALLOC_AGG:
5706                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5707                         break;
5708                 case HWRM_RING_ALLOC_CMPL:
5709                         db->db_key64 = DBR_PATH_L2;
5710                         break;
5711                 case HWRM_RING_ALLOC_NQ:
5712                         db->db_key64 = DBR_PATH_L2;
5713                         break;
5714                 }
5715                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5716         } else {
5717                 db->doorbell = bp->bar1 + map_idx * 0x80;
5718                 switch (ring_type) {
5719                 case HWRM_RING_ALLOC_TX:
5720                         db->db_key32 = DB_KEY_TX;
5721                         break;
5722                 case HWRM_RING_ALLOC_RX:
5723                 case HWRM_RING_ALLOC_AGG:
5724                         db->db_key32 = DB_KEY_RX;
5725                         break;
5726                 case HWRM_RING_ALLOC_CMPL:
5727                         db->db_key32 = DB_KEY_CP;
5728                         break;
5729                 }
5730         }
5731 }
5732
5733 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5734 {
5735         bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5736         int i, rc = 0;
5737         u32 type;
5738
5739         if (bp->flags & BNXT_FLAG_CHIP_P5)
5740                 type = HWRM_RING_ALLOC_NQ;
5741         else
5742                 type = HWRM_RING_ALLOC_CMPL;
5743         for (i = 0; i < bp->cp_nr_rings; i++) {
5744                 struct bnxt_napi *bnapi = bp->bnapi[i];
5745                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5746                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5747                 u32 map_idx = ring->map_idx;
5748                 unsigned int vector;
5749
5750                 vector = bp->irq_tbl[map_idx].vector;
5751                 disable_irq_nosync(vector);
5752                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5753                 if (rc) {
5754                         enable_irq(vector);
5755                         goto err_out;
5756                 }
5757                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5758                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5759                 enable_irq(vector);
5760                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5761
5762                 if (!i) {
5763                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5764                         if (rc)
5765                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5766                 }
5767         }
5768
5769         type = HWRM_RING_ALLOC_TX;
5770         for (i = 0; i < bp->tx_nr_rings; i++) {
5771                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5772                 struct bnxt_ring_struct *ring;
5773                 u32 map_idx;
5774
5775                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5776                         struct bnxt_napi *bnapi = txr->bnapi;
5777                         struct bnxt_cp_ring_info *cpr, *cpr2;
5778                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5779
5780                         cpr = &bnapi->cp_ring;
5781                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5782                         ring = &cpr2->cp_ring_struct;
5783                         ring->handle = BNXT_TX_HDL;
5784                         map_idx = bnapi->index;
5785                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5786                         if (rc)
5787                                 goto err_out;
5788                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5789                                     ring->fw_ring_id);
5790                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5791                 }
5792                 ring = &txr->tx_ring_struct;
5793                 map_idx = i;
5794                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5795                 if (rc)
5796                         goto err_out;
5797                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5798         }
5799
5800         type = HWRM_RING_ALLOC_RX;
5801         for (i = 0; i < bp->rx_nr_rings; i++) {
5802                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5803                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5804                 struct bnxt_napi *bnapi = rxr->bnapi;
5805                 u32 map_idx = bnapi->index;
5806
5807                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5808                 if (rc)
5809                         goto err_out;
5810                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5811                 /* If we have agg rings, post agg buffers first. */
5812                 if (!agg_rings)
5813                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5814                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5815                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5816                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5817                         u32 type2 = HWRM_RING_ALLOC_CMPL;
5818                         struct bnxt_cp_ring_info *cpr2;
5819
5820                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5821                         ring = &cpr2->cp_ring_struct;
5822                         ring->handle = BNXT_RX_HDL;
5823                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5824                         if (rc)
5825                                 goto err_out;
5826                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5827                                     ring->fw_ring_id);
5828                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5829                 }
5830         }
5831
5832         if (agg_rings) {
5833                 type = HWRM_RING_ALLOC_AGG;
5834                 for (i = 0; i < bp->rx_nr_rings; i++) {
5835                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5836                         struct bnxt_ring_struct *ring =
5837                                                 &rxr->rx_agg_ring_struct;
5838                         u32 grp_idx = ring->grp_idx;
5839                         u32 map_idx = grp_idx + bp->rx_nr_rings;
5840
5841                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5842                         if (rc)
5843                                 goto err_out;
5844
5845                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5846                                     ring->fw_ring_id);
5847                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5848                         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5849                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5850                 }
5851         }
5852 err_out:
5853         return rc;
5854 }
5855
5856 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5857                                    struct bnxt_ring_struct *ring,
5858                                    u32 ring_type, int cmpl_ring_id)
5859 {
5860         int rc;
5861         struct hwrm_ring_free_input req = {0};
5862         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5863         u16 error_code;
5864
5865         if (BNXT_NO_FW_ACCESS(bp))
5866                 return 0;
5867
5868         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5869         req.ring_type = ring_type;
5870         req.ring_id = cpu_to_le16(ring->fw_ring_id);
5871
5872         mutex_lock(&bp->hwrm_cmd_lock);
5873         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5874         error_code = le16_to_cpu(resp->error_code);
5875         mutex_unlock(&bp->hwrm_cmd_lock);
5876
5877         if (rc || error_code) {
5878                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5879                            ring_type, rc, error_code);
5880                 return -EIO;
5881         }
5882         return 0;
5883 }
5884
5885 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5886 {
5887         u32 type;
5888         int i;
5889
5890         if (!bp->bnapi)
5891                 return;
5892
5893         for (i = 0; i < bp->tx_nr_rings; i++) {
5894                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5895                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5896
5897                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5898                         u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5899
5900                         hwrm_ring_free_send_msg(bp, ring,
5901                                                 RING_FREE_REQ_RING_TYPE_TX,
5902                                                 close_path ? cmpl_ring_id :
5903                                                 INVALID_HW_RING_ID);
5904                         ring->fw_ring_id = INVALID_HW_RING_ID;
5905                 }
5906         }
5907
5908         for (i = 0; i < bp->rx_nr_rings; i++) {
5909                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5910                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5911                 u32 grp_idx = rxr->bnapi->index;
5912
5913                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5914                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5915
5916                         hwrm_ring_free_send_msg(bp, ring,
5917                                                 RING_FREE_REQ_RING_TYPE_RX,
5918                                                 close_path ? cmpl_ring_id :
5919                                                 INVALID_HW_RING_ID);
5920                         ring->fw_ring_id = INVALID_HW_RING_ID;
5921                         bp->grp_info[grp_idx].rx_fw_ring_id =
5922                                 INVALID_HW_RING_ID;
5923                 }
5924         }
5925
5926         if (bp->flags & BNXT_FLAG_CHIP_P5)
5927                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5928         else
5929                 type = RING_FREE_REQ_RING_TYPE_RX;
5930         for (i = 0; i < bp->rx_nr_rings; i++) {
5931                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5932                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5933                 u32 grp_idx = rxr->bnapi->index;
5934
5935                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5936                         u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5937
5938                         hwrm_ring_free_send_msg(bp, ring, type,
5939                                                 close_path ? cmpl_ring_id :
5940                                                 INVALID_HW_RING_ID);
5941                         ring->fw_ring_id = INVALID_HW_RING_ID;
5942                         bp->grp_info[grp_idx].agg_fw_ring_id =
5943                                 INVALID_HW_RING_ID;
5944                 }
5945         }
5946
5947         /* The completion rings are about to be freed.  After that the
5948          * IRQ doorbell will not work anymore.  So we need to disable
5949          * IRQ here.
5950          */
5951         bnxt_disable_int_sync(bp);
5952
5953         if (bp->flags & BNXT_FLAG_CHIP_P5)
5954                 type = RING_FREE_REQ_RING_TYPE_NQ;
5955         else
5956                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5957         for (i = 0; i < bp->cp_nr_rings; i++) {
5958                 struct bnxt_napi *bnapi = bp->bnapi[i];
5959                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5960                 struct bnxt_ring_struct *ring;
5961                 int j;
5962
5963                 for (j = 0; j < 2; j++) {
5964                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5965
5966                         if (cpr2) {
5967                                 ring = &cpr2->cp_ring_struct;
5968                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5969                                         continue;
5970                                 hwrm_ring_free_send_msg(bp, ring,
5971                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
5972                                         INVALID_HW_RING_ID);
5973                                 ring->fw_ring_id = INVALID_HW_RING_ID;
5974                         }
5975                 }
5976                 ring = &cpr->cp_ring_struct;
5977                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5978                         hwrm_ring_free_send_msg(bp, ring, type,
5979                                                 INVALID_HW_RING_ID);
5980                         ring->fw_ring_id = INVALID_HW_RING_ID;
5981                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5982                 }
5983         }
5984 }
5985
5986 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5987                            bool shared);
5988
5989 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5990 {
5991         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5992         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5993         struct hwrm_func_qcfg_input req = {0};
5994         int rc;
5995
5996         if (bp->hwrm_spec_code < 0x10601)
5997                 return 0;
5998
5999         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6000         req.fid = cpu_to_le16(0xffff);
6001         mutex_lock(&bp->hwrm_cmd_lock);
6002         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6003         if (rc) {
6004                 mutex_unlock(&bp->hwrm_cmd_lock);
6005                 return rc;
6006         }
6007
6008         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6009         if (BNXT_NEW_RM(bp)) {
6010                 u16 cp, stats;
6011
6012                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
6013                 hw_resc->resv_hw_ring_grps =
6014                         le32_to_cpu(resp->alloc_hw_ring_grps);
6015                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
6016                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
6017                 stats = le16_to_cpu(resp->alloc_stat_ctx);
6018                 hw_resc->resv_irqs = cp;
6019                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6020                         int rx = hw_resc->resv_rx_rings;
6021                         int tx = hw_resc->resv_tx_rings;
6022
6023                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6024                                 rx >>= 1;
6025                         if (cp < (rx + tx)) {
6026                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
6027                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6028                                         rx <<= 1;
6029                                 hw_resc->resv_rx_rings = rx;
6030                                 hw_resc->resv_tx_rings = tx;
6031                         }
6032                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
6033                         hw_resc->resv_hw_ring_grps = rx;
6034                 }
6035                 hw_resc->resv_cp_rings = cp;
6036                 hw_resc->resv_stat_ctxs = stats;
6037         }
6038         mutex_unlock(&bp->hwrm_cmd_lock);
6039         return 0;
6040 }
6041
6042 /* Caller must hold bp->hwrm_cmd_lock */
6043 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
6044 {
6045         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6046         struct hwrm_func_qcfg_input req = {0};
6047         int rc;
6048
6049         if (bp->hwrm_spec_code < 0x10601)
6050                 return 0;
6051
6052         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6053         req.fid = cpu_to_le16(fid);
6054         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6055         if (!rc)
6056                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6057
6058         return rc;
6059 }
6060
6061 static bool bnxt_rfs_supported(struct bnxt *bp);
6062
6063 static void
6064 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
6065                              int tx_rings, int rx_rings, int ring_grps,
6066                              int cp_rings, int stats, int vnics)
6067 {
6068         u32 enables = 0;
6069
6070         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
6071         req->fid = cpu_to_le16(0xffff);
6072         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6073         req->num_tx_rings = cpu_to_le16(tx_rings);
6074         if (BNXT_NEW_RM(bp)) {
6075                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
6076                 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6077                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6078                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
6079                         enables |= tx_rings + ring_grps ?
6080                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6081                         enables |= rx_rings ?
6082                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6083                 } else {
6084                         enables |= cp_rings ?
6085                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6086                         enables |= ring_grps ?
6087                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6088                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6089                 }
6090                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6091
6092                 req->num_rx_rings = cpu_to_le16(rx_rings);
6093                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6094                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6095                         req->num_msix = cpu_to_le16(cp_rings);
6096                         req->num_rsscos_ctxs =
6097                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6098                 } else {
6099                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
6100                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6101                         req->num_rsscos_ctxs = cpu_to_le16(1);
6102                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6103                             bnxt_rfs_supported(bp))
6104                                 req->num_rsscos_ctxs =
6105                                         cpu_to_le16(ring_grps + 1);
6106                 }
6107                 req->num_stat_ctxs = cpu_to_le16(stats);
6108                 req->num_vnics = cpu_to_le16(vnics);
6109         }
6110         req->enables = cpu_to_le32(enables);
6111 }
6112
6113 static void
6114 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
6115                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
6116                              int rx_rings, int ring_grps, int cp_rings,
6117                              int stats, int vnics)
6118 {
6119         u32 enables = 0;
6120
6121         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
6122         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6123         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6124                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6125         enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6126         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6127                 enables |= tx_rings + ring_grps ?
6128                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6129         } else {
6130                 enables |= cp_rings ?
6131                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6132                 enables |= ring_grps ?
6133                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6134         }
6135         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6136         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6137
6138         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6139         req->num_tx_rings = cpu_to_le16(tx_rings);
6140         req->num_rx_rings = cpu_to_le16(rx_rings);
6141         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6142                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6143                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6144         } else {
6145                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
6146                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6147                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6148         }
6149         req->num_stat_ctxs = cpu_to_le16(stats);
6150         req->num_vnics = cpu_to_le16(vnics);
6151
6152         req->enables = cpu_to_le32(enables);
6153 }
6154
6155 static int
6156 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6157                            int ring_grps, int cp_rings, int stats, int vnics)
6158 {
6159         struct hwrm_func_cfg_input req = {0};
6160         int rc;
6161
6162         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6163                                      cp_rings, stats, vnics);
6164         if (!req.enables)
6165                 return 0;
6166
6167         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6168         if (rc)
6169                 return rc;
6170
6171         if (bp->hwrm_spec_code < 0x10601)
6172                 bp->hw_resc.resv_tx_rings = tx_rings;
6173
6174         return bnxt_hwrm_get_rings(bp);
6175 }
6176
6177 static int
6178 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6179                            int ring_grps, int cp_rings, int stats, int vnics)
6180 {
6181         struct hwrm_func_vf_cfg_input req = {0};
6182         int rc;
6183
6184         if (!BNXT_NEW_RM(bp)) {
6185                 bp->hw_resc.resv_tx_rings = tx_rings;
6186                 return 0;
6187         }
6188
6189         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6190                                      cp_rings, stats, vnics);
6191         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6192         if (rc)
6193                 return rc;
6194
6195         return bnxt_hwrm_get_rings(bp);
6196 }
6197
6198 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6199                                    int cp, int stat, int vnic)
6200 {
6201         if (BNXT_PF(bp))
6202                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6203                                                   vnic);
6204         else
6205                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6206                                                   vnic);
6207 }
6208
6209 int bnxt_nq_rings_in_use(struct bnxt *bp)
6210 {
6211         int cp = bp->cp_nr_rings;
6212         int ulp_msix, ulp_base;
6213
6214         ulp_msix = bnxt_get_ulp_msix_num(bp);
6215         if (ulp_msix) {
6216                 ulp_base = bnxt_get_ulp_msix_base(bp);
6217                 cp += ulp_msix;
6218                 if ((ulp_base + ulp_msix) > cp)
6219                         cp = ulp_base + ulp_msix;
6220         }
6221         return cp;
6222 }
6223
6224 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6225 {
6226         int cp;
6227
6228         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6229                 return bnxt_nq_rings_in_use(bp);
6230
6231         cp = bp->tx_nr_rings + bp->rx_nr_rings;
6232         return cp;
6233 }
6234
6235 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6236 {
6237         int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6238         int cp = bp->cp_nr_rings;
6239
6240         if (!ulp_stat)
6241                 return cp;
6242
6243         if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6244                 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6245
6246         return cp + ulp_stat;
6247 }
6248
6249 /* Check if a default RSS map needs to be setup.  This function is only
6250  * used on older firmware that does not require reserving RX rings.
6251  */
6252 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6253 {
6254         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6255
6256         /* The RSS map is valid for RX rings set to resv_rx_rings */
6257         if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6258                 hw_resc->resv_rx_rings = bp->rx_nr_rings;
6259                 if (!netif_is_rxfh_configured(bp->dev))
6260                         bnxt_set_dflt_rss_indir_tbl(bp);
6261         }
6262 }
6263
6264 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6265 {
6266         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6267         int cp = bnxt_cp_rings_in_use(bp);
6268         int nq = bnxt_nq_rings_in_use(bp);
6269         int rx = bp->rx_nr_rings, stat;
6270         int vnic = 1, grp = rx;
6271
6272         if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6273             bp->hwrm_spec_code >= 0x10601)
6274                 return true;
6275
6276         /* Old firmware does not need RX ring reservations but we still
6277          * need to setup a default RSS map when needed.  With new firmware
6278          * we go through RX ring reservations first and then set up the
6279          * RSS map for the successfully reserved RX rings when needed.
6280          */
6281         if (!BNXT_NEW_RM(bp)) {
6282                 bnxt_check_rss_tbl_no_rmgr(bp);
6283                 return false;
6284         }
6285         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6286                 vnic = rx + 1;
6287         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6288                 rx <<= 1;
6289         stat = bnxt_get_func_stat_ctxs(bp);
6290         if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6291             hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6292             (hw_resc->resv_hw_ring_grps != grp &&
6293              !(bp->flags & BNXT_FLAG_CHIP_P5)))
6294                 return true;
6295         if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6296             hw_resc->resv_irqs != nq)
6297                 return true;
6298         return false;
6299 }
6300
6301 static int __bnxt_reserve_rings(struct bnxt *bp)
6302 {
6303         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6304         int cp = bnxt_nq_rings_in_use(bp);
6305         int tx = bp->tx_nr_rings;
6306         int rx = bp->rx_nr_rings;
6307         int grp, rx_rings, rc;
6308         int vnic = 1, stat;
6309         bool sh = false;
6310
6311         if (!bnxt_need_reserve_rings(bp))
6312                 return 0;
6313
6314         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6315                 sh = true;
6316         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6317                 vnic = rx + 1;
6318         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6319                 rx <<= 1;
6320         grp = bp->rx_nr_rings;
6321         stat = bnxt_get_func_stat_ctxs(bp);
6322
6323         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6324         if (rc)
6325                 return rc;
6326
6327         tx = hw_resc->resv_tx_rings;
6328         if (BNXT_NEW_RM(bp)) {
6329                 rx = hw_resc->resv_rx_rings;
6330                 cp = hw_resc->resv_irqs;
6331                 grp = hw_resc->resv_hw_ring_grps;
6332                 vnic = hw_resc->resv_vnics;
6333                 stat = hw_resc->resv_stat_ctxs;
6334         }
6335
6336         rx_rings = rx;
6337         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6338                 if (rx >= 2) {
6339                         rx_rings = rx >> 1;
6340                 } else {
6341                         if (netif_running(bp->dev))
6342                                 return -ENOMEM;
6343
6344                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6345                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6346                         bp->dev->hw_features &= ~NETIF_F_LRO;
6347                         bp->dev->features &= ~NETIF_F_LRO;
6348                         bnxt_set_ring_params(bp);
6349                 }
6350         }
6351         rx_rings = min_t(int, rx_rings, grp);
6352         cp = min_t(int, cp, bp->cp_nr_rings);
6353         if (stat > bnxt_get_ulp_stat_ctxs(bp))
6354                 stat -= bnxt_get_ulp_stat_ctxs(bp);
6355         cp = min_t(int, cp, stat);
6356         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6357         if (bp->flags & BNXT_FLAG_AGG_RINGS)
6358                 rx = rx_rings << 1;
6359         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6360         bp->tx_nr_rings = tx;
6361
6362         /* If we cannot reserve all the RX rings, reset the RSS map only
6363          * if absolutely necessary
6364          */
6365         if (rx_rings != bp->rx_nr_rings) {
6366                 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6367                             rx_rings, bp->rx_nr_rings);
6368                 if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) &&
6369                     (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6370                      bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6371                      bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6372                         netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6373                         bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6374                 }
6375         }
6376         bp->rx_nr_rings = rx_rings;
6377         bp->cp_nr_rings = cp;
6378
6379         if (!tx || !rx || !cp || !grp || !vnic || !stat)
6380                 return -ENOMEM;
6381
6382         if (!netif_is_rxfh_configured(bp->dev))
6383                 bnxt_set_dflt_rss_indir_tbl(bp);
6384
6385         return rc;
6386 }
6387
6388 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6389                                     int ring_grps, int cp_rings, int stats,
6390                                     int vnics)
6391 {
6392         struct hwrm_func_vf_cfg_input req = {0};
6393         u32 flags;
6394
6395         if (!BNXT_NEW_RM(bp))
6396                 return 0;
6397
6398         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6399                                      cp_rings, stats, vnics);
6400         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6401                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6402                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6403                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6404                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6405                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6406         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6407                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6408
6409         req.flags = cpu_to_le32(flags);
6410         return hwrm_send_message_silent(bp, &req, sizeof(req),
6411                                         HWRM_CMD_TIMEOUT);
6412 }
6413
6414 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6415                                     int ring_grps, int cp_rings, int stats,
6416                                     int vnics)
6417 {
6418         struct hwrm_func_cfg_input req = {0};
6419         u32 flags;
6420
6421         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6422                                      cp_rings, stats, vnics);
6423         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6424         if (BNXT_NEW_RM(bp)) {
6425                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6426                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6427                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6428                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6429                 if (bp->flags & BNXT_FLAG_CHIP_P5)
6430                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6431                                  FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6432                 else
6433                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6434         }
6435
6436         req.flags = cpu_to_le32(flags);
6437         return hwrm_send_message_silent(bp, &req, sizeof(req),
6438                                         HWRM_CMD_TIMEOUT);
6439 }
6440
6441 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6442                                  int ring_grps, int cp_rings, int stats,
6443                                  int vnics)
6444 {
6445         if (bp->hwrm_spec_code < 0x10801)
6446                 return 0;
6447
6448         if (BNXT_PF(bp))
6449                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6450                                                 ring_grps, cp_rings, stats,
6451                                                 vnics);
6452
6453         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6454                                         cp_rings, stats, vnics);
6455 }
6456
6457 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6458 {
6459         struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6460         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6461         struct hwrm_ring_aggint_qcaps_input req = {0};
6462         int rc;
6463
6464         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6465         coal_cap->num_cmpl_dma_aggr_max = 63;
6466         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6467         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6468         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6469         coal_cap->int_lat_tmr_min_max = 65535;
6470         coal_cap->int_lat_tmr_max_max = 65535;
6471         coal_cap->num_cmpl_aggr_int_max = 65535;
6472         coal_cap->timer_units = 80;
6473
6474         if (bp->hwrm_spec_code < 0x10902)
6475                 return;
6476
6477         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6478         mutex_lock(&bp->hwrm_cmd_lock);
6479         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6480         if (!rc) {
6481                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6482                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6483                 coal_cap->num_cmpl_dma_aggr_max =
6484                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6485                 coal_cap->num_cmpl_dma_aggr_during_int_max =
6486                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6487                 coal_cap->cmpl_aggr_dma_tmr_max =
6488                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6489                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6490                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6491                 coal_cap->int_lat_tmr_min_max =
6492                         le16_to_cpu(resp->int_lat_tmr_min_max);
6493                 coal_cap->int_lat_tmr_max_max =
6494                         le16_to_cpu(resp->int_lat_tmr_max_max);
6495                 coal_cap->num_cmpl_aggr_int_max =
6496                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
6497                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6498         }
6499         mutex_unlock(&bp->hwrm_cmd_lock);
6500 }
6501
6502 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6503 {
6504         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6505
6506         return usec * 1000 / coal_cap->timer_units;
6507 }
6508
6509 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6510         struct bnxt_coal *hw_coal,
6511         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6512 {
6513         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6514         u32 cmpl_params = coal_cap->cmpl_params;
6515         u16 val, tmr, max, flags = 0;
6516
6517         max = hw_coal->bufs_per_record * 128;
6518         if (hw_coal->budget)
6519                 max = hw_coal->bufs_per_record * hw_coal->budget;
6520         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6521
6522         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6523         req->num_cmpl_aggr_int = cpu_to_le16(val);
6524
6525         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6526         req->num_cmpl_dma_aggr = cpu_to_le16(val);
6527
6528         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6529                       coal_cap->num_cmpl_dma_aggr_during_int_max);
6530         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6531
6532         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6533         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6534         req->int_lat_tmr_max = cpu_to_le16(tmr);
6535
6536         /* min timer set to 1/2 of interrupt timer */
6537         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6538                 val = tmr / 2;
6539                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6540                 req->int_lat_tmr_min = cpu_to_le16(val);
6541                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6542         }
6543
6544         /* buf timer set to 1/4 of interrupt timer */
6545         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6546         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6547
6548         if (cmpl_params &
6549             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6550                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6551                 val = clamp_t(u16, tmr, 1,
6552                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6553                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6554                 req->enables |=
6555                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6556         }
6557
6558         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6559                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6560         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6561             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6562                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6563         req->flags = cpu_to_le16(flags);
6564         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6565 }
6566
6567 /* Caller holds bp->hwrm_cmd_lock */
6568 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6569                                    struct bnxt_coal *hw_coal)
6570 {
6571         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6572         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6573         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6574         u32 nq_params = coal_cap->nq_params;
6575         u16 tmr;
6576
6577         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6578                 return 0;
6579
6580         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6581                                -1, -1);
6582         req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6583         req.flags =
6584                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6585
6586         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6587         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6588         req.int_lat_tmr_min = cpu_to_le16(tmr);
6589         req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6590         return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6591 }
6592
6593 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6594 {
6595         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6596         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6597         struct bnxt_coal coal;
6598
6599         /* Tick values in micro seconds.
6600          * 1 coal_buf x bufs_per_record = 1 completion record.
6601          */
6602         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6603
6604         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6605         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6606
6607         if (!bnapi->rx_ring)
6608                 return -ENODEV;
6609
6610         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6611                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6612
6613         bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6614
6615         req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6616
6617         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6618                                  HWRM_CMD_TIMEOUT);
6619 }
6620
6621 int bnxt_hwrm_set_coal(struct bnxt *bp)
6622 {
6623         int i, rc = 0;
6624         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6625                                                            req_tx = {0}, *req;
6626
6627         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6628                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6629         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6630                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6631
6632         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6633         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6634
6635         mutex_lock(&bp->hwrm_cmd_lock);
6636         for (i = 0; i < bp->cp_nr_rings; i++) {
6637                 struct bnxt_napi *bnapi = bp->bnapi[i];
6638                 struct bnxt_coal *hw_coal;
6639                 u16 ring_id;
6640
6641                 req = &req_rx;
6642                 if (!bnapi->rx_ring) {
6643                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6644                         req = &req_tx;
6645                 } else {
6646                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6647                 }
6648                 req->ring_id = cpu_to_le16(ring_id);
6649
6650                 rc = _hwrm_send_message(bp, req, sizeof(*req),
6651                                         HWRM_CMD_TIMEOUT);
6652                 if (rc)
6653                         break;
6654
6655                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6656                         continue;
6657
6658                 if (bnapi->rx_ring && bnapi->tx_ring) {
6659                         req = &req_tx;
6660                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6661                         req->ring_id = cpu_to_le16(ring_id);
6662                         rc = _hwrm_send_message(bp, req, sizeof(*req),
6663                                                 HWRM_CMD_TIMEOUT);
6664                         if (rc)
6665                                 break;
6666                 }
6667                 if (bnapi->rx_ring)
6668                         hw_coal = &bp->rx_coal;
6669                 else
6670                         hw_coal = &bp->tx_coal;
6671                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6672         }
6673         mutex_unlock(&bp->hwrm_cmd_lock);
6674         return rc;
6675 }
6676
6677 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6678 {
6679         struct hwrm_stat_ctx_clr_stats_input req0 = {0};
6680         struct hwrm_stat_ctx_free_input req = {0};
6681         int i;
6682
6683         if (!bp->bnapi)
6684                 return;
6685
6686         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6687                 return;
6688
6689         bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1);
6690         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6691
6692         mutex_lock(&bp->hwrm_cmd_lock);
6693         for (i = 0; i < bp->cp_nr_rings; i++) {
6694                 struct bnxt_napi *bnapi = bp->bnapi[i];
6695                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6696
6697                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6698                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6699                         if (BNXT_FW_MAJ(bp) <= 20) {
6700                                 req0.stat_ctx_id = req.stat_ctx_id;
6701                                 _hwrm_send_message(bp, &req0, sizeof(req0),
6702                                                    HWRM_CMD_TIMEOUT);
6703                         }
6704                         _hwrm_send_message(bp, &req, sizeof(req),
6705                                            HWRM_CMD_TIMEOUT);
6706
6707                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6708                 }
6709         }
6710         mutex_unlock(&bp->hwrm_cmd_lock);
6711 }
6712
6713 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6714 {
6715         int rc = 0, i;
6716         struct hwrm_stat_ctx_alloc_input req = {0};
6717         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6718
6719         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6720                 return 0;
6721
6722         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6723
6724         req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6725         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6726
6727         mutex_lock(&bp->hwrm_cmd_lock);
6728         for (i = 0; i < bp->cp_nr_rings; i++) {
6729                 struct bnxt_napi *bnapi = bp->bnapi[i];
6730                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6731
6732                 req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6733
6734                 rc = _hwrm_send_message(bp, &req, sizeof(req),
6735                                         HWRM_CMD_TIMEOUT);
6736                 if (rc)
6737                         break;
6738
6739                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6740
6741                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6742         }
6743         mutex_unlock(&bp->hwrm_cmd_lock);
6744         return rc;
6745 }
6746
6747 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6748 {
6749         struct hwrm_func_qcfg_input req = {0};
6750         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6751         u32 min_db_offset = 0;
6752         u16 flags;
6753         int rc;
6754
6755         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6756         req.fid = cpu_to_le16(0xffff);
6757         mutex_lock(&bp->hwrm_cmd_lock);
6758         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6759         if (rc)
6760                 goto func_qcfg_exit;
6761
6762 #ifdef CONFIG_BNXT_SRIOV
6763         if (BNXT_VF(bp)) {
6764                 struct bnxt_vf_info *vf = &bp->vf;
6765
6766                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6767         } else {
6768                 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6769         }
6770 #endif
6771         flags = le16_to_cpu(resp->flags);
6772         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6773                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6774                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6775                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6776                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6777         }
6778         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6779                 bp->flags |= BNXT_FLAG_MULTI_HOST;
6780         if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6781                 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6782
6783         switch (resp->port_partition_type) {
6784         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6785         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6786         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6787                 bp->port_partition_type = resp->port_partition_type;
6788                 break;
6789         }
6790         if (bp->hwrm_spec_code < 0x10707 ||
6791             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6792                 bp->br_mode = BRIDGE_MODE_VEB;
6793         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6794                 bp->br_mode = BRIDGE_MODE_VEPA;
6795         else
6796                 bp->br_mode = BRIDGE_MODE_UNDEF;
6797
6798         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6799         if (!bp->max_mtu)
6800                 bp->max_mtu = BNXT_MAX_MTU;
6801
6802         if (bp->db_size)
6803                 goto func_qcfg_exit;
6804
6805         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6806                 if (BNXT_PF(bp))
6807                         min_db_offset = DB_PF_OFFSET_P5;
6808                 else
6809                         min_db_offset = DB_VF_OFFSET_P5;
6810         }
6811         bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
6812                                  1024);
6813         if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
6814             bp->db_size <= min_db_offset)
6815                 bp->db_size = pci_resource_len(bp->pdev, 2);
6816
6817 func_qcfg_exit:
6818         mutex_unlock(&bp->hwrm_cmd_lock);
6819         return rc;
6820 }
6821
6822 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx,
6823                         struct hwrm_func_backing_store_qcaps_output *resp)
6824 {
6825         struct bnxt_mem_init *mem_init;
6826         u16 init_mask;
6827         u8 init_val;
6828         u8 *offset;
6829         int i;
6830
6831         init_val = resp->ctx_kind_initializer;
6832         init_mask = le16_to_cpu(resp->ctx_init_mask);
6833         offset = &resp->qp_init_offset;
6834         mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
6835         for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) {
6836                 mem_init->init_val = init_val;
6837                 mem_init->offset = BNXT_MEM_INVALID_OFFSET;
6838                 if (!init_mask)
6839                         continue;
6840                 if (i == BNXT_CTX_MEM_INIT_STAT)
6841                         offset = &resp->stat_init_offset;
6842                 if (init_mask & (1 << i))
6843                         mem_init->offset = *offset * 4;
6844                 else
6845                         mem_init->init_val = 0;
6846         }
6847         ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size;
6848         ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size;
6849         ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size;
6850         ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size;
6851         ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size;
6852         ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size;
6853 }
6854
6855 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6856 {
6857         struct hwrm_func_backing_store_qcaps_input req = {0};
6858         struct hwrm_func_backing_store_qcaps_output *resp =
6859                 bp->hwrm_cmd_resp_addr;
6860         int rc;
6861
6862         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6863                 return 0;
6864
6865         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6866         mutex_lock(&bp->hwrm_cmd_lock);
6867         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6868         if (!rc) {
6869                 struct bnxt_ctx_pg_info *ctx_pg;
6870                 struct bnxt_ctx_mem_info *ctx;
6871                 int i, tqm_rings;
6872
6873                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6874                 if (!ctx) {
6875                         rc = -ENOMEM;
6876                         goto ctx_err;
6877                 }
6878                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6879                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6880                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6881                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6882                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6883                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6884                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6885                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6886                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6887                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6888                 ctx->vnic_max_vnic_entries =
6889                         le16_to_cpu(resp->vnic_max_vnic_entries);
6890                 ctx->vnic_max_ring_table_entries =
6891                         le16_to_cpu(resp->vnic_max_ring_table_entries);
6892                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6893                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6894                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6895                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6896                 ctx->tqm_min_entries_per_ring =
6897                         le32_to_cpu(resp->tqm_min_entries_per_ring);
6898                 ctx->tqm_max_entries_per_ring =
6899                         le32_to_cpu(resp->tqm_max_entries_per_ring);
6900                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6901                 if (!ctx->tqm_entries_multiple)
6902                         ctx->tqm_entries_multiple = 1;
6903                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6904                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6905                 ctx->mrav_num_entries_units =
6906                         le16_to_cpu(resp->mrav_num_entries_units);
6907                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6908                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6909
6910                 bnxt_init_ctx_initializer(ctx, resp);
6911
6912                 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
6913                 if (!ctx->tqm_fp_rings_count)
6914                         ctx->tqm_fp_rings_count = bp->max_q;
6915                 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
6916                         ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
6917
6918                 tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS;
6919                 ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
6920                 if (!ctx_pg) {
6921                         kfree(ctx);
6922                         rc = -ENOMEM;
6923                         goto ctx_err;
6924                 }
6925                 for (i = 0; i < tqm_rings; i++, ctx_pg++)
6926                         ctx->tqm_mem[i] = ctx_pg;
6927                 bp->ctx = ctx;
6928         } else {
6929                 rc = 0;
6930         }
6931 ctx_err:
6932         mutex_unlock(&bp->hwrm_cmd_lock);
6933         return rc;
6934 }
6935
6936 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6937                                   __le64 *pg_dir)
6938 {
6939         if (!rmem->nr_pages)
6940                 return;
6941
6942         BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
6943         if (rmem->depth >= 1) {
6944                 if (rmem->depth == 2)
6945                         *pg_attr |= 2;
6946                 else
6947                         *pg_attr |= 1;
6948                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6949         } else {
6950                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6951         }
6952 }
6953
6954 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
6955         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
6956          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
6957          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
6958          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
6959          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6960
6961 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6962 {
6963         struct hwrm_func_backing_store_cfg_input req = {0};
6964         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6965         struct bnxt_ctx_pg_info *ctx_pg;
6966         u32 req_len = sizeof(req);
6967         __le32 *num_entries;
6968         __le64 *pg_dir;
6969         u32 flags = 0;
6970         u8 *pg_attr;
6971         u32 ena;
6972         int i;
6973
6974         if (!ctx)
6975                 return 0;
6976
6977         if (req_len > bp->hwrm_max_ext_req_len)
6978                 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
6979         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6980         req.enables = cpu_to_le32(enables);
6981
6982         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6983                 ctx_pg = &ctx->qp_mem;
6984                 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6985                 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6986                 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6987                 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6988                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6989                                       &req.qpc_pg_size_qpc_lvl,
6990                                       &req.qpc_page_dir);
6991         }
6992         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6993                 ctx_pg = &ctx->srq_mem;
6994                 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6995                 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6996                 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6997                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6998                                       &req.srq_pg_size_srq_lvl,
6999                                       &req.srq_page_dir);
7000         }
7001         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
7002                 ctx_pg = &ctx->cq_mem;
7003                 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
7004                 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
7005                 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
7006                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
7007                                       &req.cq_page_dir);
7008         }
7009         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
7010                 ctx_pg = &ctx->vnic_mem;
7011                 req.vnic_num_vnic_entries =
7012                         cpu_to_le16(ctx->vnic_max_vnic_entries);
7013                 req.vnic_num_ring_table_entries =
7014                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
7015                 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
7016                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7017                                       &req.vnic_pg_size_vnic_lvl,
7018                                       &req.vnic_page_dir);
7019         }
7020         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
7021                 ctx_pg = &ctx->stat_mem;
7022                 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
7023                 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
7024                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7025                                       &req.stat_pg_size_stat_lvl,
7026                                       &req.stat_page_dir);
7027         }
7028         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
7029                 ctx_pg = &ctx->mrav_mem;
7030                 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
7031                 if (ctx->mrav_num_entries_units)
7032                         flags |=
7033                         FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
7034                 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
7035                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7036                                       &req.mrav_pg_size_mrav_lvl,
7037                                       &req.mrav_page_dir);
7038         }
7039         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
7040                 ctx_pg = &ctx->tim_mem;
7041                 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
7042                 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
7043                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7044                                       &req.tim_pg_size_tim_lvl,
7045                                       &req.tim_page_dir);
7046         }
7047         for (i = 0, num_entries = &req.tqm_sp_num_entries,
7048              pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
7049              pg_dir = &req.tqm_sp_page_dir,
7050              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
7051              i < BNXT_MAX_TQM_RINGS;
7052              i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
7053                 if (!(enables & ena))
7054                         continue;
7055
7056                 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
7057                 ctx_pg = ctx->tqm_mem[i];
7058                 *num_entries = cpu_to_le32(ctx_pg->entries);
7059                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
7060         }
7061         req.flags = cpu_to_le32(flags);
7062         return hwrm_send_message(bp, &req, req_len, HWRM_CMD_TIMEOUT);
7063 }
7064
7065 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
7066                                   struct bnxt_ctx_pg_info *ctx_pg)
7067 {
7068         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7069
7070         rmem->page_size = BNXT_PAGE_SIZE;
7071         rmem->pg_arr = ctx_pg->ctx_pg_arr;
7072         rmem->dma_arr = ctx_pg->ctx_dma_arr;
7073         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
7074         if (rmem->depth >= 1)
7075                 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
7076         return bnxt_alloc_ring(bp, rmem);
7077 }
7078
7079 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
7080                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
7081                                   u8 depth, struct bnxt_mem_init *mem_init)
7082 {
7083         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7084         int rc;
7085
7086         if (!mem_size)
7087                 return -EINVAL;
7088
7089         ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7090         if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
7091                 ctx_pg->nr_pages = 0;
7092                 return -EINVAL;
7093         }
7094         if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
7095                 int nr_tbls, i;
7096
7097                 rmem->depth = 2;
7098                 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
7099                                              GFP_KERNEL);
7100                 if (!ctx_pg->ctx_pg_tbl)
7101                         return -ENOMEM;
7102                 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
7103                 rmem->nr_pages = nr_tbls;
7104                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7105                 if (rc)
7106                         return rc;
7107                 for (i = 0; i < nr_tbls; i++) {
7108                         struct bnxt_ctx_pg_info *pg_tbl;
7109
7110                         pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
7111                         if (!pg_tbl)
7112                                 return -ENOMEM;
7113                         ctx_pg->ctx_pg_tbl[i] = pg_tbl;
7114                         rmem = &pg_tbl->ring_mem;
7115                         rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
7116                         rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
7117                         rmem->depth = 1;
7118                         rmem->nr_pages = MAX_CTX_PAGES;
7119                         rmem->mem_init = mem_init;
7120                         if (i == (nr_tbls - 1)) {
7121                                 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
7122
7123                                 if (rem)
7124                                         rmem->nr_pages = rem;
7125                         }
7126                         rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7127                         if (rc)
7128                                 break;
7129                 }
7130         } else {
7131                 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7132                 if (rmem->nr_pages > 1 || depth)
7133                         rmem->depth = 1;
7134                 rmem->mem_init = mem_init;
7135                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7136         }
7137         return rc;
7138 }
7139
7140 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7141                                   struct bnxt_ctx_pg_info *ctx_pg)
7142 {
7143         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7144
7145         if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7146             ctx_pg->ctx_pg_tbl) {
7147                 int i, nr_tbls = rmem->nr_pages;
7148
7149                 for (i = 0; i < nr_tbls; i++) {
7150                         struct bnxt_ctx_pg_info *pg_tbl;
7151                         struct bnxt_ring_mem_info *rmem2;
7152
7153                         pg_tbl = ctx_pg->ctx_pg_tbl[i];
7154                         if (!pg_tbl)
7155                                 continue;
7156                         rmem2 = &pg_tbl->ring_mem;
7157                         bnxt_free_ring(bp, rmem2);
7158                         ctx_pg->ctx_pg_arr[i] = NULL;
7159                         kfree(pg_tbl);
7160                         ctx_pg->ctx_pg_tbl[i] = NULL;
7161                 }
7162                 kfree(ctx_pg->ctx_pg_tbl);
7163                 ctx_pg->ctx_pg_tbl = NULL;
7164         }
7165         bnxt_free_ring(bp, rmem);
7166         ctx_pg->nr_pages = 0;
7167 }
7168
7169 static void bnxt_free_ctx_mem(struct bnxt *bp)
7170 {
7171         struct bnxt_ctx_mem_info *ctx = bp->ctx;
7172         int i;
7173
7174         if (!ctx)
7175                 return;
7176
7177         if (ctx->tqm_mem[0]) {
7178                 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7179                         bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7180                 kfree(ctx->tqm_mem[0]);
7181                 ctx->tqm_mem[0] = NULL;
7182         }
7183
7184         bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7185         bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7186         bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7187         bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7188         bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7189         bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7190         bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7191         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7192 }
7193
7194 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7195 {
7196         struct bnxt_ctx_pg_info *ctx_pg;
7197         struct bnxt_ctx_mem_info *ctx;
7198         struct bnxt_mem_init *init;
7199         u32 mem_size, ena, entries;
7200         u32 entries_sp, min;
7201         u32 num_mr, num_ah;
7202         u32 extra_srqs = 0;
7203         u32 extra_qps = 0;
7204         u8 pg_lvl = 1;
7205         int i, rc;
7206
7207         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7208         if (rc) {
7209                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7210                            rc);
7211                 return rc;
7212         }
7213         ctx = bp->ctx;
7214         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7215                 return 0;
7216
7217         if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7218                 pg_lvl = 2;
7219                 extra_qps = 65536;
7220                 extra_srqs = 8192;
7221         }
7222
7223         ctx_pg = &ctx->qp_mem;
7224         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7225                           extra_qps;
7226         if (ctx->qp_entry_size) {
7227                 mem_size = ctx->qp_entry_size * ctx_pg->entries;
7228                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7229                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7230                 if (rc)
7231                         return rc;
7232         }
7233
7234         ctx_pg = &ctx->srq_mem;
7235         ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7236         if (ctx->srq_entry_size) {
7237                 mem_size = ctx->srq_entry_size * ctx_pg->entries;
7238                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ];
7239                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7240                 if (rc)
7241                         return rc;
7242         }
7243
7244         ctx_pg = &ctx->cq_mem;
7245         ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7246         if (ctx->cq_entry_size) {
7247                 mem_size = ctx->cq_entry_size * ctx_pg->entries;
7248                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ];
7249                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7250                 if (rc)
7251                         return rc;
7252         }
7253
7254         ctx_pg = &ctx->vnic_mem;
7255         ctx_pg->entries = ctx->vnic_max_vnic_entries +
7256                           ctx->vnic_max_ring_table_entries;
7257         if (ctx->vnic_entry_size) {
7258                 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7259                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC];
7260                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7261                 if (rc)
7262                         return rc;
7263         }
7264
7265         ctx_pg = &ctx->stat_mem;
7266         ctx_pg->entries = ctx->stat_max_entries;
7267         if (ctx->stat_entry_size) {
7268                 mem_size = ctx->stat_entry_size * ctx_pg->entries;
7269                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT];
7270                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7271                 if (rc)
7272                         return rc;
7273         }
7274
7275         ena = 0;
7276         if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7277                 goto skip_rdma;
7278
7279         ctx_pg = &ctx->mrav_mem;
7280         /* 128K extra is needed to accommodate static AH context
7281          * allocation by f/w.
7282          */
7283         num_mr = 1024 * 256;
7284         num_ah = 1024 * 128;
7285         ctx_pg->entries = num_mr + num_ah;
7286         if (ctx->mrav_entry_size) {
7287                 mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7288                 init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV];
7289                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init);
7290                 if (rc)
7291                         return rc;
7292         }
7293         ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7294         if (ctx->mrav_num_entries_units)
7295                 ctx_pg->entries =
7296                         ((num_mr / ctx->mrav_num_entries_units) << 16) |
7297                          (num_ah / ctx->mrav_num_entries_units);
7298
7299         ctx_pg = &ctx->tim_mem;
7300         ctx_pg->entries = ctx->qp_mem.entries;
7301         if (ctx->tim_entry_size) {
7302                 mem_size = ctx->tim_entry_size * ctx_pg->entries;
7303                 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL);
7304                 if (rc)
7305                         return rc;
7306         }
7307         ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7308
7309 skip_rdma:
7310         min = ctx->tqm_min_entries_per_ring;
7311         entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7312                      2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7313         entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7314         entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries);
7315         entries = roundup(entries, ctx->tqm_entries_multiple);
7316         entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7317         for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7318                 ctx_pg = ctx->tqm_mem[i];
7319                 ctx_pg->entries = i ? entries : entries_sp;
7320                 if (ctx->tqm_entry_size) {
7321                         mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7322                         rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1,
7323                                                     NULL);
7324                         if (rc)
7325                                 return rc;
7326                 }
7327                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7328         }
7329         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7330         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7331         if (rc) {
7332                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7333                            rc);
7334                 return rc;
7335         }
7336         ctx->flags |= BNXT_CTX_FLAG_INITED;
7337         return 0;
7338 }
7339
7340 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7341 {
7342         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7343         struct hwrm_func_resource_qcaps_input req = {0};
7344         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7345         int rc;
7346
7347         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
7348         req.fid = cpu_to_le16(0xffff);
7349
7350         mutex_lock(&bp->hwrm_cmd_lock);
7351         rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
7352                                        HWRM_CMD_TIMEOUT);
7353         if (rc)
7354                 goto hwrm_func_resc_qcaps_exit;
7355
7356         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7357         if (!all)
7358                 goto hwrm_func_resc_qcaps_exit;
7359
7360         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7361         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7362         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7363         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7364         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7365         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7366         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7367         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7368         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7369         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7370         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7371         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7372         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7373         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7374         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7375         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7376
7377         if (bp->flags & BNXT_FLAG_CHIP_P5) {
7378                 u16 max_msix = le16_to_cpu(resp->max_msix);
7379
7380                 hw_resc->max_nqs = max_msix;
7381                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7382         }
7383
7384         if (BNXT_PF(bp)) {
7385                 struct bnxt_pf_info *pf = &bp->pf;
7386
7387                 pf->vf_resv_strategy =
7388                         le16_to_cpu(resp->vf_reservation_strategy);
7389                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7390                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7391         }
7392 hwrm_func_resc_qcaps_exit:
7393         mutex_unlock(&bp->hwrm_cmd_lock);
7394         return rc;
7395 }
7396
7397 /* bp->hwrm_cmd_lock already held. */
7398 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
7399 {
7400         struct hwrm_port_mac_ptp_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7401         struct hwrm_port_mac_ptp_qcfg_input req = {0};
7402         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
7403         u8 flags;
7404         int rc;
7405
7406         if (bp->hwrm_spec_code < 0x10801) {
7407                 rc = -ENODEV;
7408                 goto no_ptp;
7409         }
7410
7411         req.port_id = cpu_to_le16(bp->pf.port_id);
7412         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_MAC_PTP_QCFG, -1, -1);
7413         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7414         if (rc)
7415                 goto no_ptp;
7416
7417         flags = resp->flags;
7418         if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
7419                 rc = -ENODEV;
7420                 goto no_ptp;
7421         }
7422         if (!ptp) {
7423                 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
7424                 if (!ptp)
7425                         return -ENOMEM;
7426                 ptp->bp = bp;
7427                 bp->ptp_cfg = ptp;
7428         }
7429         if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) {
7430                 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
7431                 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
7432         } else if (bp->flags & BNXT_FLAG_CHIP_P5) {
7433                 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
7434                 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
7435         } else {
7436                 rc = -ENODEV;
7437                 goto no_ptp;
7438         }
7439         return 0;
7440
7441 no_ptp:
7442         kfree(ptp);
7443         bp->ptp_cfg = NULL;
7444         return rc;
7445 }
7446
7447 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7448 {
7449         int rc = 0;
7450         struct hwrm_func_qcaps_input req = {0};
7451         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7452         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7453         u32 flags, flags_ext;
7454
7455         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
7456         req.fid = cpu_to_le16(0xffff);
7457
7458         mutex_lock(&bp->hwrm_cmd_lock);
7459         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7460         if (rc)
7461                 goto hwrm_func_qcaps_exit;
7462
7463         flags = le32_to_cpu(resp->flags);
7464         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7465                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7466         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7467                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7468         if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7469                 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7470         if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7471                 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7472         if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7473                 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7474         if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7475                 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7476         if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7477                 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7478         if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7479                 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7480
7481         flags_ext = le32_to_cpu(resp->flags_ext);
7482         if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7483                 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7484
7485         bp->tx_push_thresh = 0;
7486         if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7487             BNXT_FW_MAJ(bp) > 217)
7488                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7489
7490         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7491         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7492         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7493         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7494         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7495         if (!hw_resc->max_hw_ring_grps)
7496                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7497         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7498         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7499         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7500
7501         if (BNXT_PF(bp)) {
7502                 struct bnxt_pf_info *pf = &bp->pf;
7503
7504                 pf->fw_fid = le16_to_cpu(resp->fid);
7505                 pf->port_id = le16_to_cpu(resp->port_id);
7506                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7507                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7508                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7509                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7510                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7511                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7512                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7513                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7514                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7515                 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7516                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7517                         bp->flags |= BNXT_FLAG_WOL_CAP;
7518                 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED)
7519                         __bnxt_hwrm_ptp_qcfg(bp);
7520         } else {
7521 #ifdef CONFIG_BNXT_SRIOV
7522                 struct bnxt_vf_info *vf = &bp->vf;
7523
7524                 vf->fw_fid = le16_to_cpu(resp->fid);
7525                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7526 #endif
7527         }
7528
7529 hwrm_func_qcaps_exit:
7530         mutex_unlock(&bp->hwrm_cmd_lock);
7531         return rc;
7532 }
7533
7534 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7535
7536 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7537 {
7538         int rc;
7539
7540         rc = __bnxt_hwrm_func_qcaps(bp);
7541         if (rc)
7542                 return rc;
7543         rc = bnxt_hwrm_queue_qportcfg(bp);
7544         if (rc) {
7545                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7546                 return rc;
7547         }
7548         if (bp->hwrm_spec_code >= 0x10803) {
7549                 rc = bnxt_alloc_ctx_mem(bp);
7550                 if (rc)
7551                         return rc;
7552                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7553                 if (!rc)
7554                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7555         }
7556         return 0;
7557 }
7558
7559 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7560 {
7561         struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7562         struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7563         int rc = 0;
7564         u32 flags;
7565
7566         if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7567                 return 0;
7568
7569         resp = bp->hwrm_cmd_resp_addr;
7570         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7571
7572         mutex_lock(&bp->hwrm_cmd_lock);
7573         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7574         if (rc)
7575                 goto hwrm_cfa_adv_qcaps_exit;
7576
7577         flags = le32_to_cpu(resp->flags);
7578         if (flags &
7579             CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7580                 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7581
7582 hwrm_cfa_adv_qcaps_exit:
7583         mutex_unlock(&bp->hwrm_cmd_lock);
7584         return rc;
7585 }
7586
7587 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7588 {
7589         if (bp->fw_health)
7590                 return 0;
7591
7592         bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7593         if (!bp->fw_health)
7594                 return -ENOMEM;
7595
7596         return 0;
7597 }
7598
7599 static int bnxt_alloc_fw_health(struct bnxt *bp)
7600 {
7601         int rc;
7602
7603         if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7604             !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7605                 return 0;
7606
7607         rc = __bnxt_alloc_fw_health(bp);
7608         if (rc) {
7609                 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7610                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7611                 return rc;
7612         }
7613
7614         return 0;
7615 }
7616
7617 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7618 {
7619         writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7620                                          BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7621                                          BNXT_FW_HEALTH_WIN_MAP_OFF);
7622 }
7623
7624 bool bnxt_is_fw_healthy(struct bnxt *bp)
7625 {
7626         if (bp->fw_health && bp->fw_health->status_reliable) {
7627                 u32 fw_status;
7628
7629                 fw_status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
7630                 if (fw_status && !BNXT_FW_IS_HEALTHY(fw_status))
7631                         return false;
7632         }
7633
7634         return true;
7635 }
7636
7637 static void bnxt_inv_fw_health_reg(struct bnxt *bp)
7638 {
7639         struct bnxt_fw_health *fw_health = bp->fw_health;
7640         u32 reg_type;
7641
7642         if (!fw_health || !fw_health->status_reliable)
7643                 return;
7644
7645         reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
7646         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7647                 fw_health->status_reliable = false;
7648 }
7649
7650 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7651 {
7652         void __iomem *hs;
7653         u32 status_loc;
7654         u32 reg_type;
7655         u32 sig;
7656
7657         if (bp->fw_health)
7658                 bp->fw_health->status_reliable = false;
7659
7660         __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7661         hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7662
7663         sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7664         if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7665                 if (!bp->chip_num) {
7666                         __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
7667                         bp->chip_num = readl(bp->bar0 +
7668                                              BNXT_FW_HEALTH_WIN_BASE +
7669                                              BNXT_GRC_REG_CHIP_NUM);
7670                 }
7671                 if (!BNXT_CHIP_P5(bp))
7672                         return;
7673
7674                 status_loc = BNXT_GRC_REG_STATUS_P5 |
7675                              BNXT_FW_HEALTH_REG_TYPE_BAR0;
7676         } else {
7677                 status_loc = readl(hs + offsetof(struct hcomm_status,
7678                                                  fw_status_loc));
7679         }
7680
7681         if (__bnxt_alloc_fw_health(bp)) {
7682                 netdev_warn(bp->dev, "no memory for firmware status checks\n");
7683                 return;
7684         }
7685
7686         bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7687         reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7688         if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7689                 __bnxt_map_fw_health_reg(bp, status_loc);
7690                 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7691                         BNXT_FW_HEALTH_WIN_OFF(status_loc);
7692         }
7693
7694         bp->fw_health->status_reliable = true;
7695 }
7696
7697 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7698 {
7699         struct bnxt_fw_health *fw_health = bp->fw_health;
7700         u32 reg_base = 0xffffffff;
7701         int i;
7702
7703         bp->fw_health->status_reliable = false;
7704         /* Only pre-map the monitoring GRC registers using window 3 */
7705         for (i = 0; i < 4; i++) {
7706                 u32 reg = fw_health->regs[i];
7707
7708                 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7709                         continue;
7710                 if (reg_base == 0xffffffff)
7711                         reg_base = reg & BNXT_GRC_BASE_MASK;
7712                 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7713                         return -ERANGE;
7714                 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7715         }
7716         bp->fw_health->status_reliable = true;
7717         if (reg_base == 0xffffffff)
7718                 return 0;
7719
7720         __bnxt_map_fw_health_reg(bp, reg_base);
7721         return 0;
7722 }
7723
7724 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7725 {
7726         struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7727         struct bnxt_fw_health *fw_health = bp->fw_health;
7728         struct hwrm_error_recovery_qcfg_input req = {0};
7729         int rc, i;
7730
7731         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7732                 return 0;
7733
7734         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7735         mutex_lock(&bp->hwrm_cmd_lock);
7736         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7737         if (rc)
7738                 goto err_recovery_out;
7739         fw_health->flags = le32_to_cpu(resp->flags);
7740         if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7741             !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7742                 rc = -EINVAL;
7743                 goto err_recovery_out;
7744         }
7745         fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7746         fw_health->master_func_wait_dsecs =
7747                 le32_to_cpu(resp->master_func_wait_period);
7748         fw_health->normal_func_wait_dsecs =
7749                 le32_to_cpu(resp->normal_func_wait_period);
7750         fw_health->post_reset_wait_dsecs =
7751                 le32_to_cpu(resp->master_func_wait_period_after_reset);
7752         fw_health->post_reset_max_wait_dsecs =
7753                 le32_to_cpu(resp->max_bailout_time_after_reset);
7754         fw_health->regs[BNXT_FW_HEALTH_REG] =
7755                 le32_to_cpu(resp->fw_health_status_reg);
7756         fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7757                 le32_to_cpu(resp->fw_heartbeat_reg);
7758         fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7759                 le32_to_cpu(resp->fw_reset_cnt_reg);
7760         fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7761                 le32_to_cpu(resp->reset_inprogress_reg);
7762         fw_health->fw_reset_inprog_reg_mask =
7763                 le32_to_cpu(resp->reset_inprogress_reg_mask);
7764         fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7765         if (fw_health->fw_reset_seq_cnt >= 16) {
7766                 rc = -EINVAL;
7767                 goto err_recovery_out;
7768         }
7769         for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7770                 fw_health->fw_reset_seq_regs[i] =
7771                         le32_to_cpu(resp->reset_reg[i]);
7772                 fw_health->fw_reset_seq_vals[i] =
7773                         le32_to_cpu(resp->reset_reg_val[i]);
7774                 fw_health->fw_reset_seq_delay_msec[i] =
7775                         resp->delay_after_reset[i];
7776         }
7777 err_recovery_out:
7778         mutex_unlock(&bp->hwrm_cmd_lock);
7779         if (!rc)
7780                 rc = bnxt_map_fw_health_regs(bp);
7781         if (rc)
7782                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7783         return rc;
7784 }
7785
7786 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7787 {
7788         struct hwrm_func_reset_input req = {0};
7789
7790         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7791         req.enables = 0;
7792
7793         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7794 }
7795
7796 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
7797 {
7798         struct hwrm_nvm_get_dev_info_output nvm_info;
7799
7800         if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
7801                 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
7802                          nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
7803                          nvm_info.nvm_cfg_ver_upd);
7804 }
7805
7806 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7807 {
7808         int rc = 0;
7809         struct hwrm_queue_qportcfg_input req = {0};
7810         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7811         u8 i, j, *qptr;
7812         bool no_rdma;
7813
7814         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7815
7816         mutex_lock(&bp->hwrm_cmd_lock);
7817         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7818         if (rc)
7819                 goto qportcfg_exit;
7820
7821         if (!resp->max_configurable_queues) {
7822                 rc = -EINVAL;
7823                 goto qportcfg_exit;
7824         }
7825         bp->max_tc = resp->max_configurable_queues;
7826         bp->max_lltc = resp->max_configurable_lossless_queues;
7827         if (bp->max_tc > BNXT_MAX_QUEUE)
7828                 bp->max_tc = BNXT_MAX_QUEUE;
7829
7830         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7831         qptr = &resp->queue_id0;
7832         for (i = 0, j = 0; i < bp->max_tc; i++) {
7833                 bp->q_info[j].queue_id = *qptr;
7834                 bp->q_ids[i] = *qptr++;
7835                 bp->q_info[j].queue_profile = *qptr++;
7836                 bp->tc_to_qidx[j] = j;
7837                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7838                     (no_rdma && BNXT_PF(bp)))
7839                         j++;
7840         }
7841         bp->max_q = bp->max_tc;
7842         bp->max_tc = max_t(u8, j, 1);
7843
7844         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7845                 bp->max_tc = 1;
7846
7847         if (bp->max_lltc > bp->max_tc)
7848                 bp->max_lltc = bp->max_tc;
7849
7850 qportcfg_exit:
7851         mutex_unlock(&bp->hwrm_cmd_lock);
7852         return rc;
7853 }
7854
7855 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7856 {
7857         struct hwrm_ver_get_input req = {0};
7858         int rc;
7859
7860         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7861         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7862         req.hwrm_intf_min = HWRM_VERSION_MINOR;
7863         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7864
7865         rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7866                                    silent);
7867         return rc;
7868 }
7869
7870 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7871 {
7872         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7873         u16 fw_maj, fw_min, fw_bld, fw_rsv;
7874         u32 dev_caps_cfg, hwrm_ver;
7875         int rc, len;
7876
7877         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7878         mutex_lock(&bp->hwrm_cmd_lock);
7879         rc = __bnxt_hwrm_ver_get(bp, false);
7880         if (rc)
7881                 goto hwrm_ver_get_exit;
7882
7883         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7884
7885         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7886                              resp->hwrm_intf_min_8b << 8 |
7887                              resp->hwrm_intf_upd_8b;
7888         if (resp->hwrm_intf_maj_8b < 1) {
7889                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7890                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7891                             resp->hwrm_intf_upd_8b);
7892                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7893         }
7894
7895         hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
7896                         HWRM_VERSION_UPDATE;
7897
7898         if (bp->hwrm_spec_code > hwrm_ver)
7899                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7900                          HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
7901                          HWRM_VERSION_UPDATE);
7902         else
7903                 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7904                          resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7905                          resp->hwrm_intf_upd_8b);
7906
7907         fw_maj = le16_to_cpu(resp->hwrm_fw_major);
7908         if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
7909                 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
7910                 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
7911                 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
7912                 len = FW_VER_STR_LEN;
7913         } else {
7914                 fw_maj = resp->hwrm_fw_maj_8b;
7915                 fw_min = resp->hwrm_fw_min_8b;
7916                 fw_bld = resp->hwrm_fw_bld_8b;
7917                 fw_rsv = resp->hwrm_fw_rsvd_8b;
7918                 len = BC_HWRM_STR_LEN;
7919         }
7920         bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
7921         snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
7922                  fw_rsv);
7923
7924         if (strlen(resp->active_pkg_name)) {
7925                 int fw_ver_len = strlen(bp->fw_ver_str);
7926
7927                 snprintf(bp->fw_ver_str + fw_ver_len,
7928                          FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7929                          resp->active_pkg_name);
7930                 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7931         }
7932
7933         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7934         if (!bp->hwrm_cmd_timeout)
7935                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7936
7937         if (resp->hwrm_intf_maj_8b >= 1) {
7938                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7939                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7940         }
7941         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7942                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7943
7944         bp->chip_num = le16_to_cpu(resp->chip_num);
7945         bp->chip_rev = resp->chip_rev;
7946         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7947             !resp->chip_metal)
7948                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7949
7950         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7951         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7952             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7953                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7954
7955         if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7956                 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7957
7958         if (dev_caps_cfg &
7959             VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7960                 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7961
7962         if (dev_caps_cfg &
7963             VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7964                 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7965
7966         if (dev_caps_cfg &
7967             VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7968                 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7969
7970 hwrm_ver_get_exit:
7971         mutex_unlock(&bp->hwrm_cmd_lock);
7972         return rc;
7973 }
7974
7975 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7976 {
7977         struct hwrm_fw_set_time_input req = {0};
7978         struct tm tm;
7979         time64_t now = ktime_get_real_seconds();
7980
7981         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7982             bp->hwrm_spec_code < 0x10400)
7983                 return -EOPNOTSUPP;
7984
7985         time64_to_tm(now, 0, &tm);
7986         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7987         req.year = cpu_to_le16(1900 + tm.tm_year);
7988         req.month = 1 + tm.tm_mon;
7989         req.day = tm.tm_mday;
7990         req.hour = tm.tm_hour;
7991         req.minute = tm.tm_min;
7992         req.second = tm.tm_sec;
7993         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7994 }
7995
7996 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
7997 {
7998         u64 sw_tmp;
7999
8000         hw &= mask;
8001         sw_tmp = (*sw & ~mask) | hw;
8002         if (hw < (*sw & mask))
8003                 sw_tmp += mask + 1;
8004         WRITE_ONCE(*sw, sw_tmp);
8005 }
8006
8007 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
8008                                     int count, bool ignore_zero)
8009 {
8010         int i;
8011
8012         for (i = 0; i < count; i++) {
8013                 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
8014
8015                 if (ignore_zero && !hw)
8016                         continue;
8017
8018                 if (masks[i] == -1ULL)
8019                         sw_stats[i] = hw;
8020                 else
8021                         bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
8022         }
8023 }
8024
8025 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
8026 {
8027         if (!stats->hw_stats)
8028                 return;
8029
8030         __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8031                                 stats->hw_masks, stats->len / 8, false);
8032 }
8033
8034 static void bnxt_accumulate_all_stats(struct bnxt *bp)
8035 {
8036         struct bnxt_stats_mem *ring0_stats;
8037         bool ignore_zero = false;
8038         int i;
8039
8040         /* Chip bug.  Counter intermittently becomes 0. */
8041         if (bp->flags & BNXT_FLAG_CHIP_P5)
8042                 ignore_zero = true;
8043
8044         for (i = 0; i < bp->cp_nr_rings; i++) {
8045                 struct bnxt_napi *bnapi = bp->bnapi[i];
8046                 struct bnxt_cp_ring_info *cpr;
8047                 struct bnxt_stats_mem *stats;
8048
8049                 cpr = &bnapi->cp_ring;
8050                 stats = &cpr->stats;
8051                 if (!i)
8052                         ring0_stats = stats;
8053                 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8054                                         ring0_stats->hw_masks,
8055                                         ring0_stats->len / 8, ignore_zero);
8056         }
8057         if (bp->flags & BNXT_FLAG_PORT_STATS) {
8058                 struct bnxt_stats_mem *stats = &bp->port_stats;
8059                 __le64 *hw_stats = stats->hw_stats;
8060                 u64 *sw_stats = stats->sw_stats;
8061                 u64 *masks = stats->hw_masks;
8062                 int cnt;
8063
8064                 cnt = sizeof(struct rx_port_stats) / 8;
8065                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8066
8067                 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8068                 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8069                 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8070                 cnt = sizeof(struct tx_port_stats) / 8;
8071                 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8072         }
8073         if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
8074                 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
8075                 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
8076         }
8077 }
8078
8079 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
8080 {
8081         struct bnxt_pf_info *pf = &bp->pf;
8082         struct hwrm_port_qstats_input req = {0};
8083
8084         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
8085                 return 0;
8086
8087         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8088                 return -EOPNOTSUPP;
8089
8090         req.flags = flags;
8091         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
8092         req.port_id = cpu_to_le16(pf->port_id);
8093         req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
8094                                             BNXT_TX_PORT_STATS_BYTE_OFFSET);
8095         req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
8096         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8097 }
8098
8099 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
8100 {
8101         struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
8102         struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
8103         struct hwrm_port_qstats_ext_input req = {0};
8104         struct bnxt_pf_info *pf = &bp->pf;
8105         u32 tx_stat_size;
8106         int rc;
8107
8108         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
8109                 return 0;
8110
8111         if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8112                 return -EOPNOTSUPP;
8113
8114         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
8115         req.flags = flags;
8116         req.port_id = cpu_to_le16(pf->port_id);
8117         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
8118         req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
8119         tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
8120                        sizeof(struct tx_port_stats_ext) : 0;
8121         req.tx_stat_size = cpu_to_le16(tx_stat_size);
8122         req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
8123         mutex_lock(&bp->hwrm_cmd_lock);
8124         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8125         if (!rc) {
8126                 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
8127                 bp->fw_tx_stats_ext_size = tx_stat_size ?
8128                         le16_to_cpu(resp->tx_stat_size) / 8 : 0;
8129         } else {
8130                 bp->fw_rx_stats_ext_size = 0;
8131                 bp->fw_tx_stats_ext_size = 0;
8132         }
8133         if (flags)
8134                 goto qstats_done;
8135
8136         if (bp->fw_tx_stats_ext_size <=
8137             offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
8138                 mutex_unlock(&bp->hwrm_cmd_lock);
8139                 bp->pri2cos_valid = 0;
8140                 return rc;
8141         }
8142
8143         bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
8144         req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
8145
8146         rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
8147         if (!rc) {
8148                 struct hwrm_queue_pri2cos_qcfg_output *resp2;
8149                 u8 *pri2cos;
8150                 int i, j;
8151
8152                 resp2 = bp->hwrm_cmd_resp_addr;
8153                 pri2cos = &resp2->pri0_cos_queue_id;
8154                 for (i = 0; i < 8; i++) {
8155                         u8 queue_id = pri2cos[i];
8156                         u8 queue_idx;
8157
8158                         /* Per port queue IDs start from 0, 10, 20, etc */
8159                         queue_idx = queue_id % 10;
8160                         if (queue_idx > BNXT_MAX_QUEUE) {
8161                                 bp->pri2cos_valid = false;
8162                                 goto qstats_done;
8163                         }
8164                         for (j = 0; j < bp->max_q; j++) {
8165                                 if (bp->q_ids[j] == queue_id)
8166                                         bp->pri2cos_idx[i] = queue_idx;
8167                         }
8168                 }
8169                 bp->pri2cos_valid = 1;
8170         }
8171 qstats_done:
8172         mutex_unlock(&bp->hwrm_cmd_lock);
8173         return rc;
8174 }
8175
8176 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
8177 {
8178         if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID)
8179                 bnxt_hwrm_tunnel_dst_port_free(
8180                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
8181         if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID)
8182                 bnxt_hwrm_tunnel_dst_port_free(
8183                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
8184 }
8185
8186 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
8187 {
8188         int rc, i;
8189         u32 tpa_flags = 0;
8190
8191         if (set_tpa)
8192                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
8193         else if (BNXT_NO_FW_ACCESS(bp))
8194                 return 0;
8195         for (i = 0; i < bp->nr_vnics; i++) {
8196                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
8197                 if (rc) {
8198                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
8199                                    i, rc);
8200                         return rc;
8201                 }
8202         }
8203         return 0;
8204 }
8205
8206 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
8207 {
8208         int i;
8209
8210         for (i = 0; i < bp->nr_vnics; i++)
8211                 bnxt_hwrm_vnic_set_rss(bp, i, false);
8212 }
8213
8214 static void bnxt_clear_vnic(struct bnxt *bp)
8215 {
8216         if (!bp->vnic_info)
8217                 return;
8218
8219         bnxt_hwrm_clear_vnic_filter(bp);
8220         if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
8221                 /* clear all RSS setting before free vnic ctx */
8222                 bnxt_hwrm_clear_vnic_rss(bp);
8223                 bnxt_hwrm_vnic_ctx_free(bp);
8224         }
8225         /* before free the vnic, undo the vnic tpa settings */
8226         if (bp->flags & BNXT_FLAG_TPA)
8227                 bnxt_set_tpa(bp, false);
8228         bnxt_hwrm_vnic_free(bp);
8229         if (bp->flags & BNXT_FLAG_CHIP_P5)
8230                 bnxt_hwrm_vnic_ctx_free(bp);
8231 }
8232
8233 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
8234                                     bool irq_re_init)
8235 {
8236         bnxt_clear_vnic(bp);
8237         bnxt_hwrm_ring_free(bp, close_path);
8238         bnxt_hwrm_ring_grp_free(bp);
8239         if (irq_re_init) {
8240                 bnxt_hwrm_stat_ctx_free(bp);
8241                 bnxt_hwrm_free_tunnel_ports(bp);
8242         }
8243 }
8244
8245 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8246 {
8247         struct hwrm_func_cfg_input req = {0};
8248
8249         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8250         req.fid = cpu_to_le16(0xffff);
8251         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8252         if (br_mode == BRIDGE_MODE_VEB)
8253                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8254         else if (br_mode == BRIDGE_MODE_VEPA)
8255                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8256         else
8257                 return -EINVAL;
8258         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8259 }
8260
8261 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8262 {
8263         struct hwrm_func_cfg_input req = {0};
8264
8265         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8266                 return 0;
8267
8268         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8269         req.fid = cpu_to_le16(0xffff);
8270         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8271         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8272         if (size == 128)
8273                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8274
8275         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8276 }
8277
8278 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8279 {
8280         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8281         int rc;
8282
8283         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8284                 goto skip_rss_ctx;
8285
8286         /* allocate context for vnic */
8287         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8288         if (rc) {
8289                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8290                            vnic_id, rc);
8291                 goto vnic_setup_err;
8292         }
8293         bp->rsscos_nr_ctxs++;
8294
8295         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8296                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8297                 if (rc) {
8298                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8299                                    vnic_id, rc);
8300                         goto vnic_setup_err;
8301                 }
8302                 bp->rsscos_nr_ctxs++;
8303         }
8304
8305 skip_rss_ctx:
8306         /* configure default vnic, ring grp */
8307         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8308         if (rc) {
8309                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8310                            vnic_id, rc);
8311                 goto vnic_setup_err;
8312         }
8313
8314         /* Enable RSS hashing on vnic */
8315         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8316         if (rc) {
8317                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8318                            vnic_id, rc);
8319                 goto vnic_setup_err;
8320         }
8321
8322         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8323                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8324                 if (rc) {
8325                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8326                                    vnic_id, rc);
8327                 }
8328         }
8329
8330 vnic_setup_err:
8331         return rc;
8332 }
8333
8334 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8335 {
8336         int rc, i, nr_ctxs;
8337
8338         nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8339         for (i = 0; i < nr_ctxs; i++) {
8340                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8341                 if (rc) {
8342                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8343                                    vnic_id, i, rc);
8344                         break;
8345                 }
8346                 bp->rsscos_nr_ctxs++;
8347         }
8348         if (i < nr_ctxs)
8349                 return -ENOMEM;
8350
8351         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8352         if (rc) {
8353                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8354                            vnic_id, rc);
8355                 return rc;
8356         }
8357         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8358         if (rc) {
8359                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8360                            vnic_id, rc);
8361                 return rc;
8362         }
8363         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8364                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8365                 if (rc) {
8366                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8367                                    vnic_id, rc);
8368                 }
8369         }
8370         return rc;
8371 }
8372
8373 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8374 {
8375         if (bp->flags & BNXT_FLAG_CHIP_P5)
8376                 return __bnxt_setup_vnic_p5(bp, vnic_id);
8377         else
8378                 return __bnxt_setup_vnic(bp, vnic_id);
8379 }
8380
8381 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8382 {
8383 #ifdef CONFIG_RFS_ACCEL
8384         int i, rc = 0;
8385
8386         if (bp->flags & BNXT_FLAG_CHIP_P5)
8387                 return 0;
8388
8389         for (i = 0; i < bp->rx_nr_rings; i++) {
8390                 struct bnxt_vnic_info *vnic;
8391                 u16 vnic_id = i + 1;
8392                 u16 ring_id = i;
8393
8394                 if (vnic_id >= bp->nr_vnics)
8395                         break;
8396
8397                 vnic = &bp->vnic_info[vnic_id];
8398                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
8399                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8400                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8401                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8402                 if (rc) {
8403                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8404                                    vnic_id, rc);
8405                         break;
8406                 }
8407                 rc = bnxt_setup_vnic(bp, vnic_id);
8408                 if (rc)
8409                         break;
8410         }
8411         return rc;
8412 #else
8413         return 0;
8414 #endif
8415 }
8416
8417 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
8418 static bool bnxt_promisc_ok(struct bnxt *bp)
8419 {
8420 #ifdef CONFIG_BNXT_SRIOV
8421         if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
8422                 return false;
8423 #endif
8424         return true;
8425 }
8426
8427 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8428 {
8429         unsigned int rc = 0;
8430
8431         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8432         if (rc) {
8433                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8434                            rc);
8435                 return rc;
8436         }
8437
8438         rc = bnxt_hwrm_vnic_cfg(bp, 1);
8439         if (rc) {
8440                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8441                            rc);
8442                 return rc;
8443         }
8444         return rc;
8445 }
8446
8447 static int bnxt_cfg_rx_mode(struct bnxt *);
8448 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8449
8450 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8451 {
8452         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8453         int rc = 0;
8454         unsigned int rx_nr_rings = bp->rx_nr_rings;
8455
8456         if (irq_re_init) {
8457                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
8458                 if (rc) {
8459                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8460                                    rc);
8461                         goto err_out;
8462                 }
8463         }
8464
8465         rc = bnxt_hwrm_ring_alloc(bp);
8466         if (rc) {
8467                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8468                 goto err_out;
8469         }
8470
8471         rc = bnxt_hwrm_ring_grp_alloc(bp);
8472         if (rc) {
8473                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8474                 goto err_out;
8475         }
8476
8477         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8478                 rx_nr_rings--;
8479
8480         /* default vnic 0 */
8481         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8482         if (rc) {
8483                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8484                 goto err_out;
8485         }
8486
8487         rc = bnxt_setup_vnic(bp, 0);
8488         if (rc)
8489                 goto err_out;
8490
8491         if (bp->flags & BNXT_FLAG_RFS) {
8492                 rc = bnxt_alloc_rfs_vnics(bp);
8493                 if (rc)
8494                         goto err_out;
8495         }
8496
8497         if (bp->flags & BNXT_FLAG_TPA) {
8498                 rc = bnxt_set_tpa(bp, true);
8499                 if (rc)
8500                         goto err_out;
8501         }
8502
8503         if (BNXT_VF(bp))
8504                 bnxt_update_vf_mac(bp);
8505
8506         /* Filter for default vnic 0 */
8507         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8508         if (rc) {
8509                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8510                 goto err_out;
8511         }
8512         vnic->uc_filter_count = 1;
8513
8514         vnic->rx_mask = 0;
8515         if (bp->dev->flags & IFF_BROADCAST)
8516                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8517
8518         if (bp->dev->flags & IFF_PROMISC)
8519                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8520
8521         if (bp->dev->flags & IFF_ALLMULTI) {
8522                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8523                 vnic->mc_list_count = 0;
8524         } else {
8525                 u32 mask = 0;
8526
8527                 bnxt_mc_list_updated(bp, &mask);
8528                 vnic->rx_mask |= mask;
8529         }
8530
8531         rc = bnxt_cfg_rx_mode(bp);
8532         if (rc)
8533                 goto err_out;
8534
8535         rc = bnxt_hwrm_set_coal(bp);
8536         if (rc)
8537                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8538                                 rc);
8539
8540         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8541                 rc = bnxt_setup_nitroa0_vnic(bp);
8542                 if (rc)
8543                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8544                                    rc);
8545         }
8546
8547         if (BNXT_VF(bp)) {
8548                 bnxt_hwrm_func_qcfg(bp);
8549                 netdev_update_features(bp->dev);
8550         }
8551
8552         return 0;
8553
8554 err_out:
8555         bnxt_hwrm_resource_free(bp, 0, true);
8556
8557         return rc;
8558 }
8559
8560 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8561 {
8562         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8563         return 0;
8564 }
8565
8566 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8567 {
8568         bnxt_init_cp_rings(bp);
8569         bnxt_init_rx_rings(bp);
8570         bnxt_init_tx_rings(bp);
8571         bnxt_init_ring_grps(bp, irq_re_init);
8572         bnxt_init_vnics(bp);
8573
8574         return bnxt_init_chip(bp, irq_re_init);
8575 }
8576
8577 static int bnxt_set_real_num_queues(struct bnxt *bp)
8578 {
8579         int rc;
8580         struct net_device *dev = bp->dev;
8581
8582         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8583                                           bp->tx_nr_rings_xdp);
8584         if (rc)
8585                 return rc;
8586
8587         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8588         if (rc)
8589                 return rc;
8590
8591 #ifdef CONFIG_RFS_ACCEL
8592         if (bp->flags & BNXT_FLAG_RFS)
8593                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8594 #endif
8595
8596         return rc;
8597 }
8598
8599 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8600                            bool shared)
8601 {
8602         int _rx = *rx, _tx = *tx;
8603
8604         if (shared) {
8605                 *rx = min_t(int, _rx, max);
8606                 *tx = min_t(int, _tx, max);
8607         } else {
8608                 if (max < 2)
8609                         return -ENOMEM;
8610
8611                 while (_rx + _tx > max) {
8612                         if (_rx > _tx && _rx > 1)
8613                                 _rx--;
8614                         else if (_tx > 1)
8615                                 _tx--;
8616                 }
8617                 *rx = _rx;
8618                 *tx = _tx;
8619         }
8620         return 0;
8621 }
8622
8623 static void bnxt_setup_msix(struct bnxt *bp)
8624 {
8625         const int len = sizeof(bp->irq_tbl[0].name);
8626         struct net_device *dev = bp->dev;
8627         int tcs, i;
8628
8629         tcs = netdev_get_num_tc(dev);
8630         if (tcs) {
8631                 int i, off, count;
8632
8633                 for (i = 0; i < tcs; i++) {
8634                         count = bp->tx_nr_rings_per_tc;
8635                         off = i * count;
8636                         netdev_set_tc_queue(dev, i, count, off);
8637                 }
8638         }
8639
8640         for (i = 0; i < bp->cp_nr_rings; i++) {
8641                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8642                 char *attr;
8643
8644                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8645                         attr = "TxRx";
8646                 else if (i < bp->rx_nr_rings)
8647                         attr = "rx";
8648                 else
8649                         attr = "tx";
8650
8651                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8652                          attr, i);
8653                 bp->irq_tbl[map_idx].handler = bnxt_msix;
8654         }
8655 }
8656
8657 static void bnxt_setup_inta(struct bnxt *bp)
8658 {
8659         const int len = sizeof(bp->irq_tbl[0].name);
8660
8661         if (netdev_get_num_tc(bp->dev))
8662                 netdev_reset_tc(bp->dev);
8663
8664         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
8665                  0);
8666         bp->irq_tbl[0].handler = bnxt_inta;
8667 }
8668
8669 static int bnxt_init_int_mode(struct bnxt *bp);
8670
8671 static int bnxt_setup_int_mode(struct bnxt *bp)
8672 {
8673         int rc;
8674
8675         if (!bp->irq_tbl) {
8676                 rc = bnxt_init_int_mode(bp);
8677                 if (rc || !bp->irq_tbl)
8678                         return rc ?: -ENODEV;
8679         }
8680
8681         if (bp->flags & BNXT_FLAG_USING_MSIX)
8682                 bnxt_setup_msix(bp);
8683         else
8684                 bnxt_setup_inta(bp);
8685
8686         rc = bnxt_set_real_num_queues(bp);
8687         return rc;
8688 }
8689
8690 #ifdef CONFIG_RFS_ACCEL
8691 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
8692 {
8693         return bp->hw_resc.max_rsscos_ctxs;
8694 }
8695
8696 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
8697 {
8698         return bp->hw_resc.max_vnics;
8699 }
8700 #endif
8701
8702 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
8703 {
8704         return bp->hw_resc.max_stat_ctxs;
8705 }
8706
8707 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
8708 {
8709         return bp->hw_resc.max_cp_rings;
8710 }
8711
8712 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
8713 {
8714         unsigned int cp = bp->hw_resc.max_cp_rings;
8715
8716         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8717                 cp -= bnxt_get_ulp_msix_num(bp);
8718
8719         return cp;
8720 }
8721
8722 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
8723 {
8724         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8725
8726         if (bp->flags & BNXT_FLAG_CHIP_P5)
8727                 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
8728
8729         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
8730 }
8731
8732 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
8733 {
8734         bp->hw_resc.max_irqs = max_irqs;
8735 }
8736
8737 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
8738 {
8739         unsigned int cp;
8740
8741         cp = bnxt_get_max_func_cp_rings_for_en(bp);
8742         if (bp->flags & BNXT_FLAG_CHIP_P5)
8743                 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
8744         else
8745                 return cp - bp->cp_nr_rings;
8746 }
8747
8748 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
8749 {
8750         return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8751 }
8752
8753 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8754 {
8755         int max_cp = bnxt_get_max_func_cp_rings(bp);
8756         int max_irq = bnxt_get_max_func_irqs(bp);
8757         int total_req = bp->cp_nr_rings + num;
8758         int max_idx, avail_msix;
8759
8760         max_idx = bp->total_irqs;
8761         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8762                 max_idx = min_t(int, bp->total_irqs, max_cp);
8763         avail_msix = max_idx - bp->cp_nr_rings;
8764         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8765                 return avail_msix;
8766
8767         if (max_irq < total_req) {
8768                 num = max_irq - bp->cp_nr_rings;
8769                 if (num <= 0)
8770                         return 0;
8771         }
8772         return num;
8773 }
8774
8775 static int bnxt_get_num_msix(struct bnxt *bp)
8776 {
8777         if (!BNXT_NEW_RM(bp))
8778                 return bnxt_get_max_func_irqs(bp);
8779
8780         return bnxt_nq_rings_in_use(bp);
8781 }
8782
8783 static int bnxt_init_msix(struct bnxt *bp)
8784 {
8785         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8786         struct msix_entry *msix_ent;
8787
8788         total_vecs = bnxt_get_num_msix(bp);
8789         max = bnxt_get_max_func_irqs(bp);
8790         if (total_vecs > max)
8791                 total_vecs = max;
8792
8793         if (!total_vecs)
8794                 return 0;
8795
8796         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8797         if (!msix_ent)
8798                 return -ENOMEM;
8799
8800         for (i = 0; i < total_vecs; i++) {
8801                 msix_ent[i].entry = i;
8802                 msix_ent[i].vector = 0;
8803         }
8804
8805         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8806                 min = 2;
8807
8808         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8809         ulp_msix = bnxt_get_ulp_msix_num(bp);
8810         if (total_vecs < 0 || total_vecs < ulp_msix) {
8811                 rc = -ENODEV;
8812                 goto msix_setup_exit;
8813         }
8814
8815         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8816         if (bp->irq_tbl) {
8817                 for (i = 0; i < total_vecs; i++)
8818                         bp->irq_tbl[i].vector = msix_ent[i].vector;
8819
8820                 bp->total_irqs = total_vecs;
8821                 /* Trim rings based upon num of vectors allocated */
8822                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8823                                      total_vecs - ulp_msix, min == 1);
8824                 if (rc)
8825                         goto msix_setup_exit;
8826
8827                 bp->cp_nr_rings = (min == 1) ?
8828                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8829                                   bp->tx_nr_rings + bp->rx_nr_rings;
8830
8831         } else {
8832                 rc = -ENOMEM;
8833                 goto msix_setup_exit;
8834         }
8835         bp->flags |= BNXT_FLAG_USING_MSIX;
8836         kfree(msix_ent);
8837         return 0;
8838
8839 msix_setup_exit:
8840         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8841         kfree(bp->irq_tbl);
8842         bp->irq_tbl = NULL;
8843         pci_disable_msix(bp->pdev);
8844         kfree(msix_ent);
8845         return rc;
8846 }
8847
8848 static int bnxt_init_inta(struct bnxt *bp)
8849 {
8850         bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL);
8851         if (!bp->irq_tbl)
8852                 return -ENOMEM;
8853
8854         bp->total_irqs = 1;
8855         bp->rx_nr_rings = 1;
8856         bp->tx_nr_rings = 1;
8857         bp->cp_nr_rings = 1;
8858         bp->flags |= BNXT_FLAG_SHARED_RINGS;
8859         bp->irq_tbl[0].vector = bp->pdev->irq;
8860         return 0;
8861 }
8862
8863 static int bnxt_init_int_mode(struct bnxt *bp)
8864 {
8865         int rc = -ENODEV;
8866
8867         if (bp->flags & BNXT_FLAG_MSIX_CAP)
8868                 rc = bnxt_init_msix(bp);
8869
8870         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8871                 /* fallback to INTA */
8872                 rc = bnxt_init_inta(bp);
8873         }
8874         return rc;
8875 }
8876
8877 static void bnxt_clear_int_mode(struct bnxt *bp)
8878 {
8879         if (bp->flags & BNXT_FLAG_USING_MSIX)
8880                 pci_disable_msix(bp->pdev);
8881
8882         kfree(bp->irq_tbl);
8883         bp->irq_tbl = NULL;
8884         bp->flags &= ~BNXT_FLAG_USING_MSIX;
8885 }
8886
8887 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8888 {
8889         int tcs = netdev_get_num_tc(bp->dev);
8890         bool irq_cleared = false;
8891         int rc;
8892
8893         if (!bnxt_need_reserve_rings(bp))
8894                 return 0;
8895
8896         if (irq_re_init && BNXT_NEW_RM(bp) &&
8897             bnxt_get_num_msix(bp) != bp->total_irqs) {
8898                 bnxt_ulp_irq_stop(bp);
8899                 bnxt_clear_int_mode(bp);
8900                 irq_cleared = true;
8901         }
8902         rc = __bnxt_reserve_rings(bp);
8903         if (irq_cleared) {
8904                 if (!rc)
8905                         rc = bnxt_init_int_mode(bp);
8906                 bnxt_ulp_irq_restart(bp, rc);
8907         }
8908         if (rc) {
8909                 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8910                 return rc;
8911         }
8912         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8913                 netdev_err(bp->dev, "tx ring reservation failure\n");
8914                 netdev_reset_tc(bp->dev);
8915                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8916                 return -ENOMEM;
8917         }
8918         return 0;
8919 }
8920
8921 static void bnxt_free_irq(struct bnxt *bp)
8922 {
8923         struct bnxt_irq *irq;
8924         int i;
8925
8926 #ifdef CONFIG_RFS_ACCEL
8927         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8928         bp->dev->rx_cpu_rmap = NULL;
8929 #endif
8930         if (!bp->irq_tbl || !bp->bnapi)
8931                 return;
8932
8933         for (i = 0; i < bp->cp_nr_rings; i++) {
8934                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8935
8936                 irq = &bp->irq_tbl[map_idx];
8937                 if (irq->requested) {
8938                         if (irq->have_cpumask) {
8939                                 irq_set_affinity_hint(irq->vector, NULL);
8940                                 free_cpumask_var(irq->cpu_mask);
8941                                 irq->have_cpumask = 0;
8942                         }
8943                         free_irq(irq->vector, bp->bnapi[i]);
8944                 }
8945
8946                 irq->requested = 0;
8947         }
8948 }
8949
8950 static int bnxt_request_irq(struct bnxt *bp)
8951 {
8952         int i, j, rc = 0;
8953         unsigned long flags = 0;
8954 #ifdef CONFIG_RFS_ACCEL
8955         struct cpu_rmap *rmap;
8956 #endif
8957
8958         rc = bnxt_setup_int_mode(bp);
8959         if (rc) {
8960                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8961                            rc);
8962                 return rc;
8963         }
8964 #ifdef CONFIG_RFS_ACCEL
8965         rmap = bp->dev->rx_cpu_rmap;
8966 #endif
8967         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8968                 flags = IRQF_SHARED;
8969
8970         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8971                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8972                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8973
8974 #ifdef CONFIG_RFS_ACCEL
8975                 if (rmap && bp->bnapi[i]->rx_ring) {
8976                         rc = irq_cpu_rmap_add(rmap, irq->vector);
8977                         if (rc)
8978                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8979                                             j);
8980                         j++;
8981                 }
8982 #endif
8983                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8984                                  bp->bnapi[i]);
8985                 if (rc)
8986                         break;
8987
8988                 irq->requested = 1;
8989
8990                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8991                         int numa_node = dev_to_node(&bp->pdev->dev);
8992
8993                         irq->have_cpumask = 1;
8994                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8995                                         irq->cpu_mask);
8996                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8997                         if (rc) {
8998                                 netdev_warn(bp->dev,
8999                                             "Set affinity failed, IRQ = %d\n",
9000                                             irq->vector);
9001                                 break;
9002                         }
9003                 }
9004         }
9005         return rc;
9006 }
9007
9008 static void bnxt_del_napi(struct bnxt *bp)
9009 {
9010         int i;
9011
9012         if (!bp->bnapi)
9013                 return;
9014
9015         for (i = 0; i < bp->cp_nr_rings; i++) {
9016                 struct bnxt_napi *bnapi = bp->bnapi[i];
9017
9018                 __netif_napi_del(&bnapi->napi);
9019         }
9020         /* We called __netif_napi_del(), we need
9021          * to respect an RCU grace period before freeing napi structures.
9022          */
9023         synchronize_net();
9024 }
9025
9026 static void bnxt_init_napi(struct bnxt *bp)
9027 {
9028         int i;
9029         unsigned int cp_nr_rings = bp->cp_nr_rings;
9030         struct bnxt_napi *bnapi;
9031
9032         if (bp->flags & BNXT_FLAG_USING_MSIX) {
9033                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
9034
9035                 if (bp->flags & BNXT_FLAG_CHIP_P5)
9036                         poll_fn = bnxt_poll_p5;
9037                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
9038                         cp_nr_rings--;
9039                 for (i = 0; i < cp_nr_rings; i++) {
9040                         bnapi = bp->bnapi[i];
9041                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
9042                 }
9043                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9044                         bnapi = bp->bnapi[cp_nr_rings];
9045                         netif_napi_add(bp->dev, &bnapi->napi,
9046                                        bnxt_poll_nitroa0, 64);
9047                 }
9048         } else {
9049                 bnapi = bp->bnapi[0];
9050                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
9051         }
9052 }
9053
9054 static void bnxt_disable_napi(struct bnxt *bp)
9055 {
9056         int i;
9057
9058         if (!bp->bnapi ||
9059             test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
9060                 return;
9061
9062         for (i = 0; i < bp->cp_nr_rings; i++) {
9063                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
9064
9065                 if (bp->bnapi[i]->rx_ring)
9066                         cancel_work_sync(&cpr->dim.work);
9067
9068                 napi_disable(&bp->bnapi[i]->napi);
9069         }
9070 }
9071
9072 static void bnxt_enable_napi(struct bnxt *bp)
9073 {
9074         int i;
9075
9076         clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
9077         for (i = 0; i < bp->cp_nr_rings; i++) {
9078                 struct bnxt_napi *bnapi = bp->bnapi[i];
9079                 struct bnxt_cp_ring_info *cpr;
9080
9081                 cpr = &bnapi->cp_ring;
9082                 if (bnapi->in_reset)
9083                         cpr->sw_stats.rx.rx_resets++;
9084                 bnapi->in_reset = false;
9085
9086                 if (bnapi->rx_ring) {
9087                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
9088                         cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
9089                 }
9090                 napi_enable(&bnapi->napi);
9091         }
9092 }
9093
9094 void bnxt_tx_disable(struct bnxt *bp)
9095 {
9096         int i;
9097         struct bnxt_tx_ring_info *txr;
9098
9099         if (bp->tx_ring) {
9100                 for (i = 0; i < bp->tx_nr_rings; i++) {
9101                         txr = &bp->tx_ring[i];
9102                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
9103                 }
9104         }
9105         /* Drop carrier first to prevent TX timeout */
9106         netif_carrier_off(bp->dev);
9107         /* Stop all TX queues */
9108         netif_tx_disable(bp->dev);
9109 }
9110
9111 void bnxt_tx_enable(struct bnxt *bp)
9112 {
9113         int i;
9114         struct bnxt_tx_ring_info *txr;
9115
9116         for (i = 0; i < bp->tx_nr_rings; i++) {
9117                 txr = &bp->tx_ring[i];
9118                 txr->dev_state = 0;
9119         }
9120         netif_tx_wake_all_queues(bp->dev);
9121         if (bp->link_info.link_up)
9122                 netif_carrier_on(bp->dev);
9123 }
9124
9125 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
9126 {
9127         u8 active_fec = link_info->active_fec_sig_mode &
9128                         PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
9129
9130         switch (active_fec) {
9131         default:
9132         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
9133                 return "None";
9134         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
9135                 return "Clause 74 BaseR";
9136         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
9137                 return "Clause 91 RS(528,514)";
9138         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
9139                 return "Clause 91 RS544_1XN";
9140         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
9141                 return "Clause 91 RS(544,514)";
9142         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
9143                 return "Clause 91 RS272_1XN";
9144         case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
9145                 return "Clause 91 RS(272,257)";
9146         }
9147 }
9148
9149 static void bnxt_report_link(struct bnxt *bp)
9150 {
9151         if (bp->link_info.link_up) {
9152                 const char *signal = "";
9153                 const char *flow_ctrl;
9154                 const char *duplex;
9155                 u32 speed;
9156                 u16 fec;
9157
9158                 netif_carrier_on(bp->dev);
9159                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
9160                 if (speed == SPEED_UNKNOWN) {
9161                         netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
9162                         return;
9163                 }
9164                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
9165                         duplex = "full";
9166                 else
9167                         duplex = "half";
9168                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
9169                         flow_ctrl = "ON - receive & transmit";
9170                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
9171                         flow_ctrl = "ON - transmit";
9172                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
9173                         flow_ctrl = "ON - receive";
9174                 else
9175                         flow_ctrl = "none";
9176                 if (bp->link_info.phy_qcfg_resp.option_flags &
9177                     PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
9178                         u8 sig_mode = bp->link_info.active_fec_sig_mode &
9179                                       PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
9180                         switch (sig_mode) {
9181                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
9182                                 signal = "(NRZ) ";
9183                                 break;
9184                         case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
9185                                 signal = "(PAM4) ";
9186                                 break;
9187                         default:
9188                                 break;
9189                         }
9190                 }
9191                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
9192                             speed, signal, duplex, flow_ctrl);
9193                 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
9194                         netdev_info(bp->dev, "EEE is %s\n",
9195                                     bp->eee.eee_active ? "active" :
9196                                                          "not active");
9197                 fec = bp->link_info.fec_cfg;
9198                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
9199                         netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
9200                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
9201                                     bnxt_report_fec(&bp->link_info));
9202         } else {
9203                 netif_carrier_off(bp->dev);
9204                 netdev_err(bp->dev, "NIC Link is Down\n");
9205         }
9206 }
9207
9208 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
9209 {
9210         if (!resp->supported_speeds_auto_mode &&
9211             !resp->supported_speeds_force_mode &&
9212             !resp->supported_pam4_speeds_auto_mode &&
9213             !resp->supported_pam4_speeds_force_mode)
9214                 return true;
9215         return false;
9216 }
9217
9218 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
9219 {
9220         int rc = 0;
9221         struct hwrm_port_phy_qcaps_input req = {0};
9222         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9223         struct bnxt_link_info *link_info = &bp->link_info;
9224
9225         if (bp->hwrm_spec_code < 0x10201)
9226                 return 0;
9227
9228         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
9229
9230         mutex_lock(&bp->hwrm_cmd_lock);
9231         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9232         if (rc)
9233                 goto hwrm_phy_qcaps_exit;
9234
9235         bp->phy_flags = resp->flags;
9236         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
9237                 struct ethtool_eee *eee = &bp->eee;
9238                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
9239
9240                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9241                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
9242                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
9243                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
9244                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
9245         }
9246
9247         if (bp->hwrm_spec_code >= 0x10a01) {
9248                 if (bnxt_phy_qcaps_no_speed(resp)) {
9249                         link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9250                         netdev_warn(bp->dev, "Ethernet link disabled\n");
9251                 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9252                         link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9253                         netdev_info(bp->dev, "Ethernet link enabled\n");
9254                         /* Phy re-enabled, reprobe the speeds */
9255                         link_info->support_auto_speeds = 0;
9256                         link_info->support_pam4_auto_speeds = 0;
9257                 }
9258         }
9259         if (resp->supported_speeds_auto_mode)
9260                 link_info->support_auto_speeds =
9261                         le16_to_cpu(resp->supported_speeds_auto_mode);
9262         if (resp->supported_pam4_speeds_auto_mode)
9263                 link_info->support_pam4_auto_speeds =
9264                         le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9265
9266         bp->port_count = resp->port_cnt;
9267
9268 hwrm_phy_qcaps_exit:
9269         mutex_unlock(&bp->hwrm_cmd_lock);
9270         return rc;
9271 }
9272
9273 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9274 {
9275         u16 diff = advertising ^ supported;
9276
9277         return ((supported | diff) != supported);
9278 }
9279
9280 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9281 {
9282         int rc = 0;
9283         struct bnxt_link_info *link_info = &bp->link_info;
9284         struct hwrm_port_phy_qcfg_input req = {0};
9285         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9286         u8 link_up = link_info->link_up;
9287         bool support_changed = false;
9288
9289         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
9290
9291         mutex_lock(&bp->hwrm_cmd_lock);
9292         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9293         if (rc) {
9294                 mutex_unlock(&bp->hwrm_cmd_lock);
9295                 return rc;
9296         }
9297
9298         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9299         link_info->phy_link_status = resp->link;
9300         link_info->duplex = resp->duplex_cfg;
9301         if (bp->hwrm_spec_code >= 0x10800)
9302                 link_info->duplex = resp->duplex_state;
9303         link_info->pause = resp->pause;
9304         link_info->auto_mode = resp->auto_mode;
9305         link_info->auto_pause_setting = resp->auto_pause;
9306         link_info->lp_pause = resp->link_partner_adv_pause;
9307         link_info->force_pause_setting = resp->force_pause;
9308         link_info->duplex_setting = resp->duplex_cfg;
9309         if (link_info->phy_link_status == BNXT_LINK_LINK)
9310                 link_info->link_speed = le16_to_cpu(resp->link_speed);
9311         else
9312                 link_info->link_speed = 0;
9313         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9314         link_info->force_pam4_link_speed =
9315                 le16_to_cpu(resp->force_pam4_link_speed);
9316         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9317         link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9318         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9319         link_info->auto_pam4_link_speeds =
9320                 le16_to_cpu(resp->auto_pam4_link_speed_mask);
9321         link_info->lp_auto_link_speeds =
9322                 le16_to_cpu(resp->link_partner_adv_speeds);
9323         link_info->lp_auto_pam4_link_speeds =
9324                 resp->link_partner_pam4_adv_speeds;
9325         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9326         link_info->phy_ver[0] = resp->phy_maj;
9327         link_info->phy_ver[1] = resp->phy_min;
9328         link_info->phy_ver[2] = resp->phy_bld;
9329         link_info->media_type = resp->media_type;
9330         link_info->phy_type = resp->phy_type;
9331         link_info->transceiver = resp->xcvr_pkg_type;
9332         link_info->phy_addr = resp->eee_config_phy_addr &
9333                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9334         link_info->module_status = resp->module_status;
9335
9336         if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
9337                 struct ethtool_eee *eee = &bp->eee;
9338                 u16 fw_speeds;
9339
9340                 eee->eee_active = 0;
9341                 if (resp->eee_config_phy_addr &
9342                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9343                         eee->eee_active = 1;
9344                         fw_speeds = le16_to_cpu(
9345                                 resp->link_partner_adv_eee_link_speed_mask);
9346                         eee->lp_advertised =
9347                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9348                 }
9349
9350                 /* Pull initial EEE config */
9351                 if (!chng_link_state) {
9352                         if (resp->eee_config_phy_addr &
9353                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9354                                 eee->eee_enabled = 1;
9355
9356                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9357                         eee->advertised =
9358                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9359
9360                         if (resp->eee_config_phy_addr &
9361                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9362                                 __le32 tmr;
9363
9364                                 eee->tx_lpi_enabled = 1;
9365                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9366                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
9367                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9368                         }
9369                 }
9370         }
9371
9372         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9373         if (bp->hwrm_spec_code >= 0x10504) {
9374                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9375                 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9376         }
9377         /* TODO: need to add more logic to report VF link */
9378         if (chng_link_state) {
9379                 if (link_info->phy_link_status == BNXT_LINK_LINK)
9380                         link_info->link_up = 1;
9381                 else
9382                         link_info->link_up = 0;
9383                 if (link_up != link_info->link_up)
9384                         bnxt_report_link(bp);
9385         } else {
9386                 /* alwasy link down if not require to update link state */
9387                 link_info->link_up = 0;
9388         }
9389         mutex_unlock(&bp->hwrm_cmd_lock);
9390
9391         if (!BNXT_PHY_CFG_ABLE(bp))
9392                 return 0;
9393
9394         /* Check if any advertised speeds are no longer supported. The caller
9395          * holds the link_lock mutex, so we can modify link_info settings.
9396          */
9397         if (bnxt_support_dropped(link_info->advertising,
9398                                  link_info->support_auto_speeds)) {
9399                 link_info->advertising = link_info->support_auto_speeds;
9400                 support_changed = true;
9401         }
9402         if (bnxt_support_dropped(link_info->advertising_pam4,
9403                                  link_info->support_pam4_auto_speeds)) {
9404                 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9405                 support_changed = true;
9406         }
9407         if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9408                 bnxt_hwrm_set_link_setting(bp, true, false);
9409         return 0;
9410 }
9411
9412 static void bnxt_get_port_module_status(struct bnxt *bp)
9413 {
9414         struct bnxt_link_info *link_info = &bp->link_info;
9415         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9416         u8 module_status;
9417
9418         if (bnxt_update_link(bp, true))
9419                 return;
9420
9421         module_status = link_info->module_status;
9422         switch (module_status) {
9423         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9424         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9425         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9426                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9427                             bp->pf.port_id);
9428                 if (bp->hwrm_spec_code >= 0x10201) {
9429                         netdev_warn(bp->dev, "Module part number %s\n",
9430                                     resp->phy_vendor_partnumber);
9431                 }
9432                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9433                         netdev_warn(bp->dev, "TX is disabled\n");
9434                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9435                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9436         }
9437 }
9438
9439 static void
9440 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9441 {
9442         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9443                 if (bp->hwrm_spec_code >= 0x10201)
9444                         req->auto_pause =
9445                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9446                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9447                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9448                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9449                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9450                 req->enables |=
9451                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9452         } else {
9453                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9454                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9455                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9456                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9457                 req->enables |=
9458                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9459                 if (bp->hwrm_spec_code >= 0x10201) {
9460                         req->auto_pause = req->force_pause;
9461                         req->enables |= cpu_to_le32(
9462                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9463                 }
9464         }
9465 }
9466
9467 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9468 {
9469         if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9470                 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9471                 if (bp->link_info.advertising) {
9472                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9473                         req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9474                 }
9475                 if (bp->link_info.advertising_pam4) {
9476                         req->enables |=
9477                                 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9478                         req->auto_link_pam4_speed_mask =
9479                                 cpu_to_le16(bp->link_info.advertising_pam4);
9480                 }
9481                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9482                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9483         } else {
9484                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9485                 if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9486                         req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9487                         req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9488                 } else {
9489                         req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9490                 }
9491         }
9492
9493         /* tell chimp that the setting takes effect immediately */
9494         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9495 }
9496
9497 int bnxt_hwrm_set_pause(struct bnxt *bp)
9498 {
9499         struct hwrm_port_phy_cfg_input req = {0};
9500         int rc;
9501
9502         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9503         bnxt_hwrm_set_pause_common(bp, &req);
9504
9505         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9506             bp->link_info.force_link_chng)
9507                 bnxt_hwrm_set_link_common(bp, &req);
9508
9509         mutex_lock(&bp->hwrm_cmd_lock);
9510         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9511         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9512                 /* since changing of pause setting doesn't trigger any link
9513                  * change event, the driver needs to update the current pause
9514                  * result upon successfully return of the phy_cfg command
9515                  */
9516                 bp->link_info.pause =
9517                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9518                 bp->link_info.auto_pause_setting = 0;
9519                 if (!bp->link_info.force_link_chng)
9520                         bnxt_report_link(bp);
9521         }
9522         bp->link_info.force_link_chng = false;
9523         mutex_unlock(&bp->hwrm_cmd_lock);
9524         return rc;
9525 }
9526
9527 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9528                               struct hwrm_port_phy_cfg_input *req)
9529 {
9530         struct ethtool_eee *eee = &bp->eee;
9531
9532         if (eee->eee_enabled) {
9533                 u16 eee_speeds;
9534                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9535
9536                 if (eee->tx_lpi_enabled)
9537                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9538                 else
9539                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9540
9541                 req->flags |= cpu_to_le32(flags);
9542                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9543                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9544                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9545         } else {
9546                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9547         }
9548 }
9549
9550 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9551 {
9552         struct hwrm_port_phy_cfg_input req = {0};
9553
9554         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9555         if (set_pause)
9556                 bnxt_hwrm_set_pause_common(bp, &req);
9557
9558         bnxt_hwrm_set_link_common(bp, &req);
9559
9560         if (set_eee)
9561                 bnxt_hwrm_set_eee(bp, &req);
9562         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9563 }
9564
9565 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9566 {
9567         struct hwrm_port_phy_cfg_input req = {0};
9568
9569         if (!BNXT_SINGLE_PF(bp))
9570                 return 0;
9571
9572         if (pci_num_vf(bp->pdev) &&
9573             !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
9574                 return 0;
9575
9576         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9577         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9578         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9579 }
9580
9581 static int bnxt_fw_init_one(struct bnxt *bp);
9582
9583 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
9584 {
9585 #ifdef CONFIG_TEE_BNXT_FW
9586         int rc = tee_bnxt_fw_load();
9587
9588         if (rc)
9589                 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
9590
9591         return rc;
9592 #else
9593         netdev_err(bp->dev, "OP-TEE not supported\n");
9594         return -ENODEV;
9595 #endif
9596 }
9597
9598 static int bnxt_try_recover_fw(struct bnxt *bp)
9599 {
9600         if (bp->fw_health && bp->fw_health->status_reliable) {
9601                 int retry = 0, rc;
9602                 u32 sts;
9603
9604                 mutex_lock(&bp->hwrm_cmd_lock);
9605                 do {
9606                         sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
9607                         rc = __bnxt_hwrm_ver_get(bp, true);
9608                         if (!BNXT_FW_IS_BOOTING(sts) &&
9609                             !BNXT_FW_IS_RECOVERING(sts))
9610                                 break;
9611                         retry++;
9612                 } while (rc == -EBUSY && retry < BNXT_FW_RETRY);
9613                 mutex_unlock(&bp->hwrm_cmd_lock);
9614
9615                 if (!BNXT_FW_IS_HEALTHY(sts)) {
9616                         netdev_err(bp->dev,
9617                                    "Firmware not responding, status: 0x%x\n",
9618                                    sts);
9619                         rc = -ENODEV;
9620                 }
9621                 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
9622                         netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
9623                         return bnxt_fw_reset_via_optee(bp);
9624                 }
9625                 return rc;
9626         }
9627
9628         return -ENODEV;
9629 }
9630
9631 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
9632 {
9633         struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
9634         struct hwrm_func_drv_if_change_input req = {0};
9635         bool fw_reset = !bp->irq_tbl;
9636         bool resc_reinit = false;
9637         int rc, retry = 0;
9638         u32 flags = 0;
9639
9640         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
9641                 return 0;
9642
9643         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
9644         if (up)
9645                 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
9646         mutex_lock(&bp->hwrm_cmd_lock);
9647         while (retry < BNXT_FW_IF_RETRY) {
9648                 rc = _hwrm_send_message(bp, &req, sizeof(req),
9649                                         HWRM_CMD_TIMEOUT);
9650                 if (rc != -EAGAIN)
9651                         break;
9652
9653                 msleep(50);
9654                 retry++;
9655         }
9656         if (!rc)
9657                 flags = le32_to_cpu(resp->flags);
9658         mutex_unlock(&bp->hwrm_cmd_lock);
9659
9660         if (rc == -EAGAIN)
9661                 return rc;
9662         if (rc && up) {
9663                 rc = bnxt_try_recover_fw(bp);
9664                 fw_reset = true;
9665         }
9666         if (rc)
9667                 return rc;
9668
9669         if (!up) {
9670                 bnxt_inv_fw_health_reg(bp);
9671                 return 0;
9672         }
9673
9674         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
9675                 resc_reinit = true;
9676         if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
9677                 fw_reset = true;
9678         else if (bp->fw_health && !bp->fw_health->status_reliable)
9679                 bnxt_try_map_fw_health_reg(bp);
9680
9681         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
9682                 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
9683                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9684                 return -ENODEV;
9685         }
9686         if (resc_reinit || fw_reset) {
9687                 if (fw_reset) {
9688                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9689                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9690                                 bnxt_ulp_stop(bp);
9691                         bnxt_free_ctx_mem(bp);
9692                         kfree(bp->ctx);
9693                         bp->ctx = NULL;
9694                         bnxt_dcb_free(bp);
9695                         rc = bnxt_fw_init_one(bp);
9696                         if (rc) {
9697                                 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9698                                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9699                                 return rc;
9700                         }
9701                         bnxt_clear_int_mode(bp);
9702                         rc = bnxt_init_int_mode(bp);
9703                         if (rc) {
9704                                 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9705                                 netdev_err(bp->dev, "init int mode failed\n");
9706                                 return rc;
9707                         }
9708                 }
9709                 if (BNXT_NEW_RM(bp)) {
9710                         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9711
9712                         rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9713                         if (rc)
9714                                 netdev_err(bp->dev, "resc_qcaps failed\n");
9715
9716                         hw_resc->resv_cp_rings = 0;
9717                         hw_resc->resv_stat_ctxs = 0;
9718                         hw_resc->resv_irqs = 0;
9719                         hw_resc->resv_tx_rings = 0;
9720                         hw_resc->resv_rx_rings = 0;
9721                         hw_resc->resv_hw_ring_grps = 0;
9722                         hw_resc->resv_vnics = 0;
9723                         if (!fw_reset) {
9724                                 bp->tx_nr_rings = 0;
9725                                 bp->rx_nr_rings = 0;
9726                         }
9727                 }
9728         }
9729         return rc;
9730 }
9731
9732 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
9733 {
9734         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9735         struct hwrm_port_led_qcaps_input req = {0};
9736         struct bnxt_pf_info *pf = &bp->pf;
9737         int rc;
9738
9739         bp->num_leds = 0;
9740         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
9741                 return 0;
9742
9743         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
9744         req.port_id = cpu_to_le16(pf->port_id);
9745         mutex_lock(&bp->hwrm_cmd_lock);
9746         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9747         if (rc) {
9748                 mutex_unlock(&bp->hwrm_cmd_lock);
9749                 return rc;
9750         }
9751         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
9752                 int i;
9753
9754                 bp->num_leds = resp->num_leds;
9755                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
9756                                                  bp->num_leds);
9757                 for (i = 0; i < bp->num_leds; i++) {
9758                         struct bnxt_led_info *led = &bp->leds[i];
9759                         __le16 caps = led->led_state_caps;
9760
9761                         if (!led->led_group_id ||
9762                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
9763                                 bp->num_leds = 0;
9764                                 break;
9765                         }
9766                 }
9767         }
9768         mutex_unlock(&bp->hwrm_cmd_lock);
9769         return 0;
9770 }
9771
9772 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
9773 {
9774         struct hwrm_wol_filter_alloc_input req = {0};
9775         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
9776         int rc;
9777
9778         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
9779         req.port_id = cpu_to_le16(bp->pf.port_id);
9780         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
9781         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
9782         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
9783         mutex_lock(&bp->hwrm_cmd_lock);
9784         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9785         if (!rc)
9786                 bp->wol_filter_id = resp->wol_filter_id;
9787         mutex_unlock(&bp->hwrm_cmd_lock);
9788         return rc;
9789 }
9790
9791 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
9792 {
9793         struct hwrm_wol_filter_free_input req = {0};
9794
9795         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
9796         req.port_id = cpu_to_le16(bp->pf.port_id);
9797         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
9798         req.wol_filter_id = bp->wol_filter_id;
9799         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9800 }
9801
9802 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
9803 {
9804         struct hwrm_wol_filter_qcfg_input req = {0};
9805         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9806         u16 next_handle = 0;
9807         int rc;
9808
9809         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
9810         req.port_id = cpu_to_le16(bp->pf.port_id);
9811         req.handle = cpu_to_le16(handle);
9812         mutex_lock(&bp->hwrm_cmd_lock);
9813         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9814         if (!rc) {
9815                 next_handle = le16_to_cpu(resp->next_handle);
9816                 if (next_handle != 0) {
9817                         if (resp->wol_type ==
9818                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
9819                                 bp->wol = 1;
9820                                 bp->wol_filter_id = resp->wol_filter_id;
9821                         }
9822                 }
9823         }
9824         mutex_unlock(&bp->hwrm_cmd_lock);
9825         return next_handle;
9826 }
9827
9828 static void bnxt_get_wol_settings(struct bnxt *bp)
9829 {
9830         u16 handle = 0;
9831
9832         bp->wol = 0;
9833         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
9834                 return;
9835
9836         do {
9837                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
9838         } while (handle && handle != 0xffff);
9839 }
9840
9841 #ifdef CONFIG_BNXT_HWMON
9842 static ssize_t bnxt_show_temp(struct device *dev,
9843                               struct device_attribute *devattr, char *buf)
9844 {
9845         struct hwrm_temp_monitor_query_input req = {0};
9846         struct hwrm_temp_monitor_query_output *resp;
9847         struct bnxt *bp = dev_get_drvdata(dev);
9848         u32 len = 0;
9849         int rc;
9850
9851         resp = bp->hwrm_cmd_resp_addr;
9852         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9853         mutex_lock(&bp->hwrm_cmd_lock);
9854         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9855         if (!rc)
9856                 len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
9857         mutex_unlock(&bp->hwrm_cmd_lock);
9858         if (rc)
9859                 return rc;
9860         return len;
9861 }
9862 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
9863
9864 static struct attribute *bnxt_attrs[] = {
9865         &sensor_dev_attr_temp1_input.dev_attr.attr,
9866         NULL
9867 };
9868 ATTRIBUTE_GROUPS(bnxt);
9869
9870 static void bnxt_hwmon_close(struct bnxt *bp)
9871 {
9872         if (bp->hwmon_dev) {
9873                 hwmon_device_unregister(bp->hwmon_dev);
9874                 bp->hwmon_dev = NULL;
9875         }
9876 }
9877
9878 static void bnxt_hwmon_open(struct bnxt *bp)
9879 {
9880         struct hwrm_temp_monitor_query_input req = {0};
9881         struct pci_dev *pdev = bp->pdev;
9882         int rc;
9883
9884         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9885         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9886         if (rc == -EACCES || rc == -EOPNOTSUPP) {
9887                 bnxt_hwmon_close(bp);
9888                 return;
9889         }
9890
9891         if (bp->hwmon_dev)
9892                 return;
9893
9894         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
9895                                                           DRV_MODULE_NAME, bp,
9896                                                           bnxt_groups);
9897         if (IS_ERR(bp->hwmon_dev)) {
9898                 bp->hwmon_dev = NULL;
9899                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
9900         }
9901 }
9902 #else
9903 static void bnxt_hwmon_close(struct bnxt *bp)
9904 {
9905 }
9906
9907 static void bnxt_hwmon_open(struct bnxt *bp)
9908 {
9909 }
9910 #endif
9911
9912 static bool bnxt_eee_config_ok(struct bnxt *bp)
9913 {
9914         struct ethtool_eee *eee = &bp->eee;
9915         struct bnxt_link_info *link_info = &bp->link_info;
9916
9917         if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
9918                 return true;
9919
9920         if (eee->eee_enabled) {
9921                 u32 advertising =
9922                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
9923
9924                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9925                         eee->eee_enabled = 0;
9926                         return false;
9927                 }
9928                 if (eee->advertised & ~advertising) {
9929                         eee->advertised = advertising & eee->supported;
9930                         return false;
9931                 }
9932         }
9933         return true;
9934 }
9935
9936 static int bnxt_update_phy_setting(struct bnxt *bp)
9937 {
9938         int rc;
9939         bool update_link = false;
9940         bool update_pause = false;
9941         bool update_eee = false;
9942         struct bnxt_link_info *link_info = &bp->link_info;
9943
9944         rc = bnxt_update_link(bp, true);
9945         if (rc) {
9946                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9947                            rc);
9948                 return rc;
9949         }
9950         if (!BNXT_SINGLE_PF(bp))
9951                 return 0;
9952
9953         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9954             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9955             link_info->req_flow_ctrl)
9956                 update_pause = true;
9957         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9958             link_info->force_pause_setting != link_info->req_flow_ctrl)
9959                 update_pause = true;
9960         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9961                 if (BNXT_AUTO_MODE(link_info->auto_mode))
9962                         update_link = true;
9963                 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
9964                     link_info->req_link_speed != link_info->force_link_speed)
9965                         update_link = true;
9966                 else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
9967                          link_info->req_link_speed != link_info->force_pam4_link_speed)
9968                         update_link = true;
9969                 if (link_info->req_duplex != link_info->duplex_setting)
9970                         update_link = true;
9971         } else {
9972                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9973                         update_link = true;
9974                 if (link_info->advertising != link_info->auto_link_speeds ||
9975                     link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
9976                         update_link = true;
9977         }
9978
9979         /* The last close may have shutdown the link, so need to call
9980          * PHY_CFG to bring it back up.
9981          */
9982         if (!bp->link_info.link_up)
9983                 update_link = true;
9984
9985         if (!bnxt_eee_config_ok(bp))
9986                 update_eee = true;
9987
9988         if (update_link)
9989                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9990         else if (update_pause)
9991                 rc = bnxt_hwrm_set_pause(bp);
9992         if (rc) {
9993                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9994                            rc);
9995                 return rc;
9996         }
9997
9998         return rc;
9999 }
10000
10001 /* Common routine to pre-map certain register block to different GRC window.
10002  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
10003  * in PF and 3 windows in VF that can be customized to map in different
10004  * register blocks.
10005  */
10006 static void bnxt_preset_reg_win(struct bnxt *bp)
10007 {
10008         if (BNXT_PF(bp)) {
10009                 /* CAG registers map to GRC window #4 */
10010                 writel(BNXT_CAG_REG_BASE,
10011                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
10012         }
10013 }
10014
10015 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
10016
10017 static int bnxt_reinit_after_abort(struct bnxt *bp)
10018 {
10019         int rc;
10020
10021         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10022                 return -EBUSY;
10023
10024         if (bp->dev->reg_state == NETREG_UNREGISTERED)
10025                 return -ENODEV;
10026
10027         rc = bnxt_fw_init_one(bp);
10028         if (!rc) {
10029                 bnxt_clear_int_mode(bp);
10030                 rc = bnxt_init_int_mode(bp);
10031                 if (!rc) {
10032                         clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10033                         set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10034                 }
10035         }
10036         return rc;
10037 }
10038
10039 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10040 {
10041         int rc = 0;
10042
10043         bnxt_preset_reg_win(bp);
10044         netif_carrier_off(bp->dev);
10045         if (irq_re_init) {
10046                 /* Reserve rings now if none were reserved at driver probe. */
10047                 rc = bnxt_init_dflt_ring_mode(bp);
10048                 if (rc) {
10049                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
10050                         return rc;
10051                 }
10052         }
10053         rc = bnxt_reserve_rings(bp, irq_re_init);
10054         if (rc)
10055                 return rc;
10056         if ((bp->flags & BNXT_FLAG_RFS) &&
10057             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
10058                 /* disable RFS if falling back to INTA */
10059                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
10060                 bp->flags &= ~BNXT_FLAG_RFS;
10061         }
10062
10063         rc = bnxt_alloc_mem(bp, irq_re_init);
10064         if (rc) {
10065                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10066                 goto open_err_free_mem;
10067         }
10068
10069         if (irq_re_init) {
10070                 bnxt_init_napi(bp);
10071                 rc = bnxt_request_irq(bp);
10072                 if (rc) {
10073                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
10074                         goto open_err_irq;
10075                 }
10076         }
10077
10078         rc = bnxt_init_nic(bp, irq_re_init);
10079         if (rc) {
10080                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10081                 goto open_err_irq;
10082         }
10083
10084         bnxt_enable_napi(bp);
10085         bnxt_debug_dev_init(bp);
10086
10087         if (link_re_init) {
10088                 mutex_lock(&bp->link_lock);
10089                 rc = bnxt_update_phy_setting(bp);
10090                 mutex_unlock(&bp->link_lock);
10091                 if (rc) {
10092                         netdev_warn(bp->dev, "failed to update phy settings\n");
10093                         if (BNXT_SINGLE_PF(bp)) {
10094                                 bp->link_info.phy_retry = true;
10095                                 bp->link_info.phy_retry_expires =
10096                                         jiffies + 5 * HZ;
10097                         }
10098                 }
10099         }
10100
10101         if (irq_re_init)
10102                 udp_tunnel_nic_reset_ntf(bp->dev);
10103
10104         set_bit(BNXT_STATE_OPEN, &bp->state);
10105         bnxt_enable_int(bp);
10106         /* Enable TX queues */
10107         bnxt_tx_enable(bp);
10108         mod_timer(&bp->timer, jiffies + bp->current_interval);
10109         /* Poll link status and check for SFP+ module status */
10110         bnxt_get_port_module_status(bp);
10111
10112         /* VF-reps may need to be re-opened after the PF is re-opened */
10113         if (BNXT_PF(bp))
10114                 bnxt_vf_reps_open(bp);
10115         return 0;
10116
10117 open_err_irq:
10118         bnxt_del_napi(bp);
10119
10120 open_err_free_mem:
10121         bnxt_free_skbs(bp);
10122         bnxt_free_irq(bp);
10123         bnxt_free_mem(bp, true);
10124         return rc;
10125 }
10126
10127 /* rtnl_lock held */
10128 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10129 {
10130         int rc = 0;
10131
10132         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
10133                 rc = -EIO;
10134         if (!rc)
10135                 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
10136         if (rc) {
10137                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
10138                 dev_close(bp->dev);
10139         }
10140         return rc;
10141 }
10142
10143 /* rtnl_lock held, open the NIC half way by allocating all resources, but
10144  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
10145  * self tests.
10146  */
10147 int bnxt_half_open_nic(struct bnxt *bp)
10148 {
10149         int rc = 0;
10150
10151         rc = bnxt_alloc_mem(bp, false);
10152         if (rc) {
10153                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10154                 goto half_open_err;
10155         }
10156         rc = bnxt_init_nic(bp, false);
10157         if (rc) {
10158                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10159                 goto half_open_err;
10160         }
10161         return 0;
10162
10163 half_open_err:
10164         bnxt_free_skbs(bp);
10165         bnxt_free_mem(bp, false);
10166         dev_close(bp->dev);
10167         return rc;
10168 }
10169
10170 /* rtnl_lock held, this call can only be made after a previous successful
10171  * call to bnxt_half_open_nic().
10172  */
10173 void bnxt_half_close_nic(struct bnxt *bp)
10174 {
10175         bnxt_hwrm_resource_free(bp, false, false);
10176         bnxt_free_skbs(bp);
10177         bnxt_free_mem(bp, false);
10178 }
10179
10180 static void bnxt_reenable_sriov(struct bnxt *bp)
10181 {
10182         if (BNXT_PF(bp)) {
10183                 struct bnxt_pf_info *pf = &bp->pf;
10184                 int n = pf->active_vfs;
10185
10186                 if (n)
10187                         bnxt_cfg_hw_sriov(bp, &n, true);
10188         }
10189 }
10190
10191 static int bnxt_open(struct net_device *dev)
10192 {
10193         struct bnxt *bp = netdev_priv(dev);
10194         int rc;
10195
10196         if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10197                 rc = bnxt_reinit_after_abort(bp);
10198                 if (rc) {
10199                         if (rc == -EBUSY)
10200                                 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
10201                         else
10202                                 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
10203                         return -ENODEV;
10204                 }
10205         }
10206
10207         rc = bnxt_hwrm_if_change(bp, true);
10208         if (rc)
10209                 return rc;
10210         rc = __bnxt_open_nic(bp, true, true);
10211         if (rc) {
10212                 bnxt_hwrm_if_change(bp, false);
10213         } else {
10214                 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
10215                         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10216                                 bnxt_ulp_start(bp, 0);
10217                                 bnxt_reenable_sriov(bp);
10218                         }
10219                 }
10220                 bnxt_hwmon_open(bp);
10221         }
10222
10223         return rc;
10224 }
10225
10226 static bool bnxt_drv_busy(struct bnxt *bp)
10227 {
10228         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
10229                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
10230 }
10231
10232 static void bnxt_get_ring_stats(struct bnxt *bp,
10233                                 struct rtnl_link_stats64 *stats);
10234
10235 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
10236                              bool link_re_init)
10237 {
10238         /* Close the VF-reps before closing PF */
10239         if (BNXT_PF(bp))
10240                 bnxt_vf_reps_close(bp);
10241
10242         /* Change device state to avoid TX queue wake up's */
10243         bnxt_tx_disable(bp);
10244
10245         clear_bit(BNXT_STATE_OPEN, &bp->state);
10246         smp_mb__after_atomic();
10247         while (bnxt_drv_busy(bp))
10248                 msleep(20);
10249
10250         /* Flush rings and and disable interrupts */
10251         bnxt_shutdown_nic(bp, irq_re_init);
10252
10253         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
10254
10255         bnxt_debug_dev_exit(bp);
10256         bnxt_disable_napi(bp);
10257         del_timer_sync(&bp->timer);
10258         bnxt_free_skbs(bp);
10259
10260         /* Save ring stats before shutdown */
10261         if (bp->bnapi && irq_re_init)
10262                 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
10263         if (irq_re_init) {
10264                 bnxt_free_irq(bp);
10265                 bnxt_del_napi(bp);
10266         }
10267         bnxt_free_mem(bp, irq_re_init);
10268 }
10269
10270 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10271 {
10272         int rc = 0;
10273
10274         if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10275                 /* If we get here, it means firmware reset is in progress
10276                  * while we are trying to close.  We can safely proceed with
10277                  * the close because we are holding rtnl_lock().  Some firmware
10278                  * messages may fail as we proceed to close.  We set the
10279                  * ABORT_ERR flag here so that the FW reset thread will later
10280                  * abort when it gets the rtnl_lock() and sees the flag.
10281                  */
10282                 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
10283                 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10284         }
10285
10286 #ifdef CONFIG_BNXT_SRIOV
10287         if (bp->sriov_cfg) {
10288                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
10289                                                       !bp->sriov_cfg,
10290                                                       BNXT_SRIOV_CFG_WAIT_TMO);
10291                 if (rc)
10292                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
10293         }
10294 #endif
10295         __bnxt_close_nic(bp, irq_re_init, link_re_init);
10296         return rc;
10297 }
10298
10299 static int bnxt_close(struct net_device *dev)
10300 {
10301         struct bnxt *bp = netdev_priv(dev);
10302
10303         bnxt_hwmon_close(bp);
10304         bnxt_close_nic(bp, true, true);
10305         bnxt_hwrm_shutdown_link(bp);
10306         bnxt_hwrm_if_change(bp, false);
10307         return 0;
10308 }
10309
10310 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
10311                                    u16 *val)
10312 {
10313         struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
10314         struct hwrm_port_phy_mdio_read_input req = {0};
10315         int rc;
10316
10317         if (bp->hwrm_spec_code < 0x10a00)
10318                 return -EOPNOTSUPP;
10319
10320         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
10321         req.port_id = cpu_to_le16(bp->pf.port_id);
10322         req.phy_addr = phy_addr;
10323         req.reg_addr = cpu_to_le16(reg & 0x1f);
10324         if (mdio_phy_id_is_c45(phy_addr)) {
10325                 req.cl45_mdio = 1;
10326                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
10327                 req.dev_addr = mdio_phy_id_devad(phy_addr);
10328                 req.reg_addr = cpu_to_le16(reg);
10329         }
10330
10331         mutex_lock(&bp->hwrm_cmd_lock);
10332         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10333         if (!rc)
10334                 *val = le16_to_cpu(resp->reg_data);
10335         mutex_unlock(&bp->hwrm_cmd_lock);
10336         return rc;
10337 }
10338
10339 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
10340                                     u16 val)
10341 {
10342         struct hwrm_port_phy_mdio_write_input req = {0};
10343
10344         if (bp->hwrm_spec_code < 0x10a00)
10345                 return -EOPNOTSUPP;
10346
10347         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
10348         req.port_id = cpu_to_le16(bp->pf.port_id);
10349         req.phy_addr = phy_addr;
10350         req.reg_addr = cpu_to_le16(reg & 0x1f);
10351         if (mdio_phy_id_is_c45(phy_addr)) {
10352                 req.cl45_mdio = 1;
10353                 req.phy_addr = mdio_phy_id_prtad(phy_addr);
10354                 req.dev_addr = mdio_phy_id_devad(phy_addr);
10355                 req.reg_addr = cpu_to_le16(reg);
10356         }
10357         req.reg_data = cpu_to_le16(val);
10358
10359         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10360 }
10361
10362 /* rtnl_lock held */
10363 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10364 {
10365         struct mii_ioctl_data *mdio = if_mii(ifr);
10366         struct bnxt *bp = netdev_priv(dev);
10367         int rc;
10368
10369         switch (cmd) {
10370         case SIOCGMIIPHY:
10371                 mdio->phy_id = bp->link_info.phy_addr;
10372
10373                 fallthrough;
10374         case SIOCGMIIREG: {
10375                 u16 mii_regval = 0;
10376
10377                 if (!netif_running(dev))
10378                         return -EAGAIN;
10379
10380                 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10381                                              &mii_regval);
10382                 mdio->val_out = mii_regval;
10383                 return rc;
10384         }
10385
10386         case SIOCSMIIREG:
10387                 if (!netif_running(dev))
10388                         return -EAGAIN;
10389
10390                 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10391                                                 mdio->val_in);
10392
10393         default:
10394                 /* do nothing */
10395                 break;
10396         }
10397         return -EOPNOTSUPP;
10398 }
10399
10400 static void bnxt_get_ring_stats(struct bnxt *bp,
10401                                 struct rtnl_link_stats64 *stats)
10402 {
10403         int i;
10404
10405         for (i = 0; i < bp->cp_nr_rings; i++) {
10406                 struct bnxt_napi *bnapi = bp->bnapi[i];
10407                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10408                 u64 *sw = cpr->stats.sw_stats;
10409
10410                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10411                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10412                 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10413
10414                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10415                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10416                 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10417
10418                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10419                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10420                 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10421
10422                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10423                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10424                 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10425
10426                 stats->rx_missed_errors +=
10427                         BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10428
10429                 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10430
10431                 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10432         }
10433 }
10434
10435 static void bnxt_add_prev_stats(struct bnxt *bp,
10436                                 struct rtnl_link_stats64 *stats)
10437 {
10438         struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10439
10440         stats->rx_packets += prev_stats->rx_packets;
10441         stats->tx_packets += prev_stats->tx_packets;
10442         stats->rx_bytes += prev_stats->rx_bytes;
10443         stats->tx_bytes += prev_stats->tx_bytes;
10444         stats->rx_missed_errors += prev_stats->rx_missed_errors;
10445         stats->multicast += prev_stats->multicast;
10446         stats->tx_dropped += prev_stats->tx_dropped;
10447 }
10448
10449 static void
10450 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10451 {
10452         struct bnxt *bp = netdev_priv(dev);
10453
10454         set_bit(BNXT_STATE_READ_STATS, &bp->state);
10455         /* Make sure bnxt_close_nic() sees that we are reading stats before
10456          * we check the BNXT_STATE_OPEN flag.
10457          */
10458         smp_mb__after_atomic();
10459         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10460                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10461                 *stats = bp->net_stats_prev;
10462                 return;
10463         }
10464
10465         bnxt_get_ring_stats(bp, stats);
10466         bnxt_add_prev_stats(bp, stats);
10467
10468         if (bp->flags & BNXT_FLAG_PORT_STATS) {
10469                 u64 *rx = bp->port_stats.sw_stats;
10470                 u64 *tx = bp->port_stats.sw_stats +
10471                           BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10472
10473                 stats->rx_crc_errors =
10474                         BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10475                 stats->rx_frame_errors =
10476                         BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10477                 stats->rx_length_errors =
10478                         BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10479                         BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10480                         BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10481                 stats->rx_errors =
10482                         BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10483                         BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10484                 stats->collisions =
10485                         BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10486                 stats->tx_fifo_errors =
10487                         BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10488                 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10489         }
10490         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10491 }
10492
10493 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10494 {
10495         struct net_device *dev = bp->dev;
10496         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10497         struct netdev_hw_addr *ha;
10498         u8 *haddr;
10499         int mc_count = 0;
10500         bool update = false;
10501         int off = 0;
10502
10503         netdev_for_each_mc_addr(ha, dev) {
10504                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
10505                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10506                         vnic->mc_list_count = 0;
10507                         return false;
10508                 }
10509                 haddr = ha->addr;
10510                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
10511                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
10512                         update = true;
10513                 }
10514                 off += ETH_ALEN;
10515                 mc_count++;
10516         }
10517         if (mc_count)
10518                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
10519
10520         if (mc_count != vnic->mc_list_count) {
10521                 vnic->mc_list_count = mc_count;
10522                 update = true;
10523         }
10524         return update;
10525 }
10526
10527 static bool bnxt_uc_list_updated(struct bnxt *bp)
10528 {
10529         struct net_device *dev = bp->dev;
10530         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10531         struct netdev_hw_addr *ha;
10532         int off = 0;
10533
10534         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
10535                 return true;
10536
10537         netdev_for_each_uc_addr(ha, dev) {
10538                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
10539                         return true;
10540
10541                 off += ETH_ALEN;
10542         }
10543         return false;
10544 }
10545
10546 static void bnxt_set_rx_mode(struct net_device *dev)
10547 {
10548         struct bnxt *bp = netdev_priv(dev);
10549         struct bnxt_vnic_info *vnic;
10550         bool mc_update = false;
10551         bool uc_update;
10552         u32 mask;
10553
10554         if (!test_bit(BNXT_STATE_OPEN, &bp->state))
10555                 return;
10556
10557         vnic = &bp->vnic_info[0];
10558         mask = vnic->rx_mask;
10559         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
10560                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
10561                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
10562                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
10563
10564         if (dev->flags & IFF_PROMISC)
10565                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10566
10567         uc_update = bnxt_uc_list_updated(bp);
10568
10569         if (dev->flags & IFF_BROADCAST)
10570                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10571         if (dev->flags & IFF_ALLMULTI) {
10572                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10573                 vnic->mc_list_count = 0;
10574         } else {
10575                 mc_update = bnxt_mc_list_updated(bp, &mask);
10576         }
10577
10578         if (mask != vnic->rx_mask || uc_update || mc_update) {
10579                 vnic->rx_mask = mask;
10580
10581                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
10582                 bnxt_queue_sp_work(bp);
10583         }
10584 }
10585
10586 static int bnxt_cfg_rx_mode(struct bnxt *bp)
10587 {
10588         struct net_device *dev = bp->dev;
10589         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10590         struct netdev_hw_addr *ha;
10591         int i, off = 0, rc;
10592         bool uc_update;
10593
10594         netif_addr_lock_bh(dev);
10595         uc_update = bnxt_uc_list_updated(bp);
10596         netif_addr_unlock_bh(dev);
10597
10598         if (!uc_update)
10599                 goto skip_uc;
10600
10601         mutex_lock(&bp->hwrm_cmd_lock);
10602         for (i = 1; i < vnic->uc_filter_count; i++) {
10603                 struct hwrm_cfa_l2_filter_free_input req = {0};
10604
10605                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
10606                                        -1);
10607
10608                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
10609
10610                 rc = _hwrm_send_message(bp, &req, sizeof(req),
10611                                         HWRM_CMD_TIMEOUT);
10612         }
10613         mutex_unlock(&bp->hwrm_cmd_lock);
10614
10615         vnic->uc_filter_count = 1;
10616
10617         netif_addr_lock_bh(dev);
10618         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
10619                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10620         } else {
10621                 netdev_for_each_uc_addr(ha, dev) {
10622                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
10623                         off += ETH_ALEN;
10624                         vnic->uc_filter_count++;
10625                 }
10626         }
10627         netif_addr_unlock_bh(dev);
10628
10629         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
10630                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
10631                 if (rc) {
10632                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
10633                                    rc);
10634                         vnic->uc_filter_count = i;
10635                         return rc;
10636                 }
10637         }
10638
10639 skip_uc:
10640         if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
10641             !bnxt_promisc_ok(bp))
10642                 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10643         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10644         if (rc && vnic->mc_list_count) {
10645                 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
10646                             rc);
10647                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10648                 vnic->mc_list_count = 0;
10649                 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10650         }
10651         if (rc)
10652                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
10653                            rc);
10654
10655         return rc;
10656 }
10657
10658 static bool bnxt_can_reserve_rings(struct bnxt *bp)
10659 {
10660 #ifdef CONFIG_BNXT_SRIOV
10661         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
10662                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10663
10664                 /* No minimum rings were provisioned by the PF.  Don't
10665                  * reserve rings by default when device is down.
10666                  */
10667                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
10668                         return true;
10669
10670                 if (!netif_running(bp->dev))
10671                         return false;
10672         }
10673 #endif
10674         return true;
10675 }
10676
10677 /* If the chip and firmware supports RFS */
10678 static bool bnxt_rfs_supported(struct bnxt *bp)
10679 {
10680         if (bp->flags & BNXT_FLAG_CHIP_P5) {
10681                 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
10682                         return true;
10683                 return false;
10684         }
10685         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
10686                 return true;
10687         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10688                 return true;
10689         return false;
10690 }
10691
10692 /* If runtime conditions support RFS */
10693 static bool bnxt_rfs_capable(struct bnxt *bp)
10694 {
10695 #ifdef CONFIG_RFS_ACCEL
10696         int vnics, max_vnics, max_rss_ctxs;
10697
10698         if (bp->flags & BNXT_FLAG_CHIP_P5)
10699                 return bnxt_rfs_supported(bp);
10700         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
10701                 return false;
10702
10703         vnics = 1 + bp->rx_nr_rings;
10704         max_vnics = bnxt_get_max_func_vnics(bp);
10705         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
10706
10707         /* RSS contexts not a limiting factor */
10708         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10709                 max_rss_ctxs = max_vnics;
10710         if (vnics > max_vnics || vnics > max_rss_ctxs) {
10711                 if (bp->rx_nr_rings > 1)
10712                         netdev_warn(bp->dev,
10713                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
10714                                     min(max_rss_ctxs - 1, max_vnics - 1));
10715                 return false;
10716         }
10717
10718         if (!BNXT_NEW_RM(bp))
10719                 return true;
10720
10721         if (vnics == bp->hw_resc.resv_vnics)
10722                 return true;
10723
10724         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
10725         if (vnics <= bp->hw_resc.resv_vnics)
10726                 return true;
10727
10728         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
10729         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
10730         return false;
10731 #else
10732         return false;
10733 #endif
10734 }
10735
10736 static netdev_features_t bnxt_fix_features(struct net_device *dev,
10737                                            netdev_features_t features)
10738 {
10739         struct bnxt *bp = netdev_priv(dev);
10740         netdev_features_t vlan_features;
10741
10742         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
10743                 features &= ~NETIF_F_NTUPLE;
10744
10745         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10746                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10747
10748         if (!(features & NETIF_F_GRO))
10749                 features &= ~NETIF_F_GRO_HW;
10750
10751         if (features & NETIF_F_GRO_HW)
10752                 features &= ~NETIF_F_LRO;
10753
10754         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
10755          * turned on or off together.
10756          */
10757         vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
10758         if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
10759                 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10760                         features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10761                 else if (vlan_features)
10762                         features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
10763         }
10764 #ifdef CONFIG_BNXT_SRIOV
10765         if (BNXT_VF(bp) && bp->vf.vlan)
10766                 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10767 #endif
10768         return features;
10769 }
10770
10771 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
10772 {
10773         struct bnxt *bp = netdev_priv(dev);
10774         u32 flags = bp->flags;
10775         u32 changes;
10776         int rc = 0;
10777         bool re_init = false;
10778         bool update_tpa = false;
10779
10780         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
10781         if (features & NETIF_F_GRO_HW)
10782                 flags |= BNXT_FLAG_GRO;
10783         else if (features & NETIF_F_LRO)
10784                 flags |= BNXT_FLAG_LRO;
10785
10786         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10787                 flags &= ~BNXT_FLAG_TPA;
10788
10789         if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10790                 flags |= BNXT_FLAG_STRIP_VLAN;
10791
10792         if (features & NETIF_F_NTUPLE)
10793                 flags |= BNXT_FLAG_RFS;
10794
10795         changes = flags ^ bp->flags;
10796         if (changes & BNXT_FLAG_TPA) {
10797                 update_tpa = true;
10798                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
10799                     (flags & BNXT_FLAG_TPA) == 0 ||
10800                     (bp->flags & BNXT_FLAG_CHIP_P5))
10801                         re_init = true;
10802         }
10803
10804         if (changes & ~BNXT_FLAG_TPA)
10805                 re_init = true;
10806
10807         if (flags != bp->flags) {
10808                 u32 old_flags = bp->flags;
10809
10810                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10811                         bp->flags = flags;
10812                         if (update_tpa)
10813                                 bnxt_set_ring_params(bp);
10814                         return rc;
10815                 }
10816
10817                 if (re_init) {
10818                         bnxt_close_nic(bp, false, false);
10819                         bp->flags = flags;
10820                         if (update_tpa)
10821                                 bnxt_set_ring_params(bp);
10822
10823                         return bnxt_open_nic(bp, false, false);
10824                 }
10825                 if (update_tpa) {
10826                         bp->flags = flags;
10827                         rc = bnxt_set_tpa(bp,
10828                                           (flags & BNXT_FLAG_TPA) ?
10829                                           true : false);
10830                         if (rc)
10831                                 bp->flags = old_flags;
10832                 }
10833         }
10834         return rc;
10835 }
10836
10837 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
10838                               u8 **nextp)
10839 {
10840         struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
10841         int hdr_count = 0;
10842         u8 *nexthdr;
10843         int start;
10844
10845         /* Check that there are at most 2 IPv6 extension headers, no
10846          * fragment header, and each is <= 64 bytes.
10847          */
10848         start = nw_off + sizeof(*ip6h);
10849         nexthdr = &ip6h->nexthdr;
10850         while (ipv6_ext_hdr(*nexthdr)) {
10851                 struct ipv6_opt_hdr *hp;
10852                 int hdrlen;
10853
10854                 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
10855                     *nexthdr == NEXTHDR_FRAGMENT)
10856                         return false;
10857                 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
10858                                           skb_headlen(skb), NULL);
10859                 if (!hp)
10860                         return false;
10861                 if (*nexthdr == NEXTHDR_AUTH)
10862                         hdrlen = ipv6_authlen(hp);
10863                 else
10864                         hdrlen = ipv6_optlen(hp);
10865
10866                 if (hdrlen > 64)
10867                         return false;
10868                 nexthdr = &hp->nexthdr;
10869                 start += hdrlen;
10870                 hdr_count++;
10871         }
10872         if (nextp) {
10873                 /* Caller will check inner protocol */
10874                 if (skb->encapsulation) {
10875                         *nextp = nexthdr;
10876                         return true;
10877                 }
10878                 *nextp = NULL;
10879         }
10880         /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
10881         return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
10882 }
10883
10884 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
10885 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
10886 {
10887         struct udphdr *uh = udp_hdr(skb);
10888         __be16 udp_port = uh->dest;
10889
10890         if (udp_port != bp->vxlan_port && udp_port != bp->nge_port)
10891                 return false;
10892         if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) {
10893                 struct ethhdr *eh = inner_eth_hdr(skb);
10894
10895                 switch (eh->h_proto) {
10896                 case htons(ETH_P_IP):
10897                         return true;
10898                 case htons(ETH_P_IPV6):
10899                         return bnxt_exthdr_check(bp, skb,
10900                                                  skb_inner_network_offset(skb),
10901                                                  NULL);
10902                 }
10903         }
10904         return false;
10905 }
10906
10907 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
10908 {
10909         switch (l4_proto) {
10910         case IPPROTO_UDP:
10911                 return bnxt_udp_tunl_check(bp, skb);
10912         case IPPROTO_IPIP:
10913                 return true;
10914         case IPPROTO_GRE: {
10915                 switch (skb->inner_protocol) {
10916                 default:
10917                         return false;
10918                 case htons(ETH_P_IP):
10919                         return true;
10920                 case htons(ETH_P_IPV6):
10921                         fallthrough;
10922                 }
10923         }
10924         case IPPROTO_IPV6:
10925                 /* Check ext headers of inner ipv6 */
10926                 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
10927                                          NULL);
10928         }
10929         return false;
10930 }
10931
10932 static netdev_features_t bnxt_features_check(struct sk_buff *skb,
10933                                              struct net_device *dev,
10934                                              netdev_features_t features)
10935 {
10936         struct bnxt *bp = netdev_priv(dev);
10937         u8 *l4_proto;
10938
10939         features = vlan_features_check(skb, features);
10940         switch (vlan_get_protocol(skb)) {
10941         case htons(ETH_P_IP):
10942                 if (!skb->encapsulation)
10943                         return features;
10944                 l4_proto = &ip_hdr(skb)->protocol;
10945                 if (bnxt_tunl_check(bp, skb, *l4_proto))
10946                         return features;
10947                 break;
10948         case htons(ETH_P_IPV6):
10949                 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
10950                                        &l4_proto))
10951                         break;
10952                 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
10953                         return features;
10954                 break;
10955         }
10956         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
10957 }
10958
10959 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
10960                          u32 *reg_buf)
10961 {
10962         struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr;
10963         struct hwrm_dbg_read_direct_input req = {0};
10964         __le32 *dbg_reg_buf;
10965         dma_addr_t mapping;
10966         int rc, i;
10967
10968         dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4,
10969                                          &mapping, GFP_KERNEL);
10970         if (!dbg_reg_buf)
10971                 return -ENOMEM;
10972         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1);
10973         req.host_dest_addr = cpu_to_le64(mapping);
10974         req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
10975         req.read_len32 = cpu_to_le32(num_words);
10976         mutex_lock(&bp->hwrm_cmd_lock);
10977         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10978         if (rc || resp->error_code) {
10979                 rc = -EIO;
10980                 goto dbg_rd_reg_exit;
10981         }
10982         for (i = 0; i < num_words; i++)
10983                 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
10984
10985 dbg_rd_reg_exit:
10986         mutex_unlock(&bp->hwrm_cmd_lock);
10987         dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping);
10988         return rc;
10989 }
10990
10991 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
10992                                        u32 ring_id, u32 *prod, u32 *cons)
10993 {
10994         struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
10995         struct hwrm_dbg_ring_info_get_input req = {0};
10996         int rc;
10997
10998         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
10999         req.ring_type = ring_type;
11000         req.fw_ring_id = cpu_to_le32(ring_id);
11001         mutex_lock(&bp->hwrm_cmd_lock);
11002         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11003         if (!rc) {
11004                 *prod = le32_to_cpu(resp->producer_index);
11005                 *cons = le32_to_cpu(resp->consumer_index);
11006         }
11007         mutex_unlock(&bp->hwrm_cmd_lock);
11008         return rc;
11009 }
11010
11011 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
11012 {
11013         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
11014         int i = bnapi->index;
11015
11016         if (!txr)
11017                 return;
11018
11019         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
11020                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
11021                     txr->tx_cons);
11022 }
11023
11024 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
11025 {
11026         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
11027         int i = bnapi->index;
11028
11029         if (!rxr)
11030                 return;
11031
11032         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
11033                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
11034                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
11035                     rxr->rx_sw_agg_prod);
11036 }
11037
11038 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
11039 {
11040         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
11041         int i = bnapi->index;
11042
11043         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
11044                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
11045 }
11046
11047 static void bnxt_dbg_dump_states(struct bnxt *bp)
11048 {
11049         int i;
11050         struct bnxt_napi *bnapi;
11051
11052         for (i = 0; i < bp->cp_nr_rings; i++) {
11053                 bnapi = bp->bnapi[i];
11054                 if (netif_msg_drv(bp)) {
11055                         bnxt_dump_tx_sw_state(bnapi);
11056                         bnxt_dump_rx_sw_state(bnapi);
11057                         bnxt_dump_cp_sw_state(bnapi);
11058                 }
11059         }
11060 }
11061
11062 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
11063 {
11064         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
11065         struct hwrm_ring_reset_input req = {0};
11066         struct bnxt_napi *bnapi = rxr->bnapi;
11067         struct bnxt_cp_ring_info *cpr;
11068         u16 cp_ring_id;
11069
11070         cpr = &bnapi->cp_ring;
11071         cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
11072         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_RESET, cp_ring_id, -1);
11073         req.ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
11074         req.ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
11075         return hwrm_send_message_silent(bp, &req, sizeof(req),
11076                                         HWRM_CMD_TIMEOUT);
11077 }
11078
11079 static void bnxt_reset_task(struct bnxt *bp, bool silent)
11080 {
11081         if (!silent)
11082                 bnxt_dbg_dump_states(bp);
11083         if (netif_running(bp->dev)) {
11084                 int rc;
11085
11086                 if (silent) {
11087                         bnxt_close_nic(bp, false, false);
11088                         bnxt_open_nic(bp, false, false);
11089                 } else {
11090                         bnxt_ulp_stop(bp);
11091                         bnxt_close_nic(bp, true, false);
11092                         rc = bnxt_open_nic(bp, true, false);
11093                         bnxt_ulp_start(bp, rc);
11094                 }
11095         }
11096 }
11097
11098 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
11099 {
11100         struct bnxt *bp = netdev_priv(dev);
11101
11102         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
11103         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
11104         bnxt_queue_sp_work(bp);
11105 }
11106
11107 static void bnxt_fw_health_check(struct bnxt *bp)
11108 {
11109         struct bnxt_fw_health *fw_health = bp->fw_health;
11110         u32 val;
11111
11112         if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11113                 return;
11114
11115         if (fw_health->tmr_counter) {
11116                 fw_health->tmr_counter--;
11117                 return;
11118         }
11119
11120         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11121         if (val == fw_health->last_fw_heartbeat)
11122                 goto fw_reset;
11123
11124         fw_health->last_fw_heartbeat = val;
11125
11126         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11127         if (val != fw_health->last_fw_reset_cnt)
11128                 goto fw_reset;
11129
11130         fw_health->tmr_counter = fw_health->tmr_multiplier;
11131         return;
11132
11133 fw_reset:
11134         set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
11135         bnxt_queue_sp_work(bp);
11136 }
11137
11138 static void bnxt_timer(struct timer_list *t)
11139 {
11140         struct bnxt *bp = from_timer(bp, t, timer);
11141         struct net_device *dev = bp->dev;
11142
11143         if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
11144                 return;
11145
11146         if (atomic_read(&bp->intr_sem) != 0)
11147                 goto bnxt_restart_timer;
11148
11149         if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
11150                 bnxt_fw_health_check(bp);
11151
11152         if (bp->link_info.link_up && bp->stats_coal_ticks) {
11153                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
11154                 bnxt_queue_sp_work(bp);
11155         }
11156
11157         if (bnxt_tc_flower_enabled(bp)) {
11158                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
11159                 bnxt_queue_sp_work(bp);
11160         }
11161
11162 #ifdef CONFIG_RFS_ACCEL
11163         if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
11164                 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11165                 bnxt_queue_sp_work(bp);
11166         }
11167 #endif /*CONFIG_RFS_ACCEL*/
11168
11169         if (bp->link_info.phy_retry) {
11170                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
11171                         bp->link_info.phy_retry = false;
11172                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
11173                 } else {
11174                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
11175                         bnxt_queue_sp_work(bp);
11176                 }
11177         }
11178
11179         if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
11180             netif_carrier_ok(dev)) {
11181                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
11182                 bnxt_queue_sp_work(bp);
11183         }
11184 bnxt_restart_timer:
11185         mod_timer(&bp->timer, jiffies + bp->current_interval);
11186 }
11187
11188 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
11189 {
11190         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
11191          * set.  If the device is being closed, bnxt_close() may be holding
11192          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
11193          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
11194          */
11195         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11196         rtnl_lock();
11197 }
11198
11199 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
11200 {
11201         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11202         rtnl_unlock();
11203 }
11204
11205 /* Only called from bnxt_sp_task() */
11206 static void bnxt_reset(struct bnxt *bp, bool silent)
11207 {
11208         bnxt_rtnl_lock_sp(bp);
11209         if (test_bit(BNXT_STATE_OPEN, &bp->state))
11210                 bnxt_reset_task(bp, silent);
11211         bnxt_rtnl_unlock_sp(bp);
11212 }
11213
11214 /* Only called from bnxt_sp_task() */
11215 static void bnxt_rx_ring_reset(struct bnxt *bp)
11216 {
11217         int i;
11218
11219         bnxt_rtnl_lock_sp(bp);
11220         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11221                 bnxt_rtnl_unlock_sp(bp);
11222                 return;
11223         }
11224         /* Disable and flush TPA before resetting the RX ring */
11225         if (bp->flags & BNXT_FLAG_TPA)
11226                 bnxt_set_tpa(bp, false);
11227         for (i = 0; i < bp->rx_nr_rings; i++) {
11228                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
11229                 struct bnxt_cp_ring_info *cpr;
11230                 int rc;
11231
11232                 if (!rxr->bnapi->in_reset)
11233                         continue;
11234
11235                 rc = bnxt_hwrm_rx_ring_reset(bp, i);
11236                 if (rc) {
11237                         if (rc == -EINVAL || rc == -EOPNOTSUPP)
11238                                 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
11239                         else
11240                                 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
11241                                             rc);
11242                         bnxt_reset_task(bp, true);
11243                         break;
11244                 }
11245                 bnxt_free_one_rx_ring_skbs(bp, i);
11246                 rxr->rx_prod = 0;
11247                 rxr->rx_agg_prod = 0;
11248                 rxr->rx_sw_agg_prod = 0;
11249                 rxr->rx_next_cons = 0;
11250                 rxr->bnapi->in_reset = false;
11251                 bnxt_alloc_one_rx_ring(bp, i);
11252                 cpr = &rxr->bnapi->cp_ring;
11253                 cpr->sw_stats.rx.rx_resets++;
11254                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
11255                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
11256                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
11257         }
11258         if (bp->flags & BNXT_FLAG_TPA)
11259                 bnxt_set_tpa(bp, true);
11260         bnxt_rtnl_unlock_sp(bp);
11261 }
11262
11263 static void bnxt_fw_reset_close(struct bnxt *bp)
11264 {
11265         bnxt_ulp_stop(bp);
11266         /* When firmware is in fatal state, quiesce device and disable
11267          * bus master to prevent any potential bad DMAs before freeing
11268          * kernel memory.
11269          */
11270         if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11271                 u16 val = 0;
11272
11273                 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
11274                 if (val == 0xffff)
11275                         bp->fw_reset_min_dsecs = 0;
11276                 bnxt_tx_disable(bp);
11277                 bnxt_disable_napi(bp);
11278                 bnxt_disable_int_sync(bp);
11279                 bnxt_free_irq(bp);
11280                 bnxt_clear_int_mode(bp);
11281                 pci_disable_device(bp->pdev);
11282         }
11283         __bnxt_close_nic(bp, true, false);
11284         bnxt_vf_reps_free(bp);
11285         bnxt_clear_int_mode(bp);
11286         bnxt_hwrm_func_drv_unrgtr(bp);
11287         if (pci_is_enabled(bp->pdev))
11288                 pci_disable_device(bp->pdev);
11289         bnxt_free_ctx_mem(bp);
11290         kfree(bp->ctx);
11291         bp->ctx = NULL;
11292 }
11293
11294 static bool is_bnxt_fw_ok(struct bnxt *bp)
11295 {
11296         struct bnxt_fw_health *fw_health = bp->fw_health;
11297         bool no_heartbeat = false, has_reset = false;
11298         u32 val;
11299
11300         val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11301         if (val == fw_health->last_fw_heartbeat)
11302                 no_heartbeat = true;
11303
11304         val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11305         if (val != fw_health->last_fw_reset_cnt)
11306                 has_reset = true;
11307
11308         if (!no_heartbeat && has_reset)
11309                 return true;
11310
11311         return false;
11312 }
11313
11314 /* rtnl_lock is acquired before calling this function */
11315 static void bnxt_force_fw_reset(struct bnxt *bp)
11316 {
11317         struct bnxt_fw_health *fw_health = bp->fw_health;
11318         u32 wait_dsecs;
11319
11320         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
11321             test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11322                 return;
11323
11324         set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11325         bnxt_fw_reset_close(bp);
11326         wait_dsecs = fw_health->master_func_wait_dsecs;
11327         if (fw_health->master) {
11328                 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
11329                         wait_dsecs = 0;
11330                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11331         } else {
11332                 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
11333                 wait_dsecs = fw_health->normal_func_wait_dsecs;
11334                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11335         }
11336
11337         bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
11338         bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
11339         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11340 }
11341
11342 void bnxt_fw_exception(struct bnxt *bp)
11343 {
11344         netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
11345         set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11346         bnxt_rtnl_lock_sp(bp);
11347         bnxt_force_fw_reset(bp);
11348         bnxt_rtnl_unlock_sp(bp);
11349 }
11350
11351 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
11352  * < 0 on error.
11353  */
11354 static int bnxt_get_registered_vfs(struct bnxt *bp)
11355 {
11356 #ifdef CONFIG_BNXT_SRIOV
11357         int rc;
11358
11359         if (!BNXT_PF(bp))
11360                 return 0;
11361
11362         rc = bnxt_hwrm_func_qcfg(bp);
11363         if (rc) {
11364                 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
11365                 return rc;
11366         }
11367         if (bp->pf.registered_vfs)
11368                 return bp->pf.registered_vfs;
11369         if (bp->sriov_cfg)
11370                 return 1;
11371 #endif
11372         return 0;
11373 }
11374
11375 void bnxt_fw_reset(struct bnxt *bp)
11376 {
11377         bnxt_rtnl_lock_sp(bp);
11378         if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
11379             !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11380                 int n = 0, tmo;
11381
11382                 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11383                 if (bp->pf.active_vfs &&
11384                     !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
11385                         n = bnxt_get_registered_vfs(bp);
11386                 if (n < 0) {
11387                         netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
11388                                    n);
11389                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11390                         dev_close(bp->dev);
11391                         goto fw_reset_exit;
11392                 } else if (n > 0) {
11393                         u16 vf_tmo_dsecs = n * 10;
11394
11395                         if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
11396                                 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
11397                         bp->fw_reset_state =
11398                                 BNXT_FW_RESET_STATE_POLL_VF;
11399                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11400                         goto fw_reset_exit;
11401                 }
11402                 bnxt_fw_reset_close(bp);
11403                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11404                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11405                         tmo = HZ / 10;
11406                 } else {
11407                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11408                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11409                 }
11410                 bnxt_queue_fw_reset_work(bp, tmo);
11411         }
11412 fw_reset_exit:
11413         bnxt_rtnl_unlock_sp(bp);
11414 }
11415
11416 static void bnxt_chk_missed_irq(struct bnxt *bp)
11417 {
11418         int i;
11419
11420         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11421                 return;
11422
11423         for (i = 0; i < bp->cp_nr_rings; i++) {
11424                 struct bnxt_napi *bnapi = bp->bnapi[i];
11425                 struct bnxt_cp_ring_info *cpr;
11426                 u32 fw_ring_id;
11427                 int j;
11428
11429                 if (!bnapi)
11430                         continue;
11431
11432                 cpr = &bnapi->cp_ring;
11433                 for (j = 0; j < 2; j++) {
11434                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
11435                         u32 val[2];
11436
11437                         if (!cpr2 || cpr2->has_more_work ||
11438                             !bnxt_has_work(bp, cpr2))
11439                                 continue;
11440
11441                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
11442                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
11443                                 continue;
11444                         }
11445                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
11446                         bnxt_dbg_hwrm_ring_info_get(bp,
11447                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
11448                                 fw_ring_id, &val[0], &val[1]);
11449                         cpr->sw_stats.cmn.missed_irqs++;
11450                 }
11451         }
11452 }
11453
11454 static void bnxt_cfg_ntp_filters(struct bnxt *);
11455
11456 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
11457 {
11458         struct bnxt_link_info *link_info = &bp->link_info;
11459
11460         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
11461                 link_info->autoneg = BNXT_AUTONEG_SPEED;
11462                 if (bp->hwrm_spec_code >= 0x10201) {
11463                         if (link_info->auto_pause_setting &
11464                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
11465                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11466                 } else {
11467                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
11468                 }
11469                 link_info->advertising = link_info->auto_link_speeds;
11470                 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
11471         } else {
11472                 link_info->req_link_speed = link_info->force_link_speed;
11473                 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
11474                 if (link_info->force_pam4_link_speed) {
11475                         link_info->req_link_speed =
11476                                 link_info->force_pam4_link_speed;
11477                         link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
11478                 }
11479                 link_info->req_duplex = link_info->duplex_setting;
11480         }
11481         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
11482                 link_info->req_flow_ctrl =
11483                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
11484         else
11485                 link_info->req_flow_ctrl = link_info->force_pause_setting;
11486 }
11487
11488 static void bnxt_fw_echo_reply(struct bnxt *bp)
11489 {
11490         struct bnxt_fw_health *fw_health = bp->fw_health;
11491         struct hwrm_func_echo_response_input req = {0};
11492
11493         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_ECHO_RESPONSE, -1, -1);
11494         req.event_data1 = cpu_to_le32(fw_health->echo_req_data1);
11495         req.event_data2 = cpu_to_le32(fw_health->echo_req_data2);
11496         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11497 }
11498
11499 static void bnxt_sp_task(struct work_struct *work)
11500 {
11501         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
11502
11503         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11504         smp_mb__after_atomic();
11505         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11506                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11507                 return;
11508         }
11509
11510         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
11511                 bnxt_cfg_rx_mode(bp);
11512
11513         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
11514                 bnxt_cfg_ntp_filters(bp);
11515         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
11516                 bnxt_hwrm_exec_fwd_req(bp);
11517         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
11518                 bnxt_hwrm_port_qstats(bp, 0);
11519                 bnxt_hwrm_port_qstats_ext(bp, 0);
11520                 bnxt_accumulate_all_stats(bp);
11521         }
11522
11523         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
11524                 int rc;
11525
11526                 mutex_lock(&bp->link_lock);
11527                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
11528                                        &bp->sp_event))
11529                         bnxt_hwrm_phy_qcaps(bp);
11530
11531                 rc = bnxt_update_link(bp, true);
11532                 if (rc)
11533                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
11534                                    rc);
11535
11536                 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
11537                                        &bp->sp_event))
11538                         bnxt_init_ethtool_link_settings(bp);
11539                 mutex_unlock(&bp->link_lock);
11540         }
11541         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
11542                 int rc;
11543
11544                 mutex_lock(&bp->link_lock);
11545                 rc = bnxt_update_phy_setting(bp);
11546                 mutex_unlock(&bp->link_lock);
11547                 if (rc) {
11548                         netdev_warn(bp->dev, "update phy settings retry failed\n");
11549                 } else {
11550                         bp->link_info.phy_retry = false;
11551                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
11552                 }
11553         }
11554         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
11555                 mutex_lock(&bp->link_lock);
11556                 bnxt_get_port_module_status(bp);
11557                 mutex_unlock(&bp->link_lock);
11558         }
11559
11560         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
11561                 bnxt_tc_flow_stats_work(bp);
11562
11563         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
11564                 bnxt_chk_missed_irq(bp);
11565
11566         if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
11567                 bnxt_fw_echo_reply(bp);
11568
11569         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
11570          * must be the last functions to be called before exiting.
11571          */
11572         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
11573                 bnxt_reset(bp, false);
11574
11575         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
11576                 bnxt_reset(bp, true);
11577
11578         if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
11579                 bnxt_rx_ring_reset(bp);
11580
11581         if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
11582                 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
11583
11584         if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
11585                 if (!is_bnxt_fw_ok(bp))
11586                         bnxt_devlink_health_report(bp,
11587                                                    BNXT_FW_EXCEPTION_SP_EVENT);
11588         }
11589
11590         smp_mb__before_atomic();
11591         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11592 }
11593
11594 /* Under rtnl_lock */
11595 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
11596                      int tx_xdp)
11597 {
11598         int max_rx, max_tx, tx_sets = 1;
11599         int tx_rings_needed, stats;
11600         int rx_rings = rx;
11601         int cp, vnics, rc;
11602
11603         if (tcs)
11604                 tx_sets = tcs;
11605
11606         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
11607         if (rc)
11608                 return rc;
11609
11610         if (max_rx < rx)
11611                 return -ENOMEM;
11612
11613         tx_rings_needed = tx * tx_sets + tx_xdp;
11614         if (max_tx < tx_rings_needed)
11615                 return -ENOMEM;
11616
11617         vnics = 1;
11618         if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
11619                 vnics += rx_rings;
11620
11621         if (bp->flags & BNXT_FLAG_AGG_RINGS)
11622                 rx_rings <<= 1;
11623         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
11624         stats = cp;
11625         if (BNXT_NEW_RM(bp)) {
11626                 cp += bnxt_get_ulp_msix_num(bp);
11627                 stats += bnxt_get_ulp_stat_ctxs(bp);
11628         }
11629         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
11630                                      stats, vnics);
11631 }
11632
11633 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
11634 {
11635         if (bp->bar2) {
11636                 pci_iounmap(pdev, bp->bar2);
11637                 bp->bar2 = NULL;
11638         }
11639
11640         if (bp->bar1) {
11641                 pci_iounmap(pdev, bp->bar1);
11642                 bp->bar1 = NULL;
11643         }
11644
11645         if (bp->bar0) {
11646                 pci_iounmap(pdev, bp->bar0);
11647                 bp->bar0 = NULL;
11648         }
11649 }
11650
11651 static void bnxt_cleanup_pci(struct bnxt *bp)
11652 {
11653         bnxt_unmap_bars(bp, bp->pdev);
11654         pci_release_regions(bp->pdev);
11655         if (pci_is_enabled(bp->pdev))
11656                 pci_disable_device(bp->pdev);
11657 }
11658
11659 static void bnxt_init_dflt_coal(struct bnxt *bp)
11660 {
11661         struct bnxt_coal *coal;
11662
11663         /* Tick values in micro seconds.
11664          * 1 coal_buf x bufs_per_record = 1 completion record.
11665          */
11666         coal = &bp->rx_coal;
11667         coal->coal_ticks = 10;
11668         coal->coal_bufs = 30;
11669         coal->coal_ticks_irq = 1;
11670         coal->coal_bufs_irq = 2;
11671         coal->idle_thresh = 50;
11672         coal->bufs_per_record = 2;
11673         coal->budget = 64;              /* NAPI budget */
11674
11675         coal = &bp->tx_coal;
11676         coal->coal_ticks = 28;
11677         coal->coal_bufs = 30;
11678         coal->coal_ticks_irq = 2;
11679         coal->coal_bufs_irq = 2;
11680         coal->bufs_per_record = 1;
11681
11682         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
11683 }
11684
11685 static int bnxt_fw_init_one_p1(struct bnxt *bp)
11686 {
11687         int rc;
11688
11689         bp->fw_cap = 0;
11690         rc = bnxt_hwrm_ver_get(bp);
11691         bnxt_try_map_fw_health_reg(bp);
11692         if (rc) {
11693                 rc = bnxt_try_recover_fw(bp);
11694                 if (rc)
11695                         return rc;
11696                 rc = bnxt_hwrm_ver_get(bp);
11697                 if (rc)
11698                         return rc;
11699         }
11700
11701         if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
11702                 rc = bnxt_alloc_kong_hwrm_resources(bp);
11703                 if (rc)
11704                         bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
11705         }
11706
11707         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
11708             bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
11709                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
11710                 if (rc)
11711                         return rc;
11712         }
11713         bnxt_nvm_cfg_ver_get(bp);
11714
11715         rc = bnxt_hwrm_func_reset(bp);
11716         if (rc)
11717                 return -ENODEV;
11718
11719         bnxt_hwrm_fw_set_time(bp);
11720         return 0;
11721 }
11722
11723 static int bnxt_fw_init_one_p2(struct bnxt *bp)
11724 {
11725         int rc;
11726
11727         /* Get the MAX capabilities for this function */
11728         rc = bnxt_hwrm_func_qcaps(bp);
11729         if (rc) {
11730                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
11731                            rc);
11732                 return -ENODEV;
11733         }
11734
11735         rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
11736         if (rc)
11737                 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
11738                             rc);
11739
11740         if (bnxt_alloc_fw_health(bp)) {
11741                 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
11742         } else {
11743                 rc = bnxt_hwrm_error_recovery_qcfg(bp);
11744                 if (rc)
11745                         netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
11746                                     rc);
11747         }
11748
11749         rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
11750         if (rc)
11751                 return -ENODEV;
11752
11753         bnxt_hwrm_func_qcfg(bp);
11754         bnxt_hwrm_vnic_qcaps(bp);
11755         bnxt_hwrm_port_led_qcaps(bp);
11756         bnxt_ethtool_init(bp);
11757         bnxt_dcb_init(bp);
11758         return 0;
11759 }
11760
11761 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
11762 {
11763         bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
11764         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
11765                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
11766                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
11767                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
11768         if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
11769                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
11770                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
11771                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
11772         }
11773 }
11774
11775 static void bnxt_set_dflt_rfs(struct bnxt *bp)
11776 {
11777         struct net_device *dev = bp->dev;
11778
11779         dev->hw_features &= ~NETIF_F_NTUPLE;
11780         dev->features &= ~NETIF_F_NTUPLE;
11781         bp->flags &= ~BNXT_FLAG_RFS;
11782         if (bnxt_rfs_supported(bp)) {
11783                 dev->hw_features |= NETIF_F_NTUPLE;
11784                 if (bnxt_rfs_capable(bp)) {
11785                         bp->flags |= BNXT_FLAG_RFS;
11786                         dev->features |= NETIF_F_NTUPLE;
11787                 }
11788         }
11789 }
11790
11791 static void bnxt_fw_init_one_p3(struct bnxt *bp)
11792 {
11793         struct pci_dev *pdev = bp->pdev;
11794
11795         bnxt_set_dflt_rss_hash_type(bp);
11796         bnxt_set_dflt_rfs(bp);
11797
11798         bnxt_get_wol_settings(bp);
11799         if (bp->flags & BNXT_FLAG_WOL_CAP)
11800                 device_set_wakeup_enable(&pdev->dev, bp->wol);
11801         else
11802                 device_set_wakeup_capable(&pdev->dev, false);
11803
11804         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
11805         bnxt_hwrm_coal_params_qcaps(bp);
11806 }
11807
11808 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
11809
11810 static int bnxt_fw_init_one(struct bnxt *bp)
11811 {
11812         int rc;
11813
11814         rc = bnxt_fw_init_one_p1(bp);
11815         if (rc) {
11816                 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
11817                 return rc;
11818         }
11819         rc = bnxt_fw_init_one_p2(bp);
11820         if (rc) {
11821                 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
11822                 return rc;
11823         }
11824         rc = bnxt_probe_phy(bp, false);
11825         if (rc)
11826                 return rc;
11827         rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
11828         if (rc)
11829                 return rc;
11830
11831         /* In case fw capabilities have changed, destroy the unneeded
11832          * reporters and create newly capable ones.
11833          */
11834         bnxt_dl_fw_reporters_destroy(bp, false);
11835         bnxt_dl_fw_reporters_create(bp);
11836         bnxt_fw_init_one_p3(bp);
11837         return 0;
11838 }
11839
11840 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
11841 {
11842         struct bnxt_fw_health *fw_health = bp->fw_health;
11843         u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
11844         u32 val = fw_health->fw_reset_seq_vals[reg_idx];
11845         u32 reg_type, reg_off, delay_msecs;
11846
11847         delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
11848         reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
11849         reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
11850         switch (reg_type) {
11851         case BNXT_FW_HEALTH_REG_TYPE_CFG:
11852                 pci_write_config_dword(bp->pdev, reg_off, val);
11853                 break;
11854         case BNXT_FW_HEALTH_REG_TYPE_GRC:
11855                 writel(reg_off & BNXT_GRC_BASE_MASK,
11856                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
11857                 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
11858                 fallthrough;
11859         case BNXT_FW_HEALTH_REG_TYPE_BAR0:
11860                 writel(val, bp->bar0 + reg_off);
11861                 break;
11862         case BNXT_FW_HEALTH_REG_TYPE_BAR1:
11863                 writel(val, bp->bar1 + reg_off);
11864                 break;
11865         }
11866         if (delay_msecs) {
11867                 pci_read_config_dword(bp->pdev, 0, &val);
11868                 msleep(delay_msecs);
11869         }
11870 }
11871
11872 static void bnxt_reset_all(struct bnxt *bp)
11873 {
11874         struct bnxt_fw_health *fw_health = bp->fw_health;
11875         int i, rc;
11876
11877         if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11878                 bnxt_fw_reset_via_optee(bp);
11879                 bp->fw_reset_timestamp = jiffies;
11880                 return;
11881         }
11882
11883         if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
11884                 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
11885                         bnxt_fw_reset_writel(bp, i);
11886         } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
11887                 struct hwrm_fw_reset_input req = {0};
11888
11889                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
11890                 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
11891                 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
11892                 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
11893                 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
11894                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11895                 if (rc != -ENODEV)
11896                         netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
11897         }
11898         bp->fw_reset_timestamp = jiffies;
11899 }
11900
11901 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
11902 {
11903         return time_after(jiffies, bp->fw_reset_timestamp +
11904                           (bp->fw_reset_max_dsecs * HZ / 10));
11905 }
11906
11907 static void bnxt_fw_reset_task(struct work_struct *work)
11908 {
11909         struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
11910         int rc;
11911
11912         if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11913                 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
11914                 return;
11915         }
11916
11917         switch (bp->fw_reset_state) {
11918         case BNXT_FW_RESET_STATE_POLL_VF: {
11919                 int n = bnxt_get_registered_vfs(bp);
11920                 int tmo;
11921
11922                 if (n < 0) {
11923                         netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
11924                                    n, jiffies_to_msecs(jiffies -
11925                                    bp->fw_reset_timestamp));
11926                         goto fw_reset_abort;
11927                 } else if (n > 0) {
11928                         if (bnxt_fw_reset_timeout(bp)) {
11929                                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11930                                 bp->fw_reset_state = 0;
11931                                 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
11932                                            n);
11933                                 return;
11934                         }
11935                         bnxt_queue_fw_reset_work(bp, HZ / 10);
11936                         return;
11937                 }
11938                 bp->fw_reset_timestamp = jiffies;
11939                 rtnl_lock();
11940                 bnxt_fw_reset_close(bp);
11941                 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11942                         bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11943                         tmo = HZ / 10;
11944                 } else {
11945                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11946                         tmo = bp->fw_reset_min_dsecs * HZ / 10;
11947                 }
11948                 rtnl_unlock();
11949                 bnxt_queue_fw_reset_work(bp, tmo);
11950                 return;
11951         }
11952         case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
11953                 u32 val;
11954
11955                 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11956                 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
11957                     !bnxt_fw_reset_timeout(bp)) {
11958                         bnxt_queue_fw_reset_work(bp, HZ / 5);
11959                         return;
11960                 }
11961
11962                 if (!bp->fw_health->master) {
11963                         u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
11964
11965                         bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11966                         bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11967                         return;
11968                 }
11969                 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11970         }
11971                 fallthrough;
11972         case BNXT_FW_RESET_STATE_RESET_FW:
11973                 bnxt_reset_all(bp);
11974                 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11975                 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
11976                 return;
11977         case BNXT_FW_RESET_STATE_ENABLE_DEV:
11978                 bnxt_inv_fw_health_reg(bp);
11979                 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
11980                     !bp->fw_reset_min_dsecs) {
11981                         u16 val;
11982
11983                         pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
11984                         if (val == 0xffff) {
11985                                 if (bnxt_fw_reset_timeout(bp)) {
11986                                         netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
11987                                         goto fw_reset_abort;
11988                                 }
11989                                 bnxt_queue_fw_reset_work(bp, HZ / 1000);
11990                                 return;
11991                         }
11992                 }
11993                 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11994                 if (pci_enable_device(bp->pdev)) {
11995                         netdev_err(bp->dev, "Cannot re-enable PCI device\n");
11996                         goto fw_reset_abort;
11997                 }
11998                 pci_set_master(bp->pdev);
11999                 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
12000                 fallthrough;
12001         case BNXT_FW_RESET_STATE_POLL_FW:
12002                 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
12003                 rc = __bnxt_hwrm_ver_get(bp, true);
12004                 if (rc) {
12005                         if (bnxt_fw_reset_timeout(bp)) {
12006                                 netdev_err(bp->dev, "Firmware reset aborted\n");
12007                                 goto fw_reset_abort_status;
12008                         }
12009                         bnxt_queue_fw_reset_work(bp, HZ / 5);
12010                         return;
12011                 }
12012                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
12013                 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
12014                 fallthrough;
12015         case BNXT_FW_RESET_STATE_OPENING:
12016                 while (!rtnl_trylock()) {
12017                         bnxt_queue_fw_reset_work(bp, HZ / 10);
12018                         return;
12019                 }
12020                 rc = bnxt_open(bp->dev);
12021                 if (rc) {
12022                         netdev_err(bp->dev, "bnxt_open_nic() failed\n");
12023                         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12024                         dev_close(bp->dev);
12025                 }
12026
12027                 bp->fw_reset_state = 0;
12028                 /* Make sure fw_reset_state is 0 before clearing the flag */
12029                 smp_mb__before_atomic();
12030                 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12031                 bnxt_ulp_start(bp, rc);
12032                 if (!rc)
12033                         bnxt_reenable_sriov(bp);
12034                 bnxt_vf_reps_alloc(bp);
12035                 bnxt_vf_reps_open(bp);
12036                 bnxt_dl_health_recovery_done(bp);
12037                 bnxt_dl_health_status_update(bp, true);
12038                 rtnl_unlock();
12039                 break;
12040         }
12041         return;
12042
12043 fw_reset_abort_status:
12044         if (bp->fw_health->status_reliable ||
12045             (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
12046                 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12047
12048                 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
12049         }
12050 fw_reset_abort:
12051         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12052         if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
12053                 bnxt_dl_health_status_update(bp, false);
12054         bp->fw_reset_state = 0;
12055         rtnl_lock();
12056         dev_close(bp->dev);
12057         rtnl_unlock();
12058 }
12059
12060 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
12061 {
12062         int rc;
12063         struct bnxt *bp = netdev_priv(dev);
12064
12065         SET_NETDEV_DEV(dev, &pdev->dev);
12066
12067         /* enable device (incl. PCI PM wakeup), and bus-mastering */
12068         rc = pci_enable_device(pdev);
12069         if (rc) {
12070                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
12071                 goto init_err;
12072         }
12073
12074         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
12075                 dev_err(&pdev->dev,
12076                         "Cannot find PCI device base address, aborting\n");
12077                 rc = -ENODEV;
12078                 goto init_err_disable;
12079         }
12080
12081         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
12082         if (rc) {
12083                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
12084                 goto init_err_disable;
12085         }
12086
12087         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
12088             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
12089                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
12090                 rc = -EIO;
12091                 goto init_err_release;
12092         }
12093
12094         pci_set_master(pdev);
12095
12096         bp->dev = dev;
12097         bp->pdev = pdev;
12098
12099         /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
12100          * determines the BAR size.
12101          */
12102         bp->bar0 = pci_ioremap_bar(pdev, 0);
12103         if (!bp->bar0) {
12104                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
12105                 rc = -ENOMEM;
12106                 goto init_err_release;
12107         }
12108
12109         bp->bar2 = pci_ioremap_bar(pdev, 4);
12110         if (!bp->bar2) {
12111                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
12112                 rc = -ENOMEM;
12113                 goto init_err_release;
12114         }
12115
12116         pci_enable_pcie_error_reporting(pdev);
12117
12118         INIT_WORK(&bp->sp_task, bnxt_sp_task);
12119         INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
12120
12121         spin_lock_init(&bp->ntp_fltr_lock);
12122 #if BITS_PER_LONG == 32
12123         spin_lock_init(&bp->db_lock);
12124 #endif
12125
12126         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
12127         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
12128
12129         bnxt_init_dflt_coal(bp);
12130
12131         timer_setup(&bp->timer, bnxt_timer, 0);
12132         bp->current_interval = BNXT_TIMER_INTERVAL;
12133
12134         bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
12135         bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
12136
12137         clear_bit(BNXT_STATE_OPEN, &bp->state);
12138         return 0;
12139
12140 init_err_release:
12141         bnxt_unmap_bars(bp, pdev);
12142         pci_release_regions(pdev);
12143
12144 init_err_disable:
12145         pci_disable_device(pdev);
12146
12147 init_err:
12148         return rc;
12149 }
12150
12151 /* rtnl_lock held */
12152 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
12153 {
12154         struct sockaddr *addr = p;
12155         struct bnxt *bp = netdev_priv(dev);
12156         int rc = 0;
12157
12158         if (!is_valid_ether_addr(addr->sa_data))
12159                 return -EADDRNOTAVAIL;
12160
12161         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
12162                 return 0;
12163
12164         rc = bnxt_approve_mac(bp, addr->sa_data, true);
12165         if (rc)
12166                 return rc;
12167
12168         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
12169         if (netif_running(dev)) {
12170                 bnxt_close_nic(bp, false, false);
12171                 rc = bnxt_open_nic(bp, false, false);
12172         }
12173
12174         return rc;
12175 }
12176
12177 /* rtnl_lock held */
12178 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
12179 {
12180         struct bnxt *bp = netdev_priv(dev);
12181
12182         if (netif_running(dev))
12183                 bnxt_close_nic(bp, true, false);
12184
12185         dev->mtu = new_mtu;
12186         bnxt_set_ring_params(bp);
12187
12188         if (netif_running(dev))
12189                 return bnxt_open_nic(bp, true, false);
12190
12191         return 0;
12192 }
12193
12194 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
12195 {
12196         struct bnxt *bp = netdev_priv(dev);
12197         bool sh = false;
12198         int rc;
12199
12200         if (tc > bp->max_tc) {
12201                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
12202                            tc, bp->max_tc);
12203                 return -EINVAL;
12204         }
12205
12206         if (netdev_get_num_tc(dev) == tc)
12207                 return 0;
12208
12209         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
12210                 sh = true;
12211
12212         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
12213                               sh, tc, bp->tx_nr_rings_xdp);
12214         if (rc)
12215                 return rc;
12216
12217         /* Needs to close the device and do hw resource re-allocations */
12218         if (netif_running(bp->dev))
12219                 bnxt_close_nic(bp, true, false);
12220
12221         if (tc) {
12222                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
12223                 netdev_set_num_tc(dev, tc);
12224         } else {
12225                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12226                 netdev_reset_tc(dev);
12227         }
12228         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
12229         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
12230                                bp->tx_nr_rings + bp->rx_nr_rings;
12231
12232         if (netif_running(bp->dev))
12233                 return bnxt_open_nic(bp, true, false);
12234
12235         return 0;
12236 }
12237
12238 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
12239                                   void *cb_priv)
12240 {
12241         struct bnxt *bp = cb_priv;
12242
12243         if (!bnxt_tc_flower_enabled(bp) ||
12244             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
12245                 return -EOPNOTSUPP;
12246
12247         switch (type) {
12248         case TC_SETUP_CLSFLOWER:
12249                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
12250         default:
12251                 return -EOPNOTSUPP;
12252         }
12253 }
12254
12255 LIST_HEAD(bnxt_block_cb_list);
12256
12257 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
12258                          void *type_data)
12259 {
12260         struct bnxt *bp = netdev_priv(dev);
12261
12262         switch (type) {
12263         case TC_SETUP_BLOCK:
12264                 return flow_block_cb_setup_simple(type_data,
12265                                                   &bnxt_block_cb_list,
12266                                                   bnxt_setup_tc_block_cb,
12267                                                   bp, bp, true);
12268         case TC_SETUP_QDISC_MQPRIO: {
12269                 struct tc_mqprio_qopt *mqprio = type_data;
12270
12271                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
12272
12273                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
12274         }
12275         default:
12276                 return -EOPNOTSUPP;
12277         }
12278 }
12279
12280 #ifdef CONFIG_RFS_ACCEL
12281 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
12282                             struct bnxt_ntuple_filter *f2)
12283 {
12284         struct flow_keys *keys1 = &f1->fkeys;
12285         struct flow_keys *keys2 = &f2->fkeys;
12286
12287         if (keys1->basic.n_proto != keys2->basic.n_proto ||
12288             keys1->basic.ip_proto != keys2->basic.ip_proto)
12289                 return false;
12290
12291         if (keys1->basic.n_proto == htons(ETH_P_IP)) {
12292                 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
12293                     keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
12294                         return false;
12295         } else {
12296                 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
12297                            sizeof(keys1->addrs.v6addrs.src)) ||
12298                     memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
12299                            sizeof(keys1->addrs.v6addrs.dst)))
12300                         return false;
12301         }
12302
12303         if (keys1->ports.ports == keys2->ports.ports &&
12304             keys1->control.flags == keys2->control.flags &&
12305             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
12306             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
12307                 return true;
12308
12309         return false;
12310 }
12311
12312 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
12313                               u16 rxq_index, u32 flow_id)
12314 {
12315         struct bnxt *bp = netdev_priv(dev);
12316         struct bnxt_ntuple_filter *fltr, *new_fltr;
12317         struct flow_keys *fkeys;
12318         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
12319         int rc = 0, idx, bit_id, l2_idx = 0;
12320         struct hlist_head *head;
12321         u32 flags;
12322
12323         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
12324                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12325                 int off = 0, j;
12326
12327                 netif_addr_lock_bh(dev);
12328                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
12329                         if (ether_addr_equal(eth->h_dest,
12330                                              vnic->uc_list + off)) {
12331                                 l2_idx = j + 1;
12332                                 break;
12333                         }
12334                 }
12335                 netif_addr_unlock_bh(dev);
12336                 if (!l2_idx)
12337                         return -EINVAL;
12338         }
12339         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
12340         if (!new_fltr)
12341                 return -ENOMEM;
12342
12343         fkeys = &new_fltr->fkeys;
12344         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
12345                 rc = -EPROTONOSUPPORT;
12346                 goto err_free;
12347         }
12348
12349         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
12350              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
12351             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
12352              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
12353                 rc = -EPROTONOSUPPORT;
12354                 goto err_free;
12355         }
12356         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
12357             bp->hwrm_spec_code < 0x10601) {
12358                 rc = -EPROTONOSUPPORT;
12359                 goto err_free;
12360         }
12361         flags = fkeys->control.flags;
12362         if (((flags & FLOW_DIS_ENCAPSULATION) &&
12363              bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
12364                 rc = -EPROTONOSUPPORT;
12365                 goto err_free;
12366         }
12367
12368         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
12369         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
12370
12371         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
12372         head = &bp->ntp_fltr_hash_tbl[idx];
12373         rcu_read_lock();
12374         hlist_for_each_entry_rcu(fltr, head, hash) {
12375                 if (bnxt_fltr_match(fltr, new_fltr)) {
12376                         rcu_read_unlock();
12377                         rc = 0;
12378                         goto err_free;
12379                 }
12380         }
12381         rcu_read_unlock();
12382
12383         spin_lock_bh(&bp->ntp_fltr_lock);
12384         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
12385                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
12386         if (bit_id < 0) {
12387                 spin_unlock_bh(&bp->ntp_fltr_lock);
12388                 rc = -ENOMEM;
12389                 goto err_free;
12390         }
12391
12392         new_fltr->sw_id = (u16)bit_id;
12393         new_fltr->flow_id = flow_id;
12394         new_fltr->l2_fltr_idx = l2_idx;
12395         new_fltr->rxq = rxq_index;
12396         hlist_add_head_rcu(&new_fltr->hash, head);
12397         bp->ntp_fltr_count++;
12398         spin_unlock_bh(&bp->ntp_fltr_lock);
12399
12400         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
12401         bnxt_queue_sp_work(bp);
12402
12403         return new_fltr->sw_id;
12404
12405 err_free:
12406         kfree(new_fltr);
12407         return rc;
12408 }
12409
12410 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
12411 {
12412         int i;
12413
12414         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
12415                 struct hlist_head *head;
12416                 struct hlist_node *tmp;
12417                 struct bnxt_ntuple_filter *fltr;
12418                 int rc;
12419
12420                 head = &bp->ntp_fltr_hash_tbl[i];
12421                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
12422                         bool del = false;
12423
12424                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
12425                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
12426                                                         fltr->flow_id,
12427                                                         fltr->sw_id)) {
12428                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
12429                                                                          fltr);
12430                                         del = true;
12431                                 }
12432                         } else {
12433                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
12434                                                                        fltr);
12435                                 if (rc)
12436                                         del = true;
12437                                 else
12438                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
12439                         }
12440
12441                         if (del) {
12442                                 spin_lock_bh(&bp->ntp_fltr_lock);
12443                                 hlist_del_rcu(&fltr->hash);
12444                                 bp->ntp_fltr_count--;
12445                                 spin_unlock_bh(&bp->ntp_fltr_lock);
12446                                 synchronize_rcu();
12447                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
12448                                 kfree(fltr);
12449                         }
12450                 }
12451         }
12452         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
12453                 netdev_info(bp->dev, "Receive PF driver unload event!\n");
12454 }
12455
12456 #else
12457
12458 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
12459 {
12460 }
12461
12462 #endif /* CONFIG_RFS_ACCEL */
12463
12464 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
12465 {
12466         struct bnxt *bp = netdev_priv(netdev);
12467         struct udp_tunnel_info ti;
12468         unsigned int cmd;
12469
12470         udp_tunnel_nic_get_port(netdev, table, 0, &ti);
12471         if (ti.type == UDP_TUNNEL_TYPE_VXLAN) {
12472                 bp->vxlan_port = ti.port;
12473                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
12474         } else {
12475                 bp->nge_port = ti.port;
12476                 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
12477         }
12478
12479         if (ti.port)
12480                 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
12481
12482         return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
12483 }
12484
12485 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
12486         .sync_table     = bnxt_udp_tunnel_sync,
12487         .flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
12488                           UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
12489         .tables         = {
12490                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
12491                 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
12492         },
12493 };
12494
12495 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
12496                                struct net_device *dev, u32 filter_mask,
12497                                int nlflags)
12498 {
12499         struct bnxt *bp = netdev_priv(dev);
12500
12501         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
12502                                        nlflags, filter_mask, NULL);
12503 }
12504
12505 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
12506                                u16 flags, struct netlink_ext_ack *extack)
12507 {
12508         struct bnxt *bp = netdev_priv(dev);
12509         struct nlattr *attr, *br_spec;
12510         int rem, rc = 0;
12511
12512         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
12513                 return -EOPNOTSUPP;
12514
12515         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12516         if (!br_spec)
12517                 return -EINVAL;
12518
12519         nla_for_each_nested(attr, br_spec, rem) {
12520                 u16 mode;
12521
12522                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12523                         continue;
12524
12525                 if (nla_len(attr) < sizeof(mode))
12526                         return -EINVAL;
12527
12528                 mode = nla_get_u16(attr);
12529                 if (mode == bp->br_mode)
12530                         break;
12531
12532                 rc = bnxt_hwrm_set_br_mode(bp, mode);
12533                 if (!rc)
12534                         bp->br_mode = mode;
12535                 break;
12536         }
12537         return rc;
12538 }
12539
12540 int bnxt_get_port_parent_id(struct net_device *dev,
12541                             struct netdev_phys_item_id *ppid)
12542 {
12543         struct bnxt *bp = netdev_priv(dev);
12544
12545         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
12546                 return -EOPNOTSUPP;
12547
12548         /* The PF and it's VF-reps only support the switchdev framework */
12549         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
12550                 return -EOPNOTSUPP;
12551
12552         ppid->id_len = sizeof(bp->dsn);
12553         memcpy(ppid->id, bp->dsn, ppid->id_len);
12554
12555         return 0;
12556 }
12557
12558 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
12559 {
12560         struct bnxt *bp = netdev_priv(dev);
12561
12562         return &bp->dl_port;
12563 }
12564
12565 static const struct net_device_ops bnxt_netdev_ops = {
12566         .ndo_open               = bnxt_open,
12567         .ndo_start_xmit         = bnxt_start_xmit,
12568         .ndo_stop               = bnxt_close,
12569         .ndo_get_stats64        = bnxt_get_stats64,
12570         .ndo_set_rx_mode        = bnxt_set_rx_mode,
12571         .ndo_do_ioctl           = bnxt_ioctl,
12572         .ndo_validate_addr      = eth_validate_addr,
12573         .ndo_set_mac_address    = bnxt_change_mac_addr,
12574         .ndo_change_mtu         = bnxt_change_mtu,
12575         .ndo_fix_features       = bnxt_fix_features,
12576         .ndo_set_features       = bnxt_set_features,
12577         .ndo_features_check     = bnxt_features_check,
12578         .ndo_tx_timeout         = bnxt_tx_timeout,
12579 #ifdef CONFIG_BNXT_SRIOV
12580         .ndo_get_vf_config      = bnxt_get_vf_config,
12581         .ndo_set_vf_mac         = bnxt_set_vf_mac,
12582         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
12583         .ndo_set_vf_rate        = bnxt_set_vf_bw,
12584         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
12585         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
12586         .ndo_set_vf_trust       = bnxt_set_vf_trust,
12587 #endif
12588         .ndo_setup_tc           = bnxt_setup_tc,
12589 #ifdef CONFIG_RFS_ACCEL
12590         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
12591 #endif
12592         .ndo_bpf                = bnxt_xdp,
12593         .ndo_xdp_xmit           = bnxt_xdp_xmit,
12594         .ndo_bridge_getlink     = bnxt_bridge_getlink,
12595         .ndo_bridge_setlink     = bnxt_bridge_setlink,
12596         .ndo_get_devlink_port   = bnxt_get_devlink_port,
12597 };
12598
12599 static void bnxt_remove_one(struct pci_dev *pdev)
12600 {
12601         struct net_device *dev = pci_get_drvdata(pdev);
12602         struct bnxt *bp = netdev_priv(dev);
12603
12604         if (BNXT_PF(bp))
12605                 bnxt_sriov_disable(bp);
12606
12607         if (BNXT_PF(bp))
12608                 devlink_port_type_clear(&bp->dl_port);
12609         pci_disable_pcie_error_reporting(pdev);
12610         unregister_netdev(dev);
12611         clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12612         /* Flush any pending tasks */
12613         cancel_work_sync(&bp->sp_task);
12614         cancel_delayed_work_sync(&bp->fw_reset_task);
12615         bp->sp_event = 0;
12616
12617         bnxt_dl_fw_reporters_destroy(bp, true);
12618         bnxt_dl_unregister(bp);
12619         bnxt_shutdown_tc(bp);
12620
12621         bnxt_clear_int_mode(bp);
12622         bnxt_hwrm_func_drv_unrgtr(bp);
12623         bnxt_free_hwrm_resources(bp);
12624         bnxt_free_hwrm_short_cmd_req(bp);
12625         bnxt_ethtool_free(bp);
12626         bnxt_dcb_free(bp);
12627         kfree(bp->edev);
12628         bp->edev = NULL;
12629         kfree(bp->ptp_cfg);
12630         bp->ptp_cfg = NULL;
12631         kfree(bp->fw_health);
12632         bp->fw_health = NULL;
12633         bnxt_cleanup_pci(bp);
12634         bnxt_free_ctx_mem(bp);
12635         kfree(bp->ctx);
12636         bp->ctx = NULL;
12637         kfree(bp->rss_indir_tbl);
12638         bp->rss_indir_tbl = NULL;
12639         bnxt_free_port_stats(bp);
12640         free_netdev(dev);
12641 }
12642
12643 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
12644 {
12645         int rc = 0;
12646         struct bnxt_link_info *link_info = &bp->link_info;
12647
12648         bp->phy_flags = 0;
12649         rc = bnxt_hwrm_phy_qcaps(bp);
12650         if (rc) {
12651                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
12652                            rc);
12653                 return rc;
12654         }
12655         if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
12656                 bp->dev->priv_flags |= IFF_SUPP_NOFCS;
12657         else
12658                 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
12659         if (!fw_dflt)
12660                 return 0;
12661
12662         rc = bnxt_update_link(bp, false);
12663         if (rc) {
12664                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
12665                            rc);
12666                 return rc;
12667         }
12668
12669         /* Older firmware does not have supported_auto_speeds, so assume
12670          * that all supported speeds can be autonegotiated.
12671          */
12672         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
12673                 link_info->support_auto_speeds = link_info->support_speeds;
12674
12675         bnxt_init_ethtool_link_settings(bp);
12676         return 0;
12677 }
12678
12679 static int bnxt_get_max_irq(struct pci_dev *pdev)
12680 {
12681         u16 ctrl;
12682
12683         if (!pdev->msix_cap)
12684                 return 1;
12685
12686         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
12687         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
12688 }
12689
12690 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12691                                 int *max_cp)
12692 {
12693         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
12694         int max_ring_grps = 0, max_irq;
12695
12696         *max_tx = hw_resc->max_tx_rings;
12697         *max_rx = hw_resc->max_rx_rings;
12698         *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
12699         max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
12700                         bnxt_get_ulp_msix_num(bp),
12701                         hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
12702         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
12703                 *max_cp = min_t(int, *max_cp, max_irq);
12704         max_ring_grps = hw_resc->max_hw_ring_grps;
12705         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
12706                 *max_cp -= 1;
12707                 *max_rx -= 2;
12708         }
12709         if (bp->flags & BNXT_FLAG_AGG_RINGS)
12710                 *max_rx >>= 1;
12711         if (bp->flags & BNXT_FLAG_CHIP_P5) {
12712                 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
12713                 /* On P5 chips, max_cp output param should be available NQs */
12714                 *max_cp = max_irq;
12715         }
12716         *max_rx = min_t(int, *max_rx, max_ring_grps);
12717 }
12718
12719 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
12720 {
12721         int rx, tx, cp;
12722
12723         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
12724         *max_rx = rx;
12725         *max_tx = tx;
12726         if (!rx || !tx || !cp)
12727                 return -ENOMEM;
12728
12729         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
12730 }
12731
12732 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12733                                bool shared)
12734 {
12735         int rc;
12736
12737         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12738         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
12739                 /* Not enough rings, try disabling agg rings. */
12740                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
12741                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12742                 if (rc) {
12743                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
12744                         bp->flags |= BNXT_FLAG_AGG_RINGS;
12745                         return rc;
12746                 }
12747                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
12748                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12749                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12750                 bnxt_set_ring_params(bp);
12751         }
12752
12753         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
12754                 int max_cp, max_stat, max_irq;
12755
12756                 /* Reserve minimum resources for RoCE */
12757                 max_cp = bnxt_get_max_func_cp_rings(bp);
12758                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
12759                 max_irq = bnxt_get_max_func_irqs(bp);
12760                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
12761                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
12762                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
12763                         return 0;
12764
12765                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
12766                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
12767                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
12768                 max_cp = min_t(int, max_cp, max_irq);
12769                 max_cp = min_t(int, max_cp, max_stat);
12770                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
12771                 if (rc)
12772                         rc = 0;
12773         }
12774         return rc;
12775 }
12776
12777 /* In initial default shared ring setting, each shared ring must have a
12778  * RX/TX ring pair.
12779  */
12780 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
12781 {
12782         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
12783         bp->rx_nr_rings = bp->cp_nr_rings;
12784         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
12785         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12786 }
12787
12788 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
12789 {
12790         int dflt_rings, max_rx_rings, max_tx_rings, rc;
12791
12792         if (!bnxt_can_reserve_rings(bp))
12793                 return 0;
12794
12795         if (sh)
12796                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
12797         dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
12798         /* Reduce default rings on multi-port cards so that total default
12799          * rings do not exceed CPU count.
12800          */
12801         if (bp->port_count > 1) {
12802                 int max_rings =
12803                         max_t(int, num_online_cpus() / bp->port_count, 1);
12804
12805                 dflt_rings = min_t(int, dflt_rings, max_rings);
12806         }
12807         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
12808         if (rc)
12809                 return rc;
12810         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
12811         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
12812         if (sh)
12813                 bnxt_trim_dflt_sh_rings(bp);
12814         else
12815                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
12816         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12817
12818         rc = __bnxt_reserve_rings(bp);
12819         if (rc)
12820                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
12821         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12822         if (sh)
12823                 bnxt_trim_dflt_sh_rings(bp);
12824
12825         /* Rings may have been trimmed, re-reserve the trimmed rings. */
12826         if (bnxt_need_reserve_rings(bp)) {
12827                 rc = __bnxt_reserve_rings(bp);
12828                 if (rc)
12829                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
12830                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12831         }
12832         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
12833                 bp->rx_nr_rings++;
12834                 bp->cp_nr_rings++;
12835         }
12836         if (rc) {
12837                 bp->tx_nr_rings = 0;
12838                 bp->rx_nr_rings = 0;
12839         }
12840         return rc;
12841 }
12842
12843 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
12844 {
12845         int rc;
12846
12847         if (bp->tx_nr_rings)
12848                 return 0;
12849
12850         bnxt_ulp_irq_stop(bp);
12851         bnxt_clear_int_mode(bp);
12852         rc = bnxt_set_dflt_rings(bp, true);
12853         if (rc) {
12854                 netdev_err(bp->dev, "Not enough rings available.\n");
12855                 goto init_dflt_ring_err;
12856         }
12857         rc = bnxt_init_int_mode(bp);
12858         if (rc)
12859                 goto init_dflt_ring_err;
12860
12861         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12862         if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
12863                 bp->flags |= BNXT_FLAG_RFS;
12864                 bp->dev->features |= NETIF_F_NTUPLE;
12865         }
12866 init_dflt_ring_err:
12867         bnxt_ulp_irq_restart(bp, rc);
12868         return rc;
12869 }
12870
12871 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
12872 {
12873         int rc;
12874
12875         ASSERT_RTNL();
12876         bnxt_hwrm_func_qcaps(bp);
12877
12878         if (netif_running(bp->dev))
12879                 __bnxt_close_nic(bp, true, false);
12880
12881         bnxt_ulp_irq_stop(bp);
12882         bnxt_clear_int_mode(bp);
12883         rc = bnxt_init_int_mode(bp);
12884         bnxt_ulp_irq_restart(bp, rc);
12885
12886         if (netif_running(bp->dev)) {
12887                 if (rc)
12888                         dev_close(bp->dev);
12889                 else
12890                         rc = bnxt_open_nic(bp, true, false);
12891         }
12892
12893         return rc;
12894 }
12895
12896 static int bnxt_init_mac_addr(struct bnxt *bp)
12897 {
12898         int rc = 0;
12899
12900         if (BNXT_PF(bp)) {
12901                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
12902         } else {
12903 #ifdef CONFIG_BNXT_SRIOV
12904                 struct bnxt_vf_info *vf = &bp->vf;
12905                 bool strict_approval = true;
12906
12907                 if (is_valid_ether_addr(vf->mac_addr)) {
12908                         /* overwrite netdev dev_addr with admin VF MAC */
12909                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
12910                         /* Older PF driver or firmware may not approve this
12911                          * correctly.
12912                          */
12913                         strict_approval = false;
12914                 } else {
12915                         eth_hw_addr_random(bp->dev);
12916                 }
12917                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
12918 #endif
12919         }
12920         return rc;
12921 }
12922
12923 #define BNXT_VPD_LEN    512
12924 static void bnxt_vpd_read_info(struct bnxt *bp)
12925 {
12926         struct pci_dev *pdev = bp->pdev;
12927         int i, len, pos, ro_size, size;
12928         ssize_t vpd_size;
12929         u8 *vpd_data;
12930
12931         vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL);
12932         if (!vpd_data)
12933                 return;
12934
12935         vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data);
12936         if (vpd_size <= 0) {
12937                 netdev_err(bp->dev, "Unable to read VPD\n");
12938                 goto exit;
12939         }
12940
12941         i = pci_vpd_find_tag(vpd_data, vpd_size, PCI_VPD_LRDT_RO_DATA);
12942         if (i < 0) {
12943                 netdev_err(bp->dev, "VPD READ-Only not found\n");
12944                 goto exit;
12945         }
12946
12947         ro_size = pci_vpd_lrdt_size(&vpd_data[i]);
12948         i += PCI_VPD_LRDT_TAG_SIZE;
12949         if (i + ro_size > vpd_size)
12950                 goto exit;
12951
12952         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12953                                         PCI_VPD_RO_KEYWORD_PARTNO);
12954         if (pos < 0)
12955                 goto read_sn;
12956
12957         len = pci_vpd_info_field_size(&vpd_data[pos]);
12958         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12959         if (len + pos > vpd_size)
12960                 goto read_sn;
12961
12962         size = min(len, BNXT_VPD_FLD_LEN - 1);
12963         memcpy(bp->board_partno, &vpd_data[pos], size);
12964
12965 read_sn:
12966         pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12967                                         PCI_VPD_RO_KEYWORD_SERIALNO);
12968         if (pos < 0)
12969                 goto exit;
12970
12971         len = pci_vpd_info_field_size(&vpd_data[pos]);
12972         pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12973         if (len + pos > vpd_size)
12974                 goto exit;
12975
12976         size = min(len, BNXT_VPD_FLD_LEN - 1);
12977         memcpy(bp->board_serialno, &vpd_data[pos], size);
12978 exit:
12979         kfree(vpd_data);
12980 }
12981
12982 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
12983 {
12984         struct pci_dev *pdev = bp->pdev;
12985         u64 qword;
12986
12987         qword = pci_get_dsn(pdev);
12988         if (!qword) {
12989                 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
12990                 return -EOPNOTSUPP;
12991         }
12992
12993         put_unaligned_le64(qword, dsn);
12994
12995         bp->flags |= BNXT_FLAG_DSN_VALID;
12996         return 0;
12997 }
12998
12999 static int bnxt_map_db_bar(struct bnxt *bp)
13000 {
13001         if (!bp->db_size)
13002                 return -ENODEV;
13003         bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
13004         if (!bp->bar1)
13005                 return -ENOMEM;
13006         return 0;
13007 }
13008
13009 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
13010 {
13011         struct net_device *dev;
13012         struct bnxt *bp;
13013         int rc, max_irqs;
13014
13015         if (pci_is_bridge(pdev))
13016                 return -ENODEV;
13017
13018         /* Clear any pending DMA transactions from crash kernel
13019          * while loading driver in capture kernel.
13020          */
13021         if (is_kdump_kernel()) {
13022                 pci_clear_master(pdev);
13023                 pcie_flr(pdev);
13024         }
13025
13026         max_irqs = bnxt_get_max_irq(pdev);
13027         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
13028         if (!dev)
13029                 return -ENOMEM;
13030
13031         bp = netdev_priv(dev);
13032         bp->msg_enable = BNXT_DEF_MSG_ENABLE;
13033         bnxt_set_max_func_irqs(bp, max_irqs);
13034
13035         if (bnxt_vf_pciid(ent->driver_data))
13036                 bp->flags |= BNXT_FLAG_VF;
13037
13038         if (pdev->msix_cap)
13039                 bp->flags |= BNXT_FLAG_MSIX_CAP;
13040
13041         rc = bnxt_init_board(pdev, dev);
13042         if (rc < 0)
13043                 goto init_err_free;
13044
13045         dev->netdev_ops = &bnxt_netdev_ops;
13046         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
13047         dev->ethtool_ops = &bnxt_ethtool_ops;
13048         pci_set_drvdata(pdev, dev);
13049
13050         rc = bnxt_alloc_hwrm_resources(bp);
13051         if (rc)
13052                 goto init_err_pci_clean;
13053
13054         mutex_init(&bp->hwrm_cmd_lock);
13055         mutex_init(&bp->link_lock);
13056
13057         rc = bnxt_fw_init_one_p1(bp);
13058         if (rc)
13059                 goto init_err_pci_clean;
13060
13061         if (BNXT_PF(bp))
13062                 bnxt_vpd_read_info(bp);
13063
13064         if (BNXT_CHIP_P5(bp)) {
13065                 bp->flags |= BNXT_FLAG_CHIP_P5;
13066                 if (BNXT_CHIP_SR2(bp))
13067                         bp->flags |= BNXT_FLAG_CHIP_SR2;
13068         }
13069
13070         rc = bnxt_alloc_rss_indir_tbl(bp);
13071         if (rc)
13072                 goto init_err_pci_clean;
13073
13074         rc = bnxt_fw_init_one_p2(bp);
13075         if (rc)
13076                 goto init_err_pci_clean;
13077
13078         rc = bnxt_map_db_bar(bp);
13079         if (rc) {
13080                 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
13081                         rc);
13082                 goto init_err_pci_clean;
13083         }
13084
13085         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13086                            NETIF_F_TSO | NETIF_F_TSO6 |
13087                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13088                            NETIF_F_GSO_IPXIP4 |
13089                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13090                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
13091                            NETIF_F_RXCSUM | NETIF_F_GRO;
13092
13093         if (BNXT_SUPPORTS_TPA(bp))
13094                 dev->hw_features |= NETIF_F_LRO;
13095
13096         dev->hw_enc_features =
13097                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13098                         NETIF_F_TSO | NETIF_F_TSO6 |
13099                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13100                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13101                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
13102         dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
13103
13104         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
13105                                     NETIF_F_GSO_GRE_CSUM;
13106         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
13107         if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
13108                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
13109         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
13110                 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
13111         if (BNXT_SUPPORTS_TPA(bp))
13112                 dev->hw_features |= NETIF_F_GRO_HW;
13113         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
13114         if (dev->features & NETIF_F_GRO_HW)
13115                 dev->features &= ~NETIF_F_LRO;
13116         dev->priv_flags |= IFF_UNICAST_FLT;
13117
13118 #ifdef CONFIG_BNXT_SRIOV
13119         init_waitqueue_head(&bp->sriov_cfg_wait);
13120         mutex_init(&bp->sriov_lock);
13121 #endif
13122         if (BNXT_SUPPORTS_TPA(bp)) {
13123                 bp->gro_func = bnxt_gro_func_5730x;
13124                 if (BNXT_CHIP_P4(bp))
13125                         bp->gro_func = bnxt_gro_func_5731x;
13126                 else if (BNXT_CHIP_P5(bp))
13127                         bp->gro_func = bnxt_gro_func_5750x;
13128         }
13129         if (!BNXT_CHIP_P4_PLUS(bp))
13130                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
13131
13132         rc = bnxt_init_mac_addr(bp);
13133         if (rc) {
13134                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
13135                 rc = -EADDRNOTAVAIL;
13136                 goto init_err_pci_clean;
13137         }
13138
13139         if (BNXT_PF(bp)) {
13140                 /* Read the adapter's DSN to use as the eswitch switch_id */
13141                 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
13142         }
13143
13144         /* MTU range: 60 - FW defined max */
13145         dev->min_mtu = ETH_ZLEN;
13146         dev->max_mtu = bp->max_mtu;
13147
13148         rc = bnxt_probe_phy(bp, true);
13149         if (rc)
13150                 goto init_err_pci_clean;
13151
13152         bnxt_set_rx_skb_mode(bp, false);
13153         bnxt_set_tpa_flags(bp);
13154         bnxt_set_ring_params(bp);
13155         rc = bnxt_set_dflt_rings(bp, true);
13156         if (rc) {
13157                 netdev_err(bp->dev, "Not enough rings available.\n");
13158                 rc = -ENOMEM;
13159                 goto init_err_pci_clean;
13160         }
13161
13162         bnxt_fw_init_one_p3(bp);
13163
13164         if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
13165                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
13166
13167         rc = bnxt_init_int_mode(bp);
13168         if (rc)
13169                 goto init_err_pci_clean;
13170
13171         /* No TC has been set yet and rings may have been trimmed due to
13172          * limited MSIX, so we re-initialize the TX rings per TC.
13173          */
13174         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13175
13176         if (BNXT_PF(bp)) {
13177                 if (!bnxt_pf_wq) {
13178                         bnxt_pf_wq =
13179                                 create_singlethread_workqueue("bnxt_pf_wq");
13180                         if (!bnxt_pf_wq) {
13181                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
13182                                 rc = -ENOMEM;
13183                                 goto init_err_pci_clean;
13184                         }
13185                 }
13186                 rc = bnxt_init_tc(bp);
13187                 if (rc)
13188                         netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
13189                                    rc);
13190         }
13191
13192         bnxt_inv_fw_health_reg(bp);
13193         bnxt_dl_register(bp);
13194
13195         rc = register_netdev(dev);
13196         if (rc)
13197                 goto init_err_cleanup;
13198
13199         if (BNXT_PF(bp))
13200                 devlink_port_type_eth_set(&bp->dl_port, bp->dev);
13201         bnxt_dl_fw_reporters_create(bp);
13202
13203         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
13204                     board_info[ent->driver_data].name,
13205                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
13206         pcie_print_link_status(pdev);
13207
13208         pci_save_state(pdev);
13209         return 0;
13210
13211 init_err_cleanup:
13212         bnxt_dl_unregister(bp);
13213         bnxt_shutdown_tc(bp);
13214         bnxt_clear_int_mode(bp);
13215
13216 init_err_pci_clean:
13217         bnxt_hwrm_func_drv_unrgtr(bp);
13218         bnxt_free_hwrm_short_cmd_req(bp);
13219         bnxt_free_hwrm_resources(bp);
13220         bnxt_ethtool_free(bp);
13221         kfree(bp->ptp_cfg);
13222         bp->ptp_cfg = NULL;
13223         kfree(bp->fw_health);
13224         bp->fw_health = NULL;
13225         bnxt_cleanup_pci(bp);
13226         bnxt_free_ctx_mem(bp);
13227         kfree(bp->ctx);
13228         bp->ctx = NULL;
13229         kfree(bp->rss_indir_tbl);
13230         bp->rss_indir_tbl = NULL;
13231
13232 init_err_free:
13233         free_netdev(dev);
13234         return rc;
13235 }
13236
13237 static void bnxt_shutdown(struct pci_dev *pdev)
13238 {
13239         struct net_device *dev = pci_get_drvdata(pdev);
13240         struct bnxt *bp;
13241
13242         if (!dev)
13243                 return;
13244
13245         rtnl_lock();
13246         bp = netdev_priv(dev);
13247         if (!bp)
13248                 goto shutdown_exit;
13249
13250         if (netif_running(dev))
13251                 dev_close(dev);
13252
13253         bnxt_ulp_shutdown(bp);
13254         bnxt_clear_int_mode(bp);
13255         pci_disable_device(pdev);
13256
13257         if (system_state == SYSTEM_POWER_OFF) {
13258                 pci_wake_from_d3(pdev, bp->wol);
13259                 pci_set_power_state(pdev, PCI_D3hot);
13260         }
13261
13262 shutdown_exit:
13263         rtnl_unlock();
13264 }
13265
13266 #ifdef CONFIG_PM_SLEEP
13267 static int bnxt_suspend(struct device *device)
13268 {
13269         struct net_device *dev = dev_get_drvdata(device);
13270         struct bnxt *bp = netdev_priv(dev);
13271         int rc = 0;
13272
13273         rtnl_lock();
13274         bnxt_ulp_stop(bp);
13275         if (netif_running(dev)) {
13276                 netif_device_detach(dev);
13277                 rc = bnxt_close(dev);
13278         }
13279         bnxt_hwrm_func_drv_unrgtr(bp);
13280         pci_disable_device(bp->pdev);
13281         bnxt_free_ctx_mem(bp);
13282         kfree(bp->ctx);
13283         bp->ctx = NULL;
13284         rtnl_unlock();
13285         return rc;
13286 }
13287
13288 static int bnxt_resume(struct device *device)
13289 {
13290         struct net_device *dev = dev_get_drvdata(device);
13291         struct bnxt *bp = netdev_priv(dev);
13292         int rc = 0;
13293
13294         rtnl_lock();
13295         rc = pci_enable_device(bp->pdev);
13296         if (rc) {
13297                 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
13298                            rc);
13299                 goto resume_exit;
13300         }
13301         pci_set_master(bp->pdev);
13302         if (bnxt_hwrm_ver_get(bp)) {
13303                 rc = -ENODEV;
13304                 goto resume_exit;
13305         }
13306         rc = bnxt_hwrm_func_reset(bp);
13307         if (rc) {
13308                 rc = -EBUSY;
13309                 goto resume_exit;
13310         }
13311
13312         rc = bnxt_hwrm_func_qcaps(bp);
13313         if (rc)
13314                 goto resume_exit;
13315
13316         if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
13317                 rc = -ENODEV;
13318                 goto resume_exit;
13319         }
13320
13321         bnxt_get_wol_settings(bp);
13322         if (netif_running(dev)) {
13323                 rc = bnxt_open(dev);
13324                 if (!rc)
13325                         netif_device_attach(dev);
13326         }
13327
13328 resume_exit:
13329         bnxt_ulp_start(bp, rc);
13330         if (!rc)
13331                 bnxt_reenable_sriov(bp);
13332         rtnl_unlock();
13333         return rc;
13334 }
13335
13336 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
13337 #define BNXT_PM_OPS (&bnxt_pm_ops)
13338
13339 #else
13340
13341 #define BNXT_PM_OPS NULL
13342
13343 #endif /* CONFIG_PM_SLEEP */
13344
13345 /**
13346  * bnxt_io_error_detected - called when PCI error is detected
13347  * @pdev: Pointer to PCI device
13348  * @state: The current pci connection state
13349  *
13350  * This function is called after a PCI bus error affecting
13351  * this device has been detected.
13352  */
13353 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
13354                                                pci_channel_state_t state)
13355 {
13356         struct net_device *netdev = pci_get_drvdata(pdev);
13357         struct bnxt *bp = netdev_priv(netdev);
13358
13359         netdev_info(netdev, "PCI I/O error detected\n");
13360
13361         rtnl_lock();
13362         netif_device_detach(netdev);
13363
13364         bnxt_ulp_stop(bp);
13365
13366         if (state == pci_channel_io_perm_failure) {
13367                 rtnl_unlock();
13368                 return PCI_ERS_RESULT_DISCONNECT;
13369         }
13370
13371         if (state == pci_channel_io_frozen)
13372                 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
13373
13374         if (netif_running(netdev))
13375                 bnxt_close(netdev);
13376
13377         pci_disable_device(pdev);
13378         bnxt_free_ctx_mem(bp);
13379         kfree(bp->ctx);
13380         bp->ctx = NULL;
13381         rtnl_unlock();
13382
13383         /* Request a slot slot reset. */
13384         return PCI_ERS_RESULT_NEED_RESET;
13385 }
13386
13387 /**
13388  * bnxt_io_slot_reset - called after the pci bus has been reset.
13389  * @pdev: Pointer to PCI device
13390  *
13391  * Restart the card from scratch, as if from a cold-boot.
13392  * At this point, the card has exprienced a hard reset,
13393  * followed by fixups by BIOS, and has its config space
13394  * set up identically to what it was at cold boot.
13395  */
13396 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
13397 {
13398         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
13399         struct net_device *netdev = pci_get_drvdata(pdev);
13400         struct bnxt *bp = netdev_priv(netdev);
13401         int err = 0, off;
13402
13403         netdev_info(bp->dev, "PCI Slot Reset\n");
13404
13405         rtnl_lock();
13406
13407         if (pci_enable_device(pdev)) {
13408                 dev_err(&pdev->dev,
13409                         "Cannot re-enable PCI device after reset.\n");
13410         } else {
13411                 pci_set_master(pdev);
13412                 /* Upon fatal error, our device internal logic that latches to
13413                  * BAR value is getting reset and will restore only upon
13414                  * rewritting the BARs.
13415                  *
13416                  * As pci_restore_state() does not re-write the BARs if the
13417                  * value is same as saved value earlier, driver needs to
13418                  * write the BARs to 0 to force restore, in case of fatal error.
13419                  */
13420                 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
13421                                        &bp->state)) {
13422                         for (off = PCI_BASE_ADDRESS_0;
13423                              off <= PCI_BASE_ADDRESS_5; off += 4)
13424                                 pci_write_config_dword(bp->pdev, off, 0);
13425                 }
13426                 pci_restore_state(pdev);
13427                 pci_save_state(pdev);
13428
13429                 err = bnxt_hwrm_func_reset(bp);
13430                 if (!err)
13431                         result = PCI_ERS_RESULT_RECOVERED;
13432         }
13433
13434         rtnl_unlock();
13435
13436         return result;
13437 }
13438
13439 /**
13440  * bnxt_io_resume - called when traffic can start flowing again.
13441  * @pdev: Pointer to PCI device
13442  *
13443  * This callback is called when the error recovery driver tells
13444  * us that its OK to resume normal operation.
13445  */
13446 static void bnxt_io_resume(struct pci_dev *pdev)
13447 {
13448         struct net_device *netdev = pci_get_drvdata(pdev);
13449         struct bnxt *bp = netdev_priv(netdev);
13450         int err;
13451
13452         netdev_info(bp->dev, "PCI Slot Resume\n");
13453         rtnl_lock();
13454
13455         err = bnxt_hwrm_func_qcaps(bp);
13456         if (!err && netif_running(netdev))
13457                 err = bnxt_open(netdev);
13458
13459         bnxt_ulp_start(bp, err);
13460         if (!err) {
13461                 bnxt_reenable_sriov(bp);
13462                 netif_device_attach(netdev);
13463         }
13464
13465         rtnl_unlock();
13466 }
13467
13468 static const struct pci_error_handlers bnxt_err_handler = {
13469         .error_detected = bnxt_io_error_detected,
13470         .slot_reset     = bnxt_io_slot_reset,
13471         .resume         = bnxt_io_resume
13472 };
13473
13474 static struct pci_driver bnxt_pci_driver = {
13475         .name           = DRV_MODULE_NAME,
13476         .id_table       = bnxt_pci_tbl,
13477         .probe          = bnxt_init_one,
13478         .remove         = bnxt_remove_one,
13479         .shutdown       = bnxt_shutdown,
13480         .driver.pm      = BNXT_PM_OPS,
13481         .err_handler    = &bnxt_err_handler,
13482 #if defined(CONFIG_BNXT_SRIOV)
13483         .sriov_configure = bnxt_sriov_configure,
13484 #endif
13485 };
13486
13487 static int __init bnxt_init(void)
13488 {
13489         bnxt_debug_init();
13490         return pci_register_driver(&bnxt_pci_driver);
13491 }
13492
13493 static void __exit bnxt_exit(void)
13494 {
13495         pci_unregister_driver(&bnxt_pci_driver);
13496         if (bnxt_pf_wq)
13497                 destroy_workqueue(bnxt_pf_wq);
13498         bnxt_debug_exit();
13499 }
13500
13501 module_init(bnxt_init);
13502 module_exit(bnxt_exit);