drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static const struct pci_device_id be_dev_ids[] = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "ERX2 ",
  85         "SPARE ",
  86         "JTAG ",
  87         "MPU_INTPEND "
  88 };
  89
  90 /* UE Status High CSR */
  91 static const char * const ue_status_hi_desc[] = {
  92         "LPCMEMHOST",
  93         "MGMT_MAC",
  94         "PCS0ONLINE",
  95         "MPU_IRAM",
  96         "PCS1ONLINE",
  97         "PCTL0",
  98         "PCTL1",
  99         "PMEM",
 100         "RR",
 101         "TXPB",
 102         "RXPP",
 103         "XAUI",
 104         "TXP",
 105         "ARM",
 106         "IPC",
 107         "HOST2",
 108         "HOST3",
 109         "HOST4",
 110         "HOST5",
 111         "HOST6",
 112         "HOST7",
 113         "ECRC",
 114         "Poison TLP",
 115         "NETC",
 116         "PERIPH",
 117         "LLTXULP",
 118         "D2P",
 119         "RCON",
 120         "LDMA",
 121         "LLTXP",
 122         "LLTXPB",
 123         "Unknown"
 124 };
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129
 130         if (mem->va) {
 131                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 132                                   mem->dma);
 133                 mem->va = NULL;
 134         }
 135 }
 136
 137 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 138                           u16 len, u16 entry_size)
 139 {
 140         struct be_dma_mem *mem = &q->dma_mem;
 141
 142         memset(q, 0, sizeof(*q));
 143         q->len = len;
 144         q->entry_size = entry_size;
 145         mem->size = len * entry_size;
 146         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 147                                       GFP_KERNEL);
 148         if (!mem->va)
 149                 return -ENOMEM;
 150         return 0;
 151 }
 152
 153 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 154 {
 155         u32 reg, enabled;
 156
 157         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 158                               &reg);
 159         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 160
 161         if (!enabled && enable)
 162                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 163         else if (enabled && !enable)
 164                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 165         else
 166                 return;
 167
 168         pci_write_config_dword(adapter->pdev,
 169                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 170 }
 171
 172 static void be_intr_set(struct be_adapter *adapter, bool enable)
 173 {
 174         int status = 0;
 175
 176         /* On lancer interrupts can't be controlled via this register */
 177         if (lancer_chip(adapter))
 178                 return;
 179
 180         if (adapter->eeh_error)
 181                 return;
 182
 183         status = be_cmd_intr_set(adapter, enable);
 184         if (status)
 185                 be_reg_intr_set(adapter, enable);
 186 }
 187
 188 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 189 {
 190         u32 val = 0;
 191
 192         val |= qid & DB_RQ_RING_ID_MASK;
 193         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 194
 195         wmb();
 196         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 197 }
 198
 199 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 200                           u16 posted)
 201 {
 202         u32 val = 0;
 203
 204         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 205         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 206
 207         wmb();
 208         iowrite32(val, adapter->db + txo->db_offset);
 209 }
 210
 211 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 212                          bool arm, bool clear_int, u16 num_popped)
 213 {
 214         u32 val = 0;
 215
 216         val |= qid & DB_EQ_RING_ID_MASK;
 217         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 218
 219         if (adapter->eeh_error)
 220                 return;
 221
 222         if (arm)
 223                 val |= 1 << DB_EQ_REARM_SHIFT;
 224         if (clear_int)
 225                 val |= 1 << DB_EQ_CLR_SHIFT;
 226         val |= 1 << DB_EQ_EVNT_SHIFT;
 227         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 228         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 229 }
 230
 231 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 232 {
 233         u32 val = 0;
 234
 235         val |= qid & DB_CQ_RING_ID_MASK;
 236         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 237                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 238
 239         if (adapter->eeh_error)
 240                 return;
 241
 242         if (arm)
 243                 val |= 1 << DB_CQ_REARM_SHIFT;
 244         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 245         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 246 }
 247
 248 static int be_mac_addr_set(struct net_device *netdev, void *p)
 249 {
 250         struct be_adapter *adapter = netdev_priv(netdev);
 251         struct device *dev = &adapter->pdev->dev;
 252         struct sockaddr *addr = p;
 253         int status;
 254         u8 mac[ETH_ALEN];
 255         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 256
 257         if (!is_valid_ether_addr(addr->sa_data))
 258                 return -EADDRNOTAVAIL;
 259
 260         /* Proceed further only if, User provided MAC is different
 261          * from active MAC
 262          */
 263         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 264                 return 0;
 265
 266         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 267          * privilege or if PF did not provision the new MAC address.
 268          * On BE3, this cmd will always fail if the VF doesn't have the
 269          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 270          * the MAC for the VF.
 271          */
 272         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 273                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 274         if (!status) {
 275                 curr_pmac_id = adapter->pmac_id[0];
 276
 277                 /* Delete the old programmed MAC. This call may fail if the
 278                  * old MAC was already deleted by the PF driver.
 279                  */
 280                 if (adapter->pmac_id[0] != old_pmac_id)
 281                         be_cmd_pmac_del(adapter, adapter->if_handle,
 282                                         old_pmac_id, 0);
 283         }
 284
 285         /* Decide if the new MAC is successfully activated only after
 286          * querying the FW
 287          */
 288         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 289                                        adapter->if_handle, true, 0);
 290         if (status)
 291                 goto err;
 292
 293         /* The MAC change did not happen, either due to lack of privilege
 294          * or PF didn't pre-provision.
 295          */
 296         if (!ether_addr_equal(addr->sa_data, mac)) {
 297                 status = -EPERM;
 298                 goto err;
 299         }
 300
 301         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 302         dev_info(dev, "MAC address changed to %pM\n", mac);
 303         return 0;
 304 err:
 305         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 306         return status;
 307 }
 308
 309 /* BE2 supports only v0 cmd */
 310 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 311 {
 312         if (BE2_chip(adapter)) {
 313                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 314
 315                 return &cmd->hw_stats;
 316         } else if (BE3_chip(adapter)) {
 317                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 318
 319                 return &cmd->hw_stats;
 320         } else {
 321                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 322
 323                 return &cmd->hw_stats;
 324         }
 325 }
 326
 327 /* BE2 supports only v0 cmd */
 328 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 329 {
 330         if (BE2_chip(adapter)) {
 331                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 332
 333                 return &hw_stats->erx;
 334         } else if (BE3_chip(adapter)) {
 335                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 336
 337                 return &hw_stats->erx;
 338         } else {
 339                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 340
 341                 return &hw_stats->erx;
 342         }
 343 }
 344
 345 static void populate_be_v0_stats(struct be_adapter *adapter)
 346 {
 347         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 348         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 349         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 350         struct be_port_rxf_stats_v0 *port_stats =
 351                                         &rxf_stats->port[adapter->port_num];
 352         struct be_drv_stats *drvs = &adapter->drv_stats;
 353
 354         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 355         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 356         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 357         drvs->rx_control_frames = port_stats->rx_control_frames;
 358         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 359         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 360         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 361         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 362         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 363         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 364         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 365         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 366         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 367         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 368         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 369         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 370         drvs->rx_dropped_header_too_small =
 371                 port_stats->rx_dropped_header_too_small;
 372         drvs->rx_address_filtered =
 373                                         port_stats->rx_address_filtered +
 374                                         port_stats->rx_vlan_filtered;
 375         drvs->rx_alignment_symbol_errors =
 376                 port_stats->rx_alignment_symbol_errors;
 377
 378         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 379         drvs->tx_controlframes = port_stats->tx_controlframes;
 380
 381         if (adapter->port_num)
 382                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 383         else
 384                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 385         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 386         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 387         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 388         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 389         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 390         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 391         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 392 }
 393
 394 static void populate_be_v1_stats(struct be_adapter *adapter)
 395 {
 396         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 397         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 398         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 399         struct be_port_rxf_stats_v1 *port_stats =
 400                                         &rxf_stats->port[adapter->port_num];
 401         struct be_drv_stats *drvs = &adapter->drv_stats;
 402
 403         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 404         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 405         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 406         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 407         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 408         drvs->rx_control_frames = port_stats->rx_control_frames;
 409         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 410         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 411         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 412         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 413         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 414         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 415         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 416         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 417         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 418         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 419         drvs->rx_dropped_header_too_small =
 420                 port_stats->rx_dropped_header_too_small;
 421         drvs->rx_input_fifo_overflow_drop =
 422                 port_stats->rx_input_fifo_overflow_drop;
 423         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 424         drvs->rx_alignment_symbol_errors =
 425                 port_stats->rx_alignment_symbol_errors;
 426         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 427         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 428         drvs->tx_controlframes = port_stats->tx_controlframes;
 429         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 430         drvs->jabber_events = port_stats->jabber_events;
 431         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 432         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 433         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 434         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 435         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 436         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 437         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 438 }
 439
 440 static void populate_be_v2_stats(struct be_adapter *adapter)
 441 {
 442         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 443         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 444         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 445         struct be_port_rxf_stats_v2 *port_stats =
 446                                         &rxf_stats->port[adapter->port_num];
 447         struct be_drv_stats *drvs = &adapter->drv_stats;
 448
 449         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 450         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 451         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 452         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 453         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 454         drvs->rx_control_frames = port_stats->rx_control_frames;
 455         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 456         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 457         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 458         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 459         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 460         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 461         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 462         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 463         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 464         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 465         drvs->rx_dropped_header_too_small =
 466                 port_stats->rx_dropped_header_too_small;
 467         drvs->rx_input_fifo_overflow_drop =
 468                 port_stats->rx_input_fifo_overflow_drop;
 469         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 470         drvs->rx_alignment_symbol_errors =
 471                 port_stats->rx_alignment_symbol_errors;
 472         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 473         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 474         drvs->tx_controlframes = port_stats->tx_controlframes;
 475         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 476         drvs->jabber_events = port_stats->jabber_events;
 477         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 478         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 479         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 480         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 481         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 482         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 483         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 484         if (be_roce_supported(adapter)) {
 485                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 486                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 487                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 488                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 489                 drvs->roce_drops_payload_len =
 490                         port_stats->roce_drops_payload_len;
 491         }
 492 }
 493
 494 static void populate_lancer_stats(struct be_adapter *adapter)
 495 {
 496
 497         struct be_drv_stats *drvs = &adapter->drv_stats;
 498         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 499
 500         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 501         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 502         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 503         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 504         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 505         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 506         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 507         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 508         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 509         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 510         drvs->rx_dropped_tcp_length =
 511                                 pport_stats->rx_dropped_invalid_tcp_length;
 512         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 513         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 514         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 515         drvs->rx_dropped_header_too_small =
 516                                 pport_stats->rx_dropped_header_too_small;
 517         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 518         drvs->rx_address_filtered =
 519                                         pport_stats->rx_address_filtered +
 520                                         pport_stats->rx_vlan_filtered;
 521         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 522         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 523         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 524         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 525         drvs->jabber_events = pport_stats->rx_jabbers;
 526         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 527         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 528         drvs->rx_drops_too_many_frags =
 529                                 pport_stats->rx_drops_too_many_frags_lo;
 530 }
 531
 532 static void accumulate_16bit_val(u32 *acc, u16 val)
 533 {
 534 #define lo(x)                   (x & 0xFFFF)
 535 #define hi(x)                   (x & 0xFFFF0000)
 536         bool wrapped = val < lo(*acc);
 537         u32 newacc = hi(*acc) + val;
 538
 539         if (wrapped)
 540                 newacc += 65536;
 541         ACCESS_ONCE(*acc) = newacc;
 542 }
 543
 544 static void populate_erx_stats(struct be_adapter *adapter,
 545                                struct be_rx_obj *rxo, u32 erx_stat)
 546 {
 547         if (!BEx_chip(adapter))
 548                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 549         else
 550                 /* below erx HW counter can actually wrap around after
 551                  * 65535. Driver accumulates a 32-bit value
 552                  */
 553                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 554                                      (u16)erx_stat);
 555 }
 556
 557 void be_parse_stats(struct be_adapter *adapter)
 558 {
 559         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 560         struct be_rx_obj *rxo;
 561         int i;
 562         u32 erx_stat;
 563
 564         if (lancer_chip(adapter)) {
 565                 populate_lancer_stats(adapter);
 566         } else {
 567                 if (BE2_chip(adapter))
 568                         populate_be_v0_stats(adapter);
 569                 else if (BE3_chip(adapter))
 570                         /* for BE3 */
 571                         populate_be_v1_stats(adapter);
 572                 else
 573                         populate_be_v2_stats(adapter);
 574
 575                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 576                 for_all_rx_queues(adapter, rxo, i) {
 577                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 578                         populate_erx_stats(adapter, rxo, erx_stat);
 579                 }
 580         }
 581 }
 582
 583 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 584                                                 struct rtnl_link_stats64 *stats)
 585 {
 586         struct be_adapter *adapter = netdev_priv(netdev);
 587         struct be_drv_stats *drvs = &adapter->drv_stats;
 588         struct be_rx_obj *rxo;
 589         struct be_tx_obj *txo;
 590         u64 pkts, bytes;
 591         unsigned int start;
 592         int i;
 593
 594         for_all_rx_queues(adapter, rxo, i) {
 595                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 596
 597                 do {
 598                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 599                         pkts = rx_stats(rxo)->rx_pkts;
 600                         bytes = rx_stats(rxo)->rx_bytes;
 601                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 602                 stats->rx_packets += pkts;
 603                 stats->rx_bytes += bytes;
 604                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 605                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 606                                         rx_stats(rxo)->rx_drops_no_frags;
 607         }
 608
 609         for_all_tx_queues(adapter, txo, i) {
 610                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 611
 612                 do {
 613                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 614                         pkts = tx_stats(txo)->tx_pkts;
 615                         bytes = tx_stats(txo)->tx_bytes;
 616                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 617                 stats->tx_packets += pkts;
 618                 stats->tx_bytes += bytes;
 619         }
 620
 621         /* bad pkts received */
 622         stats->rx_errors = drvs->rx_crc_errors +
 623                 drvs->rx_alignment_symbol_errors +
 624                 drvs->rx_in_range_errors +
 625                 drvs->rx_out_range_errors +
 626                 drvs->rx_frame_too_long +
 627                 drvs->rx_dropped_too_small +
 628                 drvs->rx_dropped_too_short +
 629                 drvs->rx_dropped_header_too_small +
 630                 drvs->rx_dropped_tcp_length +
 631                 drvs->rx_dropped_runt;
 632
 633         /* detailed rx errors */
 634         stats->rx_length_errors = drvs->rx_in_range_errors +
 635                 drvs->rx_out_range_errors +
 636                 drvs->rx_frame_too_long;
 637
 638         stats->rx_crc_errors = drvs->rx_crc_errors;
 639
 640         /* frame alignment errors */
 641         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 642
 643         /* receiver fifo overrun */
 644         /* drops_no_pbuf is no per i/f, it's per BE card */
 645         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 646                                 drvs->rx_input_fifo_overflow_drop +
 647                                 drvs->rx_drops_no_pbuf;
 648         return stats;
 649 }
 650
 651 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 652 {
 653         struct net_device *netdev = adapter->netdev;
 654
 655         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 656                 netif_carrier_off(netdev);
 657                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 658         }
 659
 660         if (link_status)
 661                 netif_carrier_on(netdev);
 662         else
 663                 netif_carrier_off(netdev);
 664 }
 665
 666 static void be_tx_stats_update(struct be_tx_obj *txo,
 667                                u32 wrb_cnt, u32 copied, u32 gso_segs,
 668                                bool stopped)
 669 {
 670         struct be_tx_stats *stats = tx_stats(txo);
 671
 672         u64_stats_update_begin(&stats->sync);
 673         stats->tx_reqs++;
 674         stats->tx_wrbs += wrb_cnt;
 675         stats->tx_bytes += copied;
 676         stats->tx_pkts += (gso_segs ? gso_segs : 1);
 677         if (stopped)
 678                 stats->tx_stops++;
 679         u64_stats_update_end(&stats->sync);
 680 }
 681
 682 /* Determine number of WRB entries needed to xmit data in an skb */
 683 static u32 wrb_cnt_for_skb(struct be_adapter *adapter, struct sk_buff *skb,
 684                            bool *dummy)
 685 {
 686         int cnt = (skb->len > skb->data_len);
 687
 688         cnt += skb_shinfo(skb)->nr_frags;
 689
 690         /* to account for hdr wrb */
 691         cnt++;
 692         if (lancer_chip(adapter) || !(cnt & 1)) {
 693                 *dummy = false;
 694         } else {
 695                 /* add a dummy to make it an even num */
 696                 cnt++;
 697                 *dummy = true;
 698         }
 699         BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT);
 700         return cnt;
 701 }
 702
 703 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 704 {
 705         wrb->frag_pa_hi = upper_32_bits(addr);
 706         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 707         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 708         wrb->rsvd0 = 0;
 709 }
 710
 711 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 712                                      struct sk_buff *skb)
 713 {
 714         u8 vlan_prio;
 715         u16 vlan_tag;
 716
 717         vlan_tag = vlan_tx_tag_get(skb);
 718         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 719         /* If vlan priority provided by OS is NOT in available bmap */
 720         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 721                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 722                                 adapter->recommended_prio;
 723
 724         return vlan_tag;
 725 }
 726
 727 /* Used only for IP tunnel packets */
 728 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 729 {
 730         return (inner_ip_hdr(skb)->version == 4) ?
 731                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 732 }
 733
 734 static u16 skb_ip_proto(struct sk_buff *skb)
 735 {
 736         return (ip_hdr(skb)->version == 4) ?
 737                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 738 }
 739
 740 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 741                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 742                          bool skip_hw_vlan)
 743 {
 744         u16 vlan_tag, proto;
 745
 746         memset(hdr, 0, sizeof(*hdr));
 747
 748         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 749
 750         if (skb_is_gso(skb)) {
 751                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 752                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 753                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 754                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 755         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 756                 if (skb->encapsulation) {
 757                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 758                         proto = skb_inner_ip_proto(skb);
 759                 } else {
 760                         proto = skb_ip_proto(skb);
 761                 }
 762                 if (proto == IPPROTO_TCP)
 763                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 764                 else if (proto == IPPROTO_UDP)
 765                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 766         }
 767
 768         if (vlan_tx_tag_present(skb)) {
 769                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 770                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 771                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 772         }
 773
 774         /* To skip HW VLAN tagging: evt = 1, compl = 0 */
 775         SET_TX_WRB_HDR_BITS(complete, hdr, !skip_hw_vlan);
 776         SET_TX_WRB_HDR_BITS(event, hdr, 1);
 777         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 778         SET_TX_WRB_HDR_BITS(len, hdr, len);
 779 }
 780
 781 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 782                           bool unmap_single)
 783 {
 784         dma_addr_t dma;
 785
 786         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 787
 788         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 789         if (wrb->frag_len) {
 790                 if (unmap_single)
 791                         dma_unmap_single(dev, dma, wrb->frag_len,
 792                                          DMA_TO_DEVICE);
 793                 else
 794                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 795         }
 796 }
 797
 798 static int make_tx_wrbs(struct be_adapter *adapter, struct be_queue_info *txq,
 799                         struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb,
 800                         bool skip_hw_vlan)
 801 {
 802         dma_addr_t busaddr;
 803         int i, copied = 0;
 804         struct device *dev = &adapter->pdev->dev;
 805         struct sk_buff *first_skb = skb;
 806         struct be_eth_wrb *wrb;
 807         struct be_eth_hdr_wrb *hdr;
 808         bool map_single = false;
 809         u16 map_head;
 810
 811         hdr = queue_head_node(txq);
 812         queue_head_inc(txq);
 813         map_head = txq->head;
 814
 815         if (skb->len > skb->data_len) {
 816                 int len = skb_headlen(skb);
 817
 818                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 819                 if (dma_mapping_error(dev, busaddr))
 820                         goto dma_err;
 821                 map_single = true;
 822                 wrb = queue_head_node(txq);
 823                 wrb_fill(wrb, busaddr, len);
 824                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 825                 queue_head_inc(txq);
 826                 copied += len;
 827         }
 828
 829         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 830                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 831
 832                 busaddr = skb_frag_dma_map(dev, frag, 0,
 833                                            skb_frag_size(frag), DMA_TO_DEVICE);
 834                 if (dma_mapping_error(dev, busaddr))
 835                         goto dma_err;
 836                 wrb = queue_head_node(txq);
 837                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 838                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 839                 queue_head_inc(txq);
 840                 copied += skb_frag_size(frag);
 841         }
 842
 843         if (dummy_wrb) {
 844                 wrb = queue_head_node(txq);
 845                 wrb_fill(wrb, 0, 0);
 846                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 847                 queue_head_inc(txq);
 848         }
 849
 850         wrb_fill_hdr(adapter, hdr, first_skb, wrb_cnt, copied, skip_hw_vlan);
 851         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 852
 853         return copied;
 854 dma_err:
 855         txq->head = map_head;
 856         while (copied) {
 857                 wrb = queue_head_node(txq);
 858                 unmap_tx_frag(dev, wrb, map_single);
 859                 map_single = false;
 860                 copied -= wrb->frag_len;
 861                 adapter->drv_stats.dma_map_errors++;
 862                 queue_head_inc(txq);
 863         }
 864         return 0;
 865 }
 866
 867 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 868                                              struct sk_buff *skb,
 869                                              bool *skip_hw_vlan)
 870 {
 871         u16 vlan_tag = 0;
 872
 873         skb = skb_share_check(skb, GFP_ATOMIC);
 874         if (unlikely(!skb))
 875                 return skb;
 876
 877         if (vlan_tx_tag_present(skb))
 878                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 879
 880         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 881                 if (!vlan_tag)
 882                         vlan_tag = adapter->pvid;
 883                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 884                  * skip VLAN insertion
 885                  */
 886                 if (skip_hw_vlan)
 887                         *skip_hw_vlan = true;
 888         }
 889
 890         if (vlan_tag) {
 891                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 892                 if (unlikely(!skb))
 893                         return skb;
 894                 skb->vlan_tci = 0;
 895         }
 896
 897         /* Insert the outer VLAN, if any */
 898         if (adapter->qnq_vid) {
 899                 vlan_tag = adapter->qnq_vid;
 900                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 901                 if (unlikely(!skb))
 902                         return skb;
 903                 if (skip_hw_vlan)
 904                         *skip_hw_vlan = true;
 905         }
 906
 907         return skb;
 908 }
 909
 910 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 911 {
 912         struct ethhdr *eh = (struct ethhdr *)skb->data;
 913         u16 offset = ETH_HLEN;
 914
 915         if (eh->h_proto == htons(ETH_P_IPV6)) {
 916                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 917
 918                 offset += sizeof(struct ipv6hdr);
 919                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 920                     ip6h->nexthdr != NEXTHDR_UDP) {
 921                         struct ipv6_opt_hdr *ehdr =
 922                                 (struct ipv6_opt_hdr *) (skb->data + offset);
 923
 924                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 925                         if (ehdr->hdrlen == 0xff)
 926                                 return true;
 927                 }
 928         }
 929         return false;
 930 }
 931
 932 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 933 {
 934         return vlan_tx_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 935 }
 936
 937 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 938 {
 939         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 940 }
 941
 942 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 943                                                   struct sk_buff *skb,
 944                                                   bool *skip_hw_vlan)
 945 {
 946         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 947         unsigned int eth_hdr_len;
 948         struct iphdr *ip;
 949
 950         /* For padded packets, BE HW modifies tot_len field in IP header
 951          * incorrecly when VLAN tag is inserted by HW.
 952          * For padded packets, Lancer computes incorrect checksum.
 953          */
 954         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 955                                                 VLAN_ETH_HLEN : ETH_HLEN;
 956         if (skb->len <= 60 &&
 957             (lancer_chip(adapter) || vlan_tx_tag_present(skb)) &&
 958             is_ipv4_pkt(skb)) {
 959                 ip = (struct iphdr *)ip_hdr(skb);
 960                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 961         }
 962
 963         /* If vlan tag is already inlined in the packet, skip HW VLAN
 964          * tagging in pvid-tagging mode
 965          */
 966         if (be_pvid_tagging_enabled(adapter) &&
 967             veh->h_vlan_proto == htons(ETH_P_8021Q))
 968                 *skip_hw_vlan = true;
 969
 970         /* HW has a bug wherein it will calculate CSUM for VLAN
 971          * pkts even though it is disabled.
 972          * Manually insert VLAN in pkt.
 973          */
 974         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 975             vlan_tx_tag_present(skb)) {
 976                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 977                 if (unlikely(!skb))
 978                         goto err;
 979         }
 980
 981         /* HW may lockup when VLAN HW tagging is requested on
 982          * certain ipv6 packets. Drop such pkts if the HW workaround to
 983          * skip HW tagging is not enabled by FW.
 984          */
 985         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 986             (adapter->pvid || adapter->qnq_vid) &&
 987             !qnq_async_evt_rcvd(adapter)))
 988                 goto tx_drop;
 989
 990         /* Manual VLAN tag insertion to prevent:
 991          * ASIC lockup when the ASIC inserts VLAN tag into
 992          * certain ipv6 packets. Insert VLAN tags in driver,
 993          * and set event, completion, vlan bits accordingly
 994          * in the Tx WRB.
 995          */
 996         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 997             be_vlan_tag_tx_chk(adapter, skb)) {
 998                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 999                 if (unlikely(!skb))
1000                         goto err;
1001         }
1002
1003         return skb;
1004 tx_drop:
1005         dev_kfree_skb_any(skb);
1006 err:
1007         return NULL;
1008 }
1009
1010 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1011                                            struct sk_buff *skb,
1012                                            bool *skip_hw_vlan)
1013 {
1014         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1015          * less may cause a transmit stall on that port. So the work-around is
1016          * to pad short packets (<= 32 bytes) to a 36-byte length.
1017          */
1018         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1019                 if (skb_padto(skb, 36))
1020                         return NULL;
1021                 skb->len = 36;
1022         }
1023
1024         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1025                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1026                 if (!skb)
1027                         return NULL;
1028         }
1029
1030         return skb;
1031 }
1032
1033 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1034 {
1035         struct be_adapter *adapter = netdev_priv(netdev);
1036         struct be_tx_obj *txo = &adapter->tx_obj[skb_get_queue_mapping(skb)];
1037         struct be_queue_info *txq = &txo->q;
1038         bool dummy_wrb, stopped = false;
1039         u32 wrb_cnt = 0, copied = 0;
1040         bool skip_hw_vlan = false;
1041         u32 start = txq->head;
1042
1043         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1044         if (!skb) {
1045                 tx_stats(txo)->tx_drv_drops++;
1046                 return NETDEV_TX_OK;
1047         }
1048
1049         wrb_cnt = wrb_cnt_for_skb(adapter, skb, &dummy_wrb);
1050
1051         copied = make_tx_wrbs(adapter, txq, skb, wrb_cnt, dummy_wrb,
1052                               skip_hw_vlan);
1053         if (copied) {
1054                 int gso_segs = skb_shinfo(skb)->gso_segs;
1055
1056                 /* record the sent skb in the sent_skb table */
1057                 BUG_ON(txo->sent_skb_list[start]);
1058                 txo->sent_skb_list[start] = skb;
1059
1060                 /* Ensure txq has space for the next skb; Else stop the queue
1061                  * *BEFORE* ringing the tx doorbell, so that we serialze the
1062                  * tx compls of the current transmit which'll wake up the queue
1063                  */
1064                 atomic_add(wrb_cnt, &txq->used);
1065                 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >=
1066                                                                 txq->len) {
1067                         netif_stop_subqueue(netdev, skb_get_queue_mapping(skb));
1068                         stopped = true;
1069                 }
1070
1071                 be_txq_notify(adapter, txo, wrb_cnt);
1072
1073                 be_tx_stats_update(txo, wrb_cnt, copied, gso_segs, stopped);
1074         } else {
1075                 txq->head = start;
1076                 tx_stats(txo)->tx_drv_drops++;
1077                 dev_kfree_skb_any(skb);
1078         }
1079         return NETDEV_TX_OK;
1080 }
1081
1082 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1083 {
1084         struct be_adapter *adapter = netdev_priv(netdev);
1085         struct device *dev = &adapter->pdev->dev;
1086
1087         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1088                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1089                          BE_MIN_MTU, BE_MAX_MTU);
1090                 return -EINVAL;
1091         }
1092
1093         dev_info(dev, "MTU changed from %d to %d bytes\n",
1094                  netdev->mtu, new_mtu);
1095         netdev->mtu = new_mtu;
1096         return 0;
1097 }
1098
1099 /*
1100  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1101  * If the user configures more, place BE in vlan promiscuous mode.
1102  */
1103 static int be_vid_config(struct be_adapter *adapter)
1104 {
1105         struct device *dev = &adapter->pdev->dev;
1106         u16 vids[BE_NUM_VLANS_SUPPORTED];
1107         u16 num = 0, i = 0;
1108         int status = 0;
1109
1110         /* No need to further configure vids if in promiscuous mode */
1111         if (adapter->promiscuous)
1112                 return 0;
1113
1114         if (adapter->vlans_added > be_max_vlans(adapter))
1115                 goto set_vlan_promisc;
1116
1117         /* Construct VLAN Table to give to HW */
1118         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1119                 vids[num++] = cpu_to_le16(i);
1120
1121         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1122         if (status) {
1123                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1124                 if (addl_status(status) ==
1125                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1126                         goto set_vlan_promisc;
1127                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1128         } else {
1129                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1130                         /* hw VLAN filtering re-enabled. */
1131                         status = be_cmd_rx_filter(adapter,
1132                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1133                         if (!status) {
1134                                 dev_info(dev,
1135                                          "Disabling VLAN Promiscuous mode\n");
1136                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1137                         }
1138                 }
1139         }
1140
1141         return status;
1142
1143 set_vlan_promisc:
1144         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1145                 return 0;
1146
1147         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1148         if (!status) {
1149                 dev_info(dev, "Enable VLAN Promiscuous mode\n");
1150                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1151         } else
1152                 dev_err(dev, "Failed to enable VLAN Promiscuous mode\n");
1153         return status;
1154 }
1155
1156 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1157 {
1158         struct be_adapter *adapter = netdev_priv(netdev);
1159         int status = 0;
1160
1161         /* Packets with VID 0 are always received by Lancer by default */
1162         if (lancer_chip(adapter) && vid == 0)
1163                 return status;
1164
1165         if (test_bit(vid, adapter->vids))
1166                 return status;
1167
1168         set_bit(vid, adapter->vids);
1169         adapter->vlans_added++;
1170
1171         status = be_vid_config(adapter);
1172         if (status) {
1173                 adapter->vlans_added--;
1174                 clear_bit(vid, adapter->vids);
1175         }
1176
1177         return status;
1178 }
1179
1180 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1181 {
1182         struct be_adapter *adapter = netdev_priv(netdev);
1183
1184         /* Packets with VID 0 are always received by Lancer by default */
1185         if (lancer_chip(adapter) && vid == 0)
1186                 return 0;
1187
1188         clear_bit(vid, adapter->vids);
1189         adapter->vlans_added--;
1190
1191         return be_vid_config(adapter);
1192 }
1193
1194 static void be_clear_promisc(struct be_adapter *adapter)
1195 {
1196         adapter->promiscuous = false;
1197         adapter->flags &= ~(BE_FLAGS_VLAN_PROMISC | BE_FLAGS_MCAST_PROMISC);
1198
1199         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1200 }
1201
1202 static void be_set_rx_mode(struct net_device *netdev)
1203 {
1204         struct be_adapter *adapter = netdev_priv(netdev);
1205         int status;
1206
1207         if (netdev->flags & IFF_PROMISC) {
1208                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1209                 adapter->promiscuous = true;
1210                 goto done;
1211         }
1212
1213         /* BE was previously in promiscuous mode; disable it */
1214         if (adapter->promiscuous) {
1215                 be_clear_promisc(adapter);
1216                 if (adapter->vlans_added)
1217                         be_vid_config(adapter);
1218         }
1219
1220         /* Enable multicast promisc if num configured exceeds what we support */
1221         if (netdev->flags & IFF_ALLMULTI ||
1222             netdev_mc_count(netdev) > be_max_mc(adapter))
1223                 goto set_mcast_promisc;
1224
1225         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1226                 struct netdev_hw_addr *ha;
1227                 int i = 1; /* First slot is claimed by the Primary MAC */
1228
1229                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1230                         be_cmd_pmac_del(adapter, adapter->if_handle,
1231                                         adapter->pmac_id[i], 0);
1232                 }
1233
1234                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1235                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1236                         adapter->promiscuous = true;
1237                         goto done;
1238                 }
1239
1240                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1241                         adapter->uc_macs++; /* First slot is for Primary MAC */
1242                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1243                                         adapter->if_handle,
1244                                         &adapter->pmac_id[adapter->uc_macs], 0);
1245                 }
1246         }
1247
1248         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1249         if (!status) {
1250                 if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1251                         adapter->flags &= ~BE_FLAGS_MCAST_PROMISC;
1252                 goto done;
1253         }
1254
1255 set_mcast_promisc:
1256         if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1257                 return;
1258
1259         /* Set to MCAST promisc mode if setting MULTICAST address fails
1260          * or if num configured exceeds what we support
1261          */
1262         status = be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1263         if (!status)
1264                 adapter->flags |= BE_FLAGS_MCAST_PROMISC;
1265 done:
1266         return;
1267 }
1268
1269 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1270 {
1271         struct be_adapter *adapter = netdev_priv(netdev);
1272         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1273         int status;
1274
1275         if (!sriov_enabled(adapter))
1276                 return -EPERM;
1277
1278         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1279                 return -EINVAL;
1280
1281         /* Proceed further only if user provided MAC is different
1282          * from active MAC
1283          */
1284         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1285                 return 0;
1286
1287         if (BEx_chip(adapter)) {
1288                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1289                                 vf + 1);
1290
1291                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1292                                          &vf_cfg->pmac_id, vf + 1);
1293         } else {
1294                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1295                                         vf + 1);
1296         }
1297
1298         if (status) {
1299                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1300                         mac, vf, status);
1301                 return be_cmd_status(status);
1302         }
1303
1304         ether_addr_copy(vf_cfg->mac_addr, mac);
1305
1306         return 0;
1307 }
1308
1309 static int be_get_vf_config(struct net_device *netdev, int vf,
1310                             struct ifla_vf_info *vi)
1311 {
1312         struct be_adapter *adapter = netdev_priv(netdev);
1313         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1314
1315         if (!sriov_enabled(adapter))
1316                 return -EPERM;
1317
1318         if (vf >= adapter->num_vfs)
1319                 return -EINVAL;
1320
1321         vi->vf = vf;
1322         vi->max_tx_rate = vf_cfg->tx_rate;
1323         vi->min_tx_rate = 0;
1324         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1325         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1326         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1327         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1328
1329         return 0;
1330 }
1331
1332 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1333 {
1334         struct be_adapter *adapter = netdev_priv(netdev);
1335         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1336         int status = 0;
1337
1338         if (!sriov_enabled(adapter))
1339                 return -EPERM;
1340
1341         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1342                 return -EINVAL;
1343
1344         if (vlan || qos) {
1345                 vlan |= qos << VLAN_PRIO_SHIFT;
1346                 if (vf_cfg->vlan_tag != vlan)
1347                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1348                                                        vf_cfg->if_handle, 0);
1349         } else {
1350                 /* Reset Transparent Vlan Tagging. */
1351                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1352                                                vf + 1, vf_cfg->if_handle, 0);
1353         }
1354
1355         if (status) {
1356                 dev_err(&adapter->pdev->dev,
1357                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1358                         vf, status);
1359                 return be_cmd_status(status);
1360         }
1361
1362         vf_cfg->vlan_tag = vlan;
1363
1364         return 0;
1365 }
1366
1367 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1368                              int min_tx_rate, int max_tx_rate)
1369 {
1370         struct be_adapter *adapter = netdev_priv(netdev);
1371         struct device *dev = &adapter->pdev->dev;
1372         int percent_rate, status = 0;
1373         u16 link_speed = 0;
1374         u8 link_status;
1375
1376         if (!sriov_enabled(adapter))
1377                 return -EPERM;
1378
1379         if (vf >= adapter->num_vfs)
1380                 return -EINVAL;
1381
1382         if (min_tx_rate)
1383                 return -EINVAL;
1384
1385         if (!max_tx_rate)
1386                 goto config_qos;
1387
1388         status = be_cmd_link_status_query(adapter, &link_speed,
1389                                           &link_status, 0);
1390         if (status)
1391                 goto err;
1392
1393         if (!link_status) {
1394                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1395                 status = -ENETDOWN;
1396                 goto err;
1397         }
1398
1399         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1400                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1401                         link_speed);
1402                 status = -EINVAL;
1403                 goto err;
1404         }
1405
1406         /* On Skyhawk the QOS setting must be done only as a % value */
1407         percent_rate = link_speed / 100;
1408         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1409                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1410                         percent_rate);
1411                 status = -EINVAL;
1412                 goto err;
1413         }
1414
1415 config_qos:
1416         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1417         if (status)
1418                 goto err;
1419
1420         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1421         return 0;
1422
1423 err:
1424         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1425                 max_tx_rate, vf);
1426         return be_cmd_status(status);
1427 }
1428
1429 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1430                                 int link_state)
1431 {
1432         struct be_adapter *adapter = netdev_priv(netdev);
1433         int status;
1434
1435         if (!sriov_enabled(adapter))
1436                 return -EPERM;
1437
1438         if (vf >= adapter->num_vfs)
1439                 return -EINVAL;
1440
1441         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1442         if (status) {
1443                 dev_err(&adapter->pdev->dev,
1444                         "Link state change on VF %d failed: %#x\n", vf, status);
1445                 return be_cmd_status(status);
1446         }
1447
1448         adapter->vf_cfg[vf].plink_tracking = link_state;
1449
1450         return 0;
1451 }
1452
1453 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1454                           ulong now)
1455 {
1456         aic->rx_pkts_prev = rx_pkts;
1457         aic->tx_reqs_prev = tx_pkts;
1458         aic->jiffies = now;
1459 }
1460
1461 static void be_eqd_update(struct be_adapter *adapter)
1462 {
1463         struct be_set_eqd set_eqd[MAX_EVT_QS];
1464         int eqd, i, num = 0, start;
1465         struct be_aic_obj *aic;
1466         struct be_eq_obj *eqo;
1467         struct be_rx_obj *rxo;
1468         struct be_tx_obj *txo;
1469         u64 rx_pkts, tx_pkts;
1470         ulong now;
1471         u32 pps, delta;
1472
1473         for_all_evt_queues(adapter, eqo, i) {
1474                 aic = &adapter->aic_obj[eqo->idx];
1475                 if (!aic->enable) {
1476                         if (aic->jiffies)
1477                                 aic->jiffies = 0;
1478                         eqd = aic->et_eqd;
1479                         goto modify_eqd;
1480                 }
1481
1482                 rxo = &adapter->rx_obj[eqo->idx];
1483                 do {
1484                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1485                         rx_pkts = rxo->stats.rx_pkts;
1486                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1487
1488                 txo = &adapter->tx_obj[eqo->idx];
1489                 do {
1490                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1491                         tx_pkts = txo->stats.tx_reqs;
1492                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1493
1494                 /* Skip, if wrapped around or first calculation */
1495                 now = jiffies;
1496                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1497                     rx_pkts < aic->rx_pkts_prev ||
1498                     tx_pkts < aic->tx_reqs_prev) {
1499                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1500                         continue;
1501                 }
1502
1503                 delta = jiffies_to_msecs(now - aic->jiffies);
1504                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1505                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1506                 eqd = (pps / 15000) << 2;
1507
1508                 if (eqd < 8)
1509                         eqd = 0;
1510                 eqd = min_t(u32, eqd, aic->max_eqd);
1511                 eqd = max_t(u32, eqd, aic->min_eqd);
1512
1513                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1514 modify_eqd:
1515                 if (eqd != aic->prev_eqd) {
1516                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1517                         set_eqd[num].eq_id = eqo->q.id;
1518                         aic->prev_eqd = eqd;
1519                         num++;
1520                 }
1521         }
1522
1523         if (num)
1524                 be_cmd_modify_eqd(adapter, set_eqd, num);
1525 }
1526
1527 static void be_rx_stats_update(struct be_rx_obj *rxo,
1528                                struct be_rx_compl_info *rxcp)
1529 {
1530         struct be_rx_stats *stats = rx_stats(rxo);
1531
1532         u64_stats_update_begin(&stats->sync);
1533         stats->rx_compl++;
1534         stats->rx_bytes += rxcp->pkt_size;
1535         stats->rx_pkts++;
1536         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1537                 stats->rx_mcast_pkts++;
1538         if (rxcp->err)
1539                 stats->rx_compl_err++;
1540         u64_stats_update_end(&stats->sync);
1541 }
1542
1543 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1544 {
1545         /* L4 checksum is not reliable for non TCP/UDP packets.
1546          * Also ignore ipcksm for ipv6 pkts
1547          */
1548         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1549                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1550 }
1551
1552 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1553 {
1554         struct be_adapter *adapter = rxo->adapter;
1555         struct be_rx_page_info *rx_page_info;
1556         struct be_queue_info *rxq = &rxo->q;
1557         u16 frag_idx = rxq->tail;
1558
1559         rx_page_info = &rxo->page_info_tbl[frag_idx];
1560         BUG_ON(!rx_page_info->page);
1561
1562         if (rx_page_info->last_frag) {
1563                 dma_unmap_page(&adapter->pdev->dev,
1564                                dma_unmap_addr(rx_page_info, bus),
1565                                adapter->big_page_size, DMA_FROM_DEVICE);
1566                 rx_page_info->last_frag = false;
1567         } else {
1568                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1569                                         dma_unmap_addr(rx_page_info, bus),
1570                                         rx_frag_size, DMA_FROM_DEVICE);
1571         }
1572
1573         queue_tail_inc(rxq);
1574         atomic_dec(&rxq->used);
1575         return rx_page_info;
1576 }
1577
1578 /* Throwaway the data in the Rx completion */
1579 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1580                                 struct be_rx_compl_info *rxcp)
1581 {
1582         struct be_rx_page_info *page_info;
1583         u16 i, num_rcvd = rxcp->num_rcvd;
1584
1585         for (i = 0; i < num_rcvd; i++) {
1586                 page_info = get_rx_page_info(rxo);
1587                 put_page(page_info->page);
1588                 memset(page_info, 0, sizeof(*page_info));
1589         }
1590 }
1591
1592 /*
1593  * skb_fill_rx_data forms a complete skb for an ether frame
1594  * indicated by rxcp.
1595  */
1596 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1597                              struct be_rx_compl_info *rxcp)
1598 {
1599         struct be_rx_page_info *page_info;
1600         u16 i, j;
1601         u16 hdr_len, curr_frag_len, remaining;
1602         u8 *start;
1603
1604         page_info = get_rx_page_info(rxo);
1605         start = page_address(page_info->page) + page_info->page_offset;
1606         prefetch(start);
1607
1608         /* Copy data in the first descriptor of this completion */
1609         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1610
1611         skb->len = curr_frag_len;
1612         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1613                 memcpy(skb->data, start, curr_frag_len);
1614                 /* Complete packet has now been moved to data */
1615                 put_page(page_info->page);
1616                 skb->data_len = 0;
1617                 skb->tail += curr_frag_len;
1618         } else {
1619                 hdr_len = ETH_HLEN;
1620                 memcpy(skb->data, start, hdr_len);
1621                 skb_shinfo(skb)->nr_frags = 1;
1622                 skb_frag_set_page(skb, 0, page_info->page);
1623                 skb_shinfo(skb)->frags[0].page_offset =
1624                                         page_info->page_offset + hdr_len;
1625                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1626                                   curr_frag_len - hdr_len);
1627                 skb->data_len = curr_frag_len - hdr_len;
1628                 skb->truesize += rx_frag_size;
1629                 skb->tail += hdr_len;
1630         }
1631         page_info->page = NULL;
1632
1633         if (rxcp->pkt_size <= rx_frag_size) {
1634                 BUG_ON(rxcp->num_rcvd != 1);
1635                 return;
1636         }
1637
1638         /* More frags present for this completion */
1639         remaining = rxcp->pkt_size - curr_frag_len;
1640         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1641                 page_info = get_rx_page_info(rxo);
1642                 curr_frag_len = min(remaining, rx_frag_size);
1643
1644                 /* Coalesce all frags from the same physical page in one slot */
1645                 if (page_info->page_offset == 0) {
1646                         /* Fresh page */
1647                         j++;
1648                         skb_frag_set_page(skb, j, page_info->page);
1649                         skb_shinfo(skb)->frags[j].page_offset =
1650                                                         page_info->page_offset;
1651                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1652                         skb_shinfo(skb)->nr_frags++;
1653                 } else {
1654                         put_page(page_info->page);
1655                 }
1656
1657                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1658                 skb->len += curr_frag_len;
1659                 skb->data_len += curr_frag_len;
1660                 skb->truesize += rx_frag_size;
1661                 remaining -= curr_frag_len;
1662                 page_info->page = NULL;
1663         }
1664         BUG_ON(j > MAX_SKB_FRAGS);
1665 }
1666
1667 /* Process the RX completion indicated by rxcp when GRO is disabled */
1668 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1669                                 struct be_rx_compl_info *rxcp)
1670 {
1671         struct be_adapter *adapter = rxo->adapter;
1672         struct net_device *netdev = adapter->netdev;
1673         struct sk_buff *skb;
1674
1675         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1676         if (unlikely(!skb)) {
1677                 rx_stats(rxo)->rx_drops_no_skbs++;
1678                 be_rx_compl_discard(rxo, rxcp);
1679                 return;
1680         }
1681
1682         skb_fill_rx_data(rxo, skb, rxcp);
1683
1684         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1685                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1686         else
1687                 skb_checksum_none_assert(skb);
1688
1689         skb->protocol = eth_type_trans(skb, netdev);
1690         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1691         if (netdev->features & NETIF_F_RXHASH)
1692                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1693
1694         skb->csum_level = rxcp->tunneled;
1695         skb_mark_napi_id(skb, napi);
1696
1697         if (rxcp->vlanf)
1698                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1699
1700         netif_receive_skb(skb);
1701 }
1702
1703 /* Process the RX completion indicated by rxcp when GRO is enabled */
1704 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1705                                     struct napi_struct *napi,
1706                                     struct be_rx_compl_info *rxcp)
1707 {
1708         struct be_adapter *adapter = rxo->adapter;
1709         struct be_rx_page_info *page_info;
1710         struct sk_buff *skb = NULL;
1711         u16 remaining, curr_frag_len;
1712         u16 i, j;
1713
1714         skb = napi_get_frags(napi);
1715         if (!skb) {
1716                 be_rx_compl_discard(rxo, rxcp);
1717                 return;
1718         }
1719
1720         remaining = rxcp->pkt_size;
1721         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1722                 page_info = get_rx_page_info(rxo);
1723
1724                 curr_frag_len = min(remaining, rx_frag_size);
1725
1726                 /* Coalesce all frags from the same physical page in one slot */
1727                 if (i == 0 || page_info->page_offset == 0) {
1728                         /* First frag or Fresh page */
1729                         j++;
1730                         skb_frag_set_page(skb, j, page_info->page);
1731                         skb_shinfo(skb)->frags[j].page_offset =
1732                                                         page_info->page_offset;
1733                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1734                 } else {
1735                         put_page(page_info->page);
1736                 }
1737                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1738                 skb->truesize += rx_frag_size;
1739                 remaining -= curr_frag_len;
1740                 memset(page_info, 0, sizeof(*page_info));
1741         }
1742         BUG_ON(j > MAX_SKB_FRAGS);
1743
1744         skb_shinfo(skb)->nr_frags = j + 1;
1745         skb->len = rxcp->pkt_size;
1746         skb->data_len = rxcp->pkt_size;
1747         skb->ip_summed = CHECKSUM_UNNECESSARY;
1748         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1749         if (adapter->netdev->features & NETIF_F_RXHASH)
1750                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1751
1752         skb->csum_level = rxcp->tunneled;
1753         skb_mark_napi_id(skb, napi);
1754
1755         if (rxcp->vlanf)
1756                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1757
1758         napi_gro_frags(napi);
1759 }
1760
1761 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1762                                  struct be_rx_compl_info *rxcp)
1763 {
1764         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1765         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1766         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1767         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1768         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1769         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1770         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1771         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1772         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1773         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1774         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1775         if (rxcp->vlanf) {
1776                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1777                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1778         }
1779         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1780         rxcp->tunneled =
1781                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1782 }
1783
1784 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1785                                  struct be_rx_compl_info *rxcp)
1786 {
1787         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1788         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1789         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1790         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1791         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1792         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1793         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1794         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1795         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1796         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1797         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1798         if (rxcp->vlanf) {
1799                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1800                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1801         }
1802         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1803         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1804 }
1805
1806 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1807 {
1808         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1809         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1810         struct be_adapter *adapter = rxo->adapter;
1811
1812         /* For checking the valid bit it is Ok to use either definition as the
1813          * valid bit is at the same position in both v0 and v1 Rx compl */
1814         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1815                 return NULL;
1816
1817         rmb();
1818         be_dws_le_to_cpu(compl, sizeof(*compl));
1819
1820         if (adapter->be3_native)
1821                 be_parse_rx_compl_v1(compl, rxcp);
1822         else
1823                 be_parse_rx_compl_v0(compl, rxcp);
1824
1825         if (rxcp->ip_frag)
1826                 rxcp->l4_csum = 0;
1827
1828         if (rxcp->vlanf) {
1829                 /* In QNQ modes, if qnq bit is not set, then the packet was
1830                  * tagged only with the transparent outer vlan-tag and must
1831                  * not be treated as a vlan packet by host
1832                  */
1833                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1834                         rxcp->vlanf = 0;
1835
1836                 if (!lancer_chip(adapter))
1837                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1838
1839                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1840                     !test_bit(rxcp->vlan_tag, adapter->vids))
1841                         rxcp->vlanf = 0;
1842         }
1843
1844         /* As the compl has been parsed, reset it; we wont touch it again */
1845         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1846
1847         queue_tail_inc(&rxo->cq);
1848         return rxcp;
1849 }
1850
1851 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1852 {
1853         u32 order = get_order(size);
1854
1855         if (order > 0)
1856                 gfp |= __GFP_COMP;
1857         return  alloc_pages(gfp, order);
1858 }
1859
1860 /*
1861  * Allocate a page, split it to fragments of size rx_frag_size and post as
1862  * receive buffers to BE
1863  */
1864 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
1865 {
1866         struct be_adapter *adapter = rxo->adapter;
1867         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1868         struct be_queue_info *rxq = &rxo->q;
1869         struct page *pagep = NULL;
1870         struct device *dev = &adapter->pdev->dev;
1871         struct be_eth_rx_d *rxd;
1872         u64 page_dmaaddr = 0, frag_dmaaddr;
1873         u32 posted, page_offset = 0, notify = 0;
1874
1875         page_info = &rxo->page_info_tbl[rxq->head];
1876         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
1877                 if (!pagep) {
1878                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1879                         if (unlikely(!pagep)) {
1880                                 rx_stats(rxo)->rx_post_fail++;
1881                                 break;
1882                         }
1883                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1884                                                     adapter->big_page_size,
1885                                                     DMA_FROM_DEVICE);
1886                         if (dma_mapping_error(dev, page_dmaaddr)) {
1887                                 put_page(pagep);
1888                                 pagep = NULL;
1889                                 adapter->drv_stats.dma_map_errors++;
1890                                 break;
1891                         }
1892                         page_offset = 0;
1893                 } else {
1894                         get_page(pagep);
1895                         page_offset += rx_frag_size;
1896                 }
1897                 page_info->page_offset = page_offset;
1898                 page_info->page = pagep;
1899
1900                 rxd = queue_head_node(rxq);
1901                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1902                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1903                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1904
1905                 /* Any space left in the current big page for another frag? */
1906                 if ((page_offset + rx_frag_size + rx_frag_size) >
1907                                         adapter->big_page_size) {
1908                         pagep = NULL;
1909                         page_info->last_frag = true;
1910                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1911                 } else {
1912                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1913                 }
1914
1915                 prev_page_info = page_info;
1916                 queue_head_inc(rxq);
1917                 page_info = &rxo->page_info_tbl[rxq->head];
1918         }
1919
1920         /* Mark the last frag of a page when we break out of the above loop
1921          * with no more slots available in the RXQ
1922          */
1923         if (pagep) {
1924                 prev_page_info->last_frag = true;
1925                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1926         }
1927
1928         if (posted) {
1929                 atomic_add(posted, &rxq->used);
1930                 if (rxo->rx_post_starved)
1931                         rxo->rx_post_starved = false;
1932                 do {
1933                         notify = min(256u, posted);
1934                         be_rxq_notify(adapter, rxq->id, notify);
1935                         posted -= notify;
1936                 } while (posted);
1937         } else if (atomic_read(&rxq->used) == 0) {
1938                 /* Let be_worker replenish when memory is available */
1939                 rxo->rx_post_starved = true;
1940         }
1941 }
1942
1943 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1944 {
1945         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1946
1947         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1948                 return NULL;
1949
1950         rmb();
1951         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1952
1953         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1954
1955         queue_tail_inc(tx_cq);
1956         return txcp;
1957 }
1958
1959 static u16 be_tx_compl_process(struct be_adapter *adapter,
1960                                struct be_tx_obj *txo, u16 last_index)
1961 {
1962         struct be_queue_info *txq = &txo->q;
1963         struct be_eth_wrb *wrb;
1964         struct sk_buff **sent_skbs = txo->sent_skb_list;
1965         struct sk_buff *sent_skb;
1966         u16 cur_index, num_wrbs = 1; /* account for hdr wrb */
1967         bool unmap_skb_hdr = true;
1968
1969         sent_skb = sent_skbs[txq->tail];
1970         BUG_ON(!sent_skb);
1971         sent_skbs[txq->tail] = NULL;
1972
1973         /* skip header wrb */
1974         queue_tail_inc(txq);
1975
1976         do {
1977                 cur_index = txq->tail;
1978                 wrb = queue_tail_node(txq);
1979                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1980                               (unmap_skb_hdr && skb_headlen(sent_skb)));
1981                 unmap_skb_hdr = false;
1982
1983                 num_wrbs++;
1984                 queue_tail_inc(txq);
1985         } while (cur_index != last_index);
1986
1987         dev_consume_skb_any(sent_skb);
1988         return num_wrbs;
1989 }
1990
1991 /* Return the number of events in the event queue */
1992 static inline int events_get(struct be_eq_obj *eqo)
1993 {
1994         struct be_eq_entry *eqe;
1995         int num = 0;
1996
1997         do {
1998                 eqe = queue_tail_node(&eqo->q);
1999                 if (eqe->evt == 0)
2000                         break;
2001
2002                 rmb();
2003                 eqe->evt = 0;
2004                 num++;
2005                 queue_tail_inc(&eqo->q);
2006         } while (true);
2007
2008         return num;
2009 }
2010
2011 /* Leaves the EQ is disarmed state */
2012 static void be_eq_clean(struct be_eq_obj *eqo)
2013 {
2014         int num = events_get(eqo);
2015
2016         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2017 }
2018
2019 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2020 {
2021         struct be_rx_page_info *page_info;
2022         struct be_queue_info *rxq = &rxo->q;
2023         struct be_queue_info *rx_cq = &rxo->cq;
2024         struct be_rx_compl_info *rxcp;
2025         struct be_adapter *adapter = rxo->adapter;
2026         int flush_wait = 0;
2027
2028         /* Consume pending rx completions.
2029          * Wait for the flush completion (identified by zero num_rcvd)
2030          * to arrive. Notify CQ even when there are no more CQ entries
2031          * for HW to flush partially coalesced CQ entries.
2032          * In Lancer, there is no need to wait for flush compl.
2033          */
2034         for (;;) {
2035                 rxcp = be_rx_compl_get(rxo);
2036                 if (!rxcp) {
2037                         if (lancer_chip(adapter))
2038                                 break;
2039
2040                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2041                                 dev_warn(&adapter->pdev->dev,
2042                                          "did not receive flush compl\n");
2043                                 break;
2044                         }
2045                         be_cq_notify(adapter, rx_cq->id, true, 0);
2046                         mdelay(1);
2047                 } else {
2048                         be_rx_compl_discard(rxo, rxcp);
2049                         be_cq_notify(adapter, rx_cq->id, false, 1);
2050                         if (rxcp->num_rcvd == 0)
2051                                 break;
2052                 }
2053         }
2054
2055         /* After cleanup, leave the CQ in unarmed state */
2056         be_cq_notify(adapter, rx_cq->id, false, 0);
2057
2058         /* Then free posted rx buffers that were not used */
2059         while (atomic_read(&rxq->used) > 0) {
2060                 page_info = get_rx_page_info(rxo);
2061                 put_page(page_info->page);
2062                 memset(page_info, 0, sizeof(*page_info));
2063         }
2064         BUG_ON(atomic_read(&rxq->used));
2065         rxq->tail = rxq->head = 0;
2066 }
2067
2068 static void be_tx_compl_clean(struct be_adapter *adapter)
2069 {
2070         struct be_tx_obj *txo;
2071         struct be_queue_info *txq;
2072         struct be_eth_tx_compl *txcp;
2073         u16 end_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2074         struct sk_buff *sent_skb;
2075         bool dummy_wrb;
2076         int i, pending_txqs;
2077
2078         /* Stop polling for compls when HW has been silent for 10ms */
2079         do {
2080                 pending_txqs = adapter->num_tx_qs;
2081
2082                 for_all_tx_queues(adapter, txo, i) {
2083                         cmpl = 0;
2084                         num_wrbs = 0;
2085                         txq = &txo->q;
2086                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2087                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2088                                 num_wrbs += be_tx_compl_process(adapter, txo,
2089                                                                 end_idx);
2090                                 cmpl++;
2091                         }
2092                         if (cmpl) {
2093                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2094                                 atomic_sub(num_wrbs, &txq->used);
2095                                 timeo = 0;
2096                         }
2097                         if (atomic_read(&txq->used) == 0)
2098                                 pending_txqs--;
2099                 }
2100
2101                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2102                         break;
2103
2104                 mdelay(1);
2105         } while (true);
2106
2107         for_all_tx_queues(adapter, txo, i) {
2108                 txq = &txo->q;
2109                 if (atomic_read(&txq->used))
2110                         dev_err(&adapter->pdev->dev, "%d pending tx-compls\n",
2111                                 atomic_read(&txq->used));
2112
2113                 /* free posted tx for which compls will never arrive */
2114                 while (atomic_read(&txq->used)) {
2115                         sent_skb = txo->sent_skb_list[txq->tail];
2116                         end_idx = txq->tail;
2117                         num_wrbs = wrb_cnt_for_skb(adapter, sent_skb,
2118                                                    &dummy_wrb);
2119                         index_adv(&end_idx, num_wrbs - 1, txq->len);
2120                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2121                         atomic_sub(num_wrbs, &txq->used);
2122                 }
2123         }
2124 }
2125
2126 static void be_evt_queues_destroy(struct be_adapter *adapter)
2127 {
2128         struct be_eq_obj *eqo;
2129         int i;
2130
2131         for_all_evt_queues(adapter, eqo, i) {
2132                 if (eqo->q.created) {
2133                         be_eq_clean(eqo);
2134                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2135                         napi_hash_del(&eqo->napi);
2136                         netif_napi_del(&eqo->napi);
2137                 }
2138                 be_queue_free(adapter, &eqo->q);
2139         }
2140 }
2141
2142 static int be_evt_queues_create(struct be_adapter *adapter)
2143 {
2144         struct be_queue_info *eq;
2145         struct be_eq_obj *eqo;
2146         struct be_aic_obj *aic;
2147         int i, rc;
2148
2149         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2150                                     adapter->cfg_num_qs);
2151
2152         for_all_evt_queues(adapter, eqo, i) {
2153                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2154                                BE_NAPI_WEIGHT);
2155                 napi_hash_add(&eqo->napi);
2156                 aic = &adapter->aic_obj[i];
2157                 eqo->adapter = adapter;
2158                 eqo->idx = i;
2159                 aic->max_eqd = BE_MAX_EQD;
2160                 aic->enable = true;
2161
2162                 eq = &eqo->q;
2163                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2164                                     sizeof(struct be_eq_entry));
2165                 if (rc)
2166                         return rc;
2167
2168                 rc = be_cmd_eq_create(adapter, eqo);
2169                 if (rc)
2170                         return rc;
2171         }
2172         return 0;
2173 }
2174
2175 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2176 {
2177         struct be_queue_info *q;
2178
2179         q = &adapter->mcc_obj.q;
2180         if (q->created)
2181                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2182         be_queue_free(adapter, q);
2183
2184         q = &adapter->mcc_obj.cq;
2185         if (q->created)
2186                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2187         be_queue_free(adapter, q);
2188 }
2189
2190 /* Must be called only after TX qs are created as MCC shares TX EQ */
2191 static int be_mcc_queues_create(struct be_adapter *adapter)
2192 {
2193         struct be_queue_info *q, *cq;
2194
2195         cq = &adapter->mcc_obj.cq;
2196         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2197                            sizeof(struct be_mcc_compl)))
2198                 goto err;
2199
2200         /* Use the default EQ for MCC completions */
2201         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2202                 goto mcc_cq_free;
2203
2204         q = &adapter->mcc_obj.q;
2205         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2206                 goto mcc_cq_destroy;
2207
2208         if (be_cmd_mccq_create(adapter, q, cq))
2209                 goto mcc_q_free;
2210
2211         return 0;
2212
2213 mcc_q_free:
2214         be_queue_free(adapter, q);
2215 mcc_cq_destroy:
2216         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2217 mcc_cq_free:
2218         be_queue_free(adapter, cq);
2219 err:
2220         return -1;
2221 }
2222
2223 static void be_tx_queues_destroy(struct be_adapter *adapter)
2224 {
2225         struct be_queue_info *q;
2226         struct be_tx_obj *txo;
2227         u8 i;
2228
2229         for_all_tx_queues(adapter, txo, i) {
2230                 q = &txo->q;
2231                 if (q->created)
2232                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2233                 be_queue_free(adapter, q);
2234
2235                 q = &txo->cq;
2236                 if (q->created)
2237                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2238                 be_queue_free(adapter, q);
2239         }
2240 }
2241
2242 static int be_tx_qs_create(struct be_adapter *adapter)
2243 {
2244         struct be_queue_info *cq, *eq;
2245         struct be_tx_obj *txo;
2246         int status, i;
2247
2248         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2249
2250         for_all_tx_queues(adapter, txo, i) {
2251                 cq = &txo->cq;
2252                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2253                                         sizeof(struct be_eth_tx_compl));
2254                 if (status)
2255                         return status;
2256
2257                 u64_stats_init(&txo->stats.sync);
2258                 u64_stats_init(&txo->stats.sync_compl);
2259
2260                 /* If num_evt_qs is less than num_tx_qs, then more than
2261                  * one txq share an eq
2262                  */
2263                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2264                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2265                 if (status)
2266                         return status;
2267
2268                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2269                                         sizeof(struct be_eth_wrb));
2270                 if (status)
2271                         return status;
2272
2273                 status = be_cmd_txq_create(adapter, txo);
2274                 if (status)
2275                         return status;
2276         }
2277
2278         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2279                  adapter->num_tx_qs);
2280         return 0;
2281 }
2282
2283 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2284 {
2285         struct be_queue_info *q;
2286         struct be_rx_obj *rxo;
2287         int i;
2288
2289         for_all_rx_queues(adapter, rxo, i) {
2290                 q = &rxo->cq;
2291                 if (q->created)
2292                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2293                 be_queue_free(adapter, q);
2294         }
2295 }
2296
2297 static int be_rx_cqs_create(struct be_adapter *adapter)
2298 {
2299         struct be_queue_info *eq, *cq;
2300         struct be_rx_obj *rxo;
2301         int rc, i;
2302
2303         /* We can create as many RSS rings as there are EQs. */
2304         adapter->num_rx_qs = adapter->num_evt_qs;
2305
2306         /* We'll use RSS only if atleast 2 RSS rings are supported.
2307          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2308          */
2309         if (adapter->num_rx_qs > 1)
2310                 adapter->num_rx_qs++;
2311
2312         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2313         for_all_rx_queues(adapter, rxo, i) {
2314                 rxo->adapter = adapter;
2315                 cq = &rxo->cq;
2316                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2317                                     sizeof(struct be_eth_rx_compl));
2318                 if (rc)
2319                         return rc;
2320
2321                 u64_stats_init(&rxo->stats.sync);
2322                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2323                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2324                 if (rc)
2325                         return rc;
2326         }
2327
2328         dev_info(&adapter->pdev->dev,
2329                  "created %d RSS queue(s) and 1 default RX queue\n",
2330                  adapter->num_rx_qs - 1);
2331         return 0;
2332 }
2333
2334 static irqreturn_t be_intx(int irq, void *dev)
2335 {
2336         struct be_eq_obj *eqo = dev;
2337         struct be_adapter *adapter = eqo->adapter;
2338         int num_evts = 0;
2339
2340         /* IRQ is not expected when NAPI is scheduled as the EQ
2341          * will not be armed.
2342          * But, this can happen on Lancer INTx where it takes
2343          * a while to de-assert INTx or in BE2 where occasionaly
2344          * an interrupt may be raised even when EQ is unarmed.
2345          * If NAPI is already scheduled, then counting & notifying
2346          * events will orphan them.
2347          */
2348         if (napi_schedule_prep(&eqo->napi)) {
2349                 num_evts = events_get(eqo);
2350                 __napi_schedule(&eqo->napi);
2351                 if (num_evts)
2352                         eqo->spurious_intr = 0;
2353         }
2354         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2355
2356         /* Return IRQ_HANDLED only for the the first spurious intr
2357          * after a valid intr to stop the kernel from branding
2358          * this irq as a bad one!
2359          */
2360         if (num_evts || eqo->spurious_intr++ == 0)
2361                 return IRQ_HANDLED;
2362         else
2363                 return IRQ_NONE;
2364 }
2365
2366 static irqreturn_t be_msix(int irq, void *dev)
2367 {
2368         struct be_eq_obj *eqo = dev;
2369
2370         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2371         napi_schedule(&eqo->napi);
2372         return IRQ_HANDLED;
2373 }
2374
2375 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2376 {
2377         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2378 }
2379
2380 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2381                          int budget, int polling)
2382 {
2383         struct be_adapter *adapter = rxo->adapter;
2384         struct be_queue_info *rx_cq = &rxo->cq;
2385         struct be_rx_compl_info *rxcp;
2386         u32 work_done;
2387         u32 frags_consumed = 0;
2388
2389         for (work_done = 0; work_done < budget; work_done++) {
2390                 rxcp = be_rx_compl_get(rxo);
2391                 if (!rxcp)
2392                         break;
2393
2394                 /* Is it a flush compl that has no data */
2395                 if (unlikely(rxcp->num_rcvd == 0))
2396                         goto loop_continue;
2397
2398                 /* Discard compl with partial DMA Lancer B0 */
2399                 if (unlikely(!rxcp->pkt_size)) {
2400                         be_rx_compl_discard(rxo, rxcp);
2401                         goto loop_continue;
2402                 }
2403
2404                 /* On BE drop pkts that arrive due to imperfect filtering in
2405                  * promiscuous mode on some skews
2406                  */
2407                 if (unlikely(rxcp->port != adapter->port_num &&
2408                              !lancer_chip(adapter))) {
2409                         be_rx_compl_discard(rxo, rxcp);
2410                         goto loop_continue;
2411                 }
2412
2413                 /* Don't do gro when we're busy_polling */
2414                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2415                         be_rx_compl_process_gro(rxo, napi, rxcp);
2416                 else
2417                         be_rx_compl_process(rxo, napi, rxcp);
2418
2419 loop_continue:
2420                 frags_consumed += rxcp->num_rcvd;
2421                 be_rx_stats_update(rxo, rxcp);
2422         }
2423
2424         if (work_done) {
2425                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2426
2427                 /* When an rx-obj gets into post_starved state, just
2428                  * let be_worker do the posting.
2429                  */
2430                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2431                     !rxo->rx_post_starved)
2432                         be_post_rx_frags(rxo, GFP_ATOMIC,
2433                                          max_t(u32, MAX_RX_POST,
2434                                                frags_consumed));
2435         }
2436
2437         return work_done;
2438 }
2439
2440 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2441 {
2442         switch (status) {
2443         case BE_TX_COMP_HDR_PARSE_ERR:
2444                 tx_stats(txo)->tx_hdr_parse_err++;
2445                 break;
2446         case BE_TX_COMP_NDMA_ERR:
2447                 tx_stats(txo)->tx_dma_err++;
2448                 break;
2449         case BE_TX_COMP_ACL_ERR:
2450                 tx_stats(txo)->tx_spoof_check_err++;
2451                 break;
2452         }
2453 }
2454
2455 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2456 {
2457         switch (status) {
2458         case LANCER_TX_COMP_LSO_ERR:
2459                 tx_stats(txo)->tx_tso_err++;
2460                 break;
2461         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2462         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2463                 tx_stats(txo)->tx_spoof_check_err++;
2464                 break;
2465         case LANCER_TX_COMP_QINQ_ERR:
2466                 tx_stats(txo)->tx_qinq_err++;
2467                 break;
2468         case LANCER_TX_COMP_PARITY_ERR:
2469                 tx_stats(txo)->tx_internal_parity_err++;
2470                 break;
2471         case LANCER_TX_COMP_DMA_ERR:
2472                 tx_stats(txo)->tx_dma_err++;
2473                 break;
2474         }
2475 }
2476
2477 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2478                           int idx)
2479 {
2480         struct be_eth_tx_compl *txcp;
2481         int num_wrbs = 0, work_done = 0;
2482         u32 compl_status;
2483         u16 last_idx;
2484
2485         while ((txcp = be_tx_compl_get(&txo->cq))) {
2486                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2487                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2488                 work_done++;
2489
2490                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2491                 if (compl_status) {
2492                         if (lancer_chip(adapter))
2493                                 lancer_update_tx_err(txo, compl_status);
2494                         else
2495                                 be_update_tx_err(txo, compl_status);
2496                 }
2497         }
2498
2499         if (work_done) {
2500                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2501                 atomic_sub(num_wrbs, &txo->q.used);
2502
2503                 /* As Tx wrbs have been freed up, wake up netdev queue
2504                  * if it was stopped due to lack of tx wrbs.  */
2505                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2506                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2507                         netif_wake_subqueue(adapter->netdev, idx);
2508                 }
2509
2510                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2511                 tx_stats(txo)->tx_compl += work_done;
2512                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2513         }
2514 }
2515
2516 int be_poll(struct napi_struct *napi, int budget)
2517 {
2518         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2519         struct be_adapter *adapter = eqo->adapter;
2520         int max_work = 0, work, i, num_evts;
2521         struct be_rx_obj *rxo;
2522         struct be_tx_obj *txo;
2523
2524         num_evts = events_get(eqo);
2525
2526         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2527                 be_process_tx(adapter, txo, i);
2528
2529         if (be_lock_napi(eqo)) {
2530                 /* This loop will iterate twice for EQ0 in which
2531                  * completions of the last RXQ (default one) are also processed
2532                  * For other EQs the loop iterates only once
2533                  */
2534                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2535                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2536                         max_work = max(work, max_work);
2537                 }
2538                 be_unlock_napi(eqo);
2539         } else {
2540                 max_work = budget;
2541         }
2542
2543         if (is_mcc_eqo(eqo))
2544                 be_process_mcc(adapter);
2545
2546         if (max_work < budget) {
2547                 napi_complete(napi);
2548                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2549         } else {
2550                 /* As we'll continue in polling mode, count and clear events */
2551                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2552         }
2553         return max_work;
2554 }
2555
2556 #ifdef CONFIG_NET_RX_BUSY_POLL
2557 static int be_busy_poll(struct napi_struct *napi)
2558 {
2559         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2560         struct be_adapter *adapter = eqo->adapter;
2561         struct be_rx_obj *rxo;
2562         int i, work = 0;
2563
2564         if (!be_lock_busy_poll(eqo))
2565                 return LL_FLUSH_BUSY;
2566
2567         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2568                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2569                 if (work)
2570                         break;
2571         }
2572
2573         be_unlock_busy_poll(eqo);
2574         return work;
2575 }
2576 #endif
2577
2578 void be_detect_error(struct be_adapter *adapter)
2579 {
2580         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2581         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2582         u32 i;
2583         bool error_detected = false;
2584         struct device *dev = &adapter->pdev->dev;
2585         struct net_device *netdev = adapter->netdev;
2586
2587         if (be_hw_error(adapter))
2588                 return;
2589
2590         if (lancer_chip(adapter)) {
2591                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2592                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2593                         sliport_err1 = ioread32(adapter->db +
2594                                                 SLIPORT_ERROR1_OFFSET);
2595                         sliport_err2 = ioread32(adapter->db +
2596                                                 SLIPORT_ERROR2_OFFSET);
2597                         adapter->hw_error = true;
2598                         /* Do not log error messages if its a FW reset */
2599                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2600                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2601                                 dev_info(dev, "Firmware update in progress\n");
2602                         } else {
2603                                 error_detected = true;
2604                                 dev_err(dev, "Error detected in the card\n");
2605                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2606                                         sliport_status);
2607                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2608                                         sliport_err1);
2609                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2610                                         sliport_err2);
2611                         }
2612                 }
2613         } else {
2614                 pci_read_config_dword(adapter->pdev,
2615                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2616                 pci_read_config_dword(adapter->pdev,
2617                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2618                 pci_read_config_dword(adapter->pdev,
2619                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2620                 pci_read_config_dword(adapter->pdev,
2621                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2622
2623                 ue_lo = (ue_lo & ~ue_lo_mask);
2624                 ue_hi = (ue_hi & ~ue_hi_mask);
2625
2626                 /* On certain platforms BE hardware can indicate spurious UEs.
2627                  * Allow HW to stop working completely in case of a real UE.
2628                  * Hence not setting the hw_error for UE detection.
2629                  */
2630
2631                 if (ue_lo || ue_hi) {
2632                         error_detected = true;
2633                         dev_err(dev,
2634                                 "Unrecoverable Error detected in the adapter");
2635                         dev_err(dev, "Please reboot server to recover");
2636                         if (skyhawk_chip(adapter))
2637                                 adapter->hw_error = true;
2638                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2639                                 if (ue_lo & 1)
2640                                         dev_err(dev, "UE: %s bit set\n",
2641                                                 ue_status_low_desc[i]);
2642                         }
2643                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2644                                 if (ue_hi & 1)
2645                                         dev_err(dev, "UE: %s bit set\n",
2646                                                 ue_status_hi_desc[i]);
2647                         }
2648                 }
2649         }
2650         if (error_detected)
2651                 netif_carrier_off(netdev);
2652 }
2653
2654 static void be_msix_disable(struct be_adapter *adapter)
2655 {
2656         if (msix_enabled(adapter)) {
2657                 pci_disable_msix(adapter->pdev);
2658                 adapter->num_msix_vec = 0;
2659                 adapter->num_msix_roce_vec = 0;
2660         }
2661 }
2662
2663 static int be_msix_enable(struct be_adapter *adapter)
2664 {
2665         int i, num_vec;
2666         struct device *dev = &adapter->pdev->dev;
2667
2668         /* If RoCE is supported, program the max number of NIC vectors that
2669          * may be configured via set-channels, along with vectors needed for
2670          * RoCe. Else, just program the number we'll use initially.
2671          */
2672         if (be_roce_supported(adapter))
2673                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2674                                 2 * num_online_cpus());
2675         else
2676                 num_vec = adapter->cfg_num_qs;
2677
2678         for (i = 0; i < num_vec; i++)
2679                 adapter->msix_entries[i].entry = i;
2680
2681         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2682                                         MIN_MSIX_VECTORS, num_vec);
2683         if (num_vec < 0)
2684                 goto fail;
2685
2686         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2687                 adapter->num_msix_roce_vec = num_vec / 2;
2688                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2689                          adapter->num_msix_roce_vec);
2690         }
2691
2692         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2693
2694         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2695                  adapter->num_msix_vec);
2696         return 0;
2697
2698 fail:
2699         dev_warn(dev, "MSIx enable failed\n");
2700
2701         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2702         if (!be_physfn(adapter))
2703                 return num_vec;
2704         return 0;
2705 }
2706
2707 static inline int be_msix_vec_get(struct be_adapter *adapter,
2708                                   struct be_eq_obj *eqo)
2709 {
2710         return adapter->msix_entries[eqo->msix_idx].vector;
2711 }
2712
2713 static int be_msix_register(struct be_adapter *adapter)
2714 {
2715         struct net_device *netdev = adapter->netdev;
2716         struct be_eq_obj *eqo;
2717         int status, i, vec;
2718
2719         for_all_evt_queues(adapter, eqo, i) {
2720                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2721                 vec = be_msix_vec_get(adapter, eqo);
2722                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2723                 if (status)
2724                         goto err_msix;
2725         }
2726
2727         return 0;
2728 err_msix:
2729         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2730                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2731         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2732                  status);
2733         be_msix_disable(adapter);
2734         return status;
2735 }
2736
2737 static int be_irq_register(struct be_adapter *adapter)
2738 {
2739         struct net_device *netdev = adapter->netdev;
2740         int status;
2741
2742         if (msix_enabled(adapter)) {
2743                 status = be_msix_register(adapter);
2744                 if (status == 0)
2745                         goto done;
2746                 /* INTx is not supported for VF */
2747                 if (!be_physfn(adapter))
2748                         return status;
2749         }
2750
2751         /* INTx: only the first EQ is used */
2752         netdev->irq = adapter->pdev->irq;
2753         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2754                              &adapter->eq_obj[0]);
2755         if (status) {
2756                 dev_err(&adapter->pdev->dev,
2757                         "INTx request IRQ failed - err %d\n", status);
2758                 return status;
2759         }
2760 done:
2761         adapter->isr_registered = true;
2762         return 0;
2763 }
2764
2765 static void be_irq_unregister(struct be_adapter *adapter)
2766 {
2767         struct net_device *netdev = adapter->netdev;
2768         struct be_eq_obj *eqo;
2769         int i;
2770
2771         if (!adapter->isr_registered)
2772                 return;
2773
2774         /* INTx */
2775         if (!msix_enabled(adapter)) {
2776                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2777                 goto done;
2778         }
2779
2780         /* MSIx */
2781         for_all_evt_queues(adapter, eqo, i)
2782                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2783
2784 done:
2785         adapter->isr_registered = false;
2786 }
2787
2788 static void be_rx_qs_destroy(struct be_adapter *adapter)
2789 {
2790         struct be_queue_info *q;
2791         struct be_rx_obj *rxo;
2792         int i;
2793
2794         for_all_rx_queues(adapter, rxo, i) {
2795                 q = &rxo->q;
2796                 if (q->created) {
2797                         be_cmd_rxq_destroy(adapter, q);
2798                         be_rx_cq_clean(rxo);
2799                 }
2800                 be_queue_free(adapter, q);
2801         }
2802 }
2803
2804 static int be_close(struct net_device *netdev)
2805 {
2806         struct be_adapter *adapter = netdev_priv(netdev);
2807         struct be_eq_obj *eqo;
2808         int i;
2809
2810         /* This protection is needed as be_close() may be called even when the
2811          * adapter is in cleared state (after eeh perm failure)
2812          */
2813         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2814                 return 0;
2815
2816         be_roce_dev_close(adapter);
2817
2818         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2819                 for_all_evt_queues(adapter, eqo, i) {
2820                         napi_disable(&eqo->napi);
2821                         be_disable_busy_poll(eqo);
2822                 }
2823                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2824         }
2825
2826         be_async_mcc_disable(adapter);
2827
2828         /* Wait for all pending tx completions to arrive so that
2829          * all tx skbs are freed.
2830          */
2831         netif_tx_disable(netdev);
2832         be_tx_compl_clean(adapter);
2833
2834         be_rx_qs_destroy(adapter);
2835
2836         for (i = 1; i < (adapter->uc_macs + 1); i++)
2837                 be_cmd_pmac_del(adapter, adapter->if_handle,
2838                                 adapter->pmac_id[i], 0);
2839         adapter->uc_macs = 0;
2840
2841         for_all_evt_queues(adapter, eqo, i) {
2842                 if (msix_enabled(adapter))
2843                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2844                 else
2845                         synchronize_irq(netdev->irq);
2846                 be_eq_clean(eqo);
2847         }
2848
2849         be_irq_unregister(adapter);
2850
2851         return 0;
2852 }
2853
2854 static int be_rx_qs_create(struct be_adapter *adapter)
2855 {
2856         struct be_rx_obj *rxo;
2857         int rc, i, j;
2858         u8 rss_hkey[RSS_HASH_KEY_LEN];
2859         struct rss_info *rss = &adapter->rss_info;
2860
2861         for_all_rx_queues(adapter, rxo, i) {
2862                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2863                                     sizeof(struct be_eth_rx_d));
2864                 if (rc)
2865                         return rc;
2866         }
2867
2868         /* The FW would like the default RXQ to be created first */
2869         rxo = default_rxo(adapter);
2870         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2871                                adapter->if_handle, false, &rxo->rss_id);
2872         if (rc)
2873                 return rc;
2874
2875         for_all_rss_queues(adapter, rxo, i) {
2876                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2877                                        rx_frag_size, adapter->if_handle,
2878                                        true, &rxo->rss_id);
2879                 if (rc)
2880                         return rc;
2881         }
2882
2883         if (be_multi_rxq(adapter)) {
2884                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
2885                         j += adapter->num_rx_qs - 1) {
2886                         for_all_rss_queues(adapter, rxo, i) {
2887                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
2888                                         break;
2889                                 rss->rsstable[j + i] = rxo->rss_id;
2890                                 rss->rss_queue[j + i] = i;
2891                         }
2892                 }
2893                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2894                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2895
2896                 if (!BEx_chip(adapter))
2897                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2898                                 RSS_ENABLE_UDP_IPV6;
2899         } else {
2900                 /* Disable RSS, if only default RX Q is created */
2901                 rss->rss_flags = RSS_ENABLE_NONE;
2902         }
2903
2904         get_random_bytes(rss_hkey, RSS_HASH_KEY_LEN);
2905         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
2906                                128, rss_hkey);
2907         if (rc) {
2908                 rss->rss_flags = RSS_ENABLE_NONE;
2909                 return rc;
2910         }
2911
2912         memcpy(rss->rss_hkey, rss_hkey, RSS_HASH_KEY_LEN);
2913
2914         /* First time posting */
2915         for_all_rx_queues(adapter, rxo, i)
2916                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
2917         return 0;
2918 }
2919
2920 static int be_open(struct net_device *netdev)
2921 {
2922         struct be_adapter *adapter = netdev_priv(netdev);
2923         struct be_eq_obj *eqo;
2924         struct be_rx_obj *rxo;
2925         struct be_tx_obj *txo;
2926         u8 link_status;
2927         int status, i;
2928
2929         status = be_rx_qs_create(adapter);
2930         if (status)
2931                 goto err;
2932
2933         status = be_irq_register(adapter);
2934         if (status)
2935                 goto err;
2936
2937         for_all_rx_queues(adapter, rxo, i)
2938                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2939
2940         for_all_tx_queues(adapter, txo, i)
2941                 be_cq_notify(adapter, txo->cq.id, true, 0);
2942
2943         be_async_mcc_enable(adapter);
2944
2945         for_all_evt_queues(adapter, eqo, i) {
2946                 napi_enable(&eqo->napi);
2947                 be_enable_busy_poll(eqo);
2948                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
2949         }
2950         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2951
2952         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2953         if (!status)
2954                 be_link_status_update(adapter, link_status);
2955
2956         netif_tx_start_all_queues(netdev);
2957         be_roce_dev_open(adapter);
2958
2959 #ifdef CONFIG_BE2NET_VXLAN
2960         if (skyhawk_chip(adapter))
2961                 vxlan_get_rx_port(netdev);
2962 #endif
2963
2964         return 0;
2965 err:
2966         be_close(adapter->netdev);
2967         return -EIO;
2968 }
2969
2970 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2971 {
2972         struct be_dma_mem cmd;
2973         int status = 0;
2974         u8 mac[ETH_ALEN];
2975
2976         memset(mac, 0, ETH_ALEN);
2977
2978         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2979         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2980                                      GFP_KERNEL);
2981         if (!cmd.va)
2982                 return -ENOMEM;
2983
2984         if (enable) {
2985                 status = pci_write_config_dword(adapter->pdev,
2986                                                 PCICFG_PM_CONTROL_OFFSET,
2987                                                 PCICFG_PM_CONTROL_MASK);
2988                 if (status) {
2989                         dev_err(&adapter->pdev->dev,
2990                                 "Could not enable Wake-on-lan\n");
2991                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
2992                                           cmd.dma);
2993                         return status;
2994                 }
2995                 status = be_cmd_enable_magic_wol(adapter,
2996                                                  adapter->netdev->dev_addr,
2997                                                  &cmd);
2998                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
2999                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3000         } else {
3001                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3002                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3003                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3004         }
3005
3006         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3007         return status;
3008 }
3009
3010 /*
3011  * Generate a seed MAC address from the PF MAC Address using jhash.
3012  * MAC Address for VFs are assigned incrementally starting from the seed.
3013  * These addresses are programmed in the ASIC by the PF and the VF driver
3014  * queries for the MAC address during its probe.
3015  */
3016 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3017 {
3018         u32 vf;
3019         int status = 0;
3020         u8 mac[ETH_ALEN];
3021         struct be_vf_cfg *vf_cfg;
3022
3023         be_vf_eth_addr_generate(adapter, mac);
3024
3025         for_all_vfs(adapter, vf_cfg, vf) {
3026                 if (BEx_chip(adapter))
3027                         status = be_cmd_pmac_add(adapter, mac,
3028                                                  vf_cfg->if_handle,
3029                                                  &vf_cfg->pmac_id, vf + 1);
3030                 else
3031                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3032                                                 vf + 1);
3033
3034                 if (status)
3035                         dev_err(&adapter->pdev->dev,
3036                                 "Mac address assignment failed for VF %d\n",
3037                                 vf);
3038                 else
3039                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3040
3041                 mac[5] += 1;
3042         }
3043         return status;
3044 }
3045
3046 static int be_vfs_mac_query(struct be_adapter *adapter)
3047 {
3048         int status, vf;
3049         u8 mac[ETH_ALEN];
3050         struct be_vf_cfg *vf_cfg;
3051
3052         for_all_vfs(adapter, vf_cfg, vf) {
3053                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3054                                                mac, vf_cfg->if_handle,
3055                                                false, vf+1);
3056                 if (status)
3057                         return status;
3058                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3059         }
3060         return 0;
3061 }
3062
3063 static void be_vf_clear(struct be_adapter *adapter)
3064 {
3065         struct be_vf_cfg *vf_cfg;
3066         u32 vf;
3067
3068         if (pci_vfs_assigned(adapter->pdev)) {
3069                 dev_warn(&adapter->pdev->dev,
3070                          "VFs are assigned to VMs: not disabling VFs\n");
3071                 goto done;
3072         }
3073
3074         pci_disable_sriov(adapter->pdev);
3075
3076         for_all_vfs(adapter, vf_cfg, vf) {
3077                 if (BEx_chip(adapter))
3078                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3079                                         vf_cfg->pmac_id, vf + 1);
3080                 else
3081                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3082                                        vf + 1);
3083
3084                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3085         }
3086 done:
3087         kfree(adapter->vf_cfg);
3088         adapter->num_vfs = 0;
3089         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3090 }
3091
3092 static void be_clear_queues(struct be_adapter *adapter)
3093 {
3094         be_mcc_queues_destroy(adapter);
3095         be_rx_cqs_destroy(adapter);
3096         be_tx_queues_destroy(adapter);
3097         be_evt_queues_destroy(adapter);
3098 }
3099
3100 static void be_cancel_worker(struct be_adapter *adapter)
3101 {
3102         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3103                 cancel_delayed_work_sync(&adapter->work);
3104                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3105         }
3106 }
3107
3108 static void be_mac_clear(struct be_adapter *adapter)
3109 {
3110         int i;
3111
3112         if (adapter->pmac_id) {
3113                 for (i = 0; i < (adapter->uc_macs + 1); i++)
3114                         be_cmd_pmac_del(adapter, adapter->if_handle,
3115                                         adapter->pmac_id[i], 0);
3116                 adapter->uc_macs = 0;
3117
3118                 kfree(adapter->pmac_id);
3119                 adapter->pmac_id = NULL;
3120         }
3121 }
3122
3123 #ifdef CONFIG_BE2NET_VXLAN
3124 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3125 {
3126         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3127                 be_cmd_manage_iface(adapter, adapter->if_handle,
3128                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3129
3130         if (adapter->vxlan_port)
3131                 be_cmd_set_vxlan_port(adapter, 0);
3132
3133         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3134         adapter->vxlan_port = 0;
3135 }
3136 #endif
3137
3138 static int be_clear(struct be_adapter *adapter)
3139 {
3140         be_cancel_worker(adapter);
3141
3142         if (sriov_enabled(adapter))
3143                 be_vf_clear(adapter);
3144
3145         /* Re-configure FW to distribute resources evenly across max-supported
3146          * number of VFs, only when VFs are not already enabled.
3147          */
3148         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3149                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3150                                         pci_sriov_get_totalvfs(adapter->pdev));
3151
3152 #ifdef CONFIG_BE2NET_VXLAN
3153         be_disable_vxlan_offloads(adapter);
3154 #endif
3155         /* delete the primary mac along with the uc-mac list */
3156         be_mac_clear(adapter);
3157
3158         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3159
3160         be_clear_queues(adapter);
3161
3162         be_msix_disable(adapter);
3163         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3164         return 0;
3165 }
3166
3167 static int be_vfs_if_create(struct be_adapter *adapter)
3168 {
3169         struct be_resources res = {0};
3170         struct be_vf_cfg *vf_cfg;
3171         u32 cap_flags, en_flags, vf;
3172         int status = 0;
3173
3174         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3175                     BE_IF_FLAGS_MULTICAST;
3176
3177         for_all_vfs(adapter, vf_cfg, vf) {
3178                 if (!BE3_chip(adapter)) {
3179                         status = be_cmd_get_profile_config(adapter, &res,
3180                                                            vf + 1);
3181                         if (!status)
3182                                 cap_flags = res.if_cap_flags;
3183                 }
3184
3185                 /* If a FW profile exists, then cap_flags are updated */
3186                 en_flags = cap_flags & (BE_IF_FLAGS_UNTAGGED |
3187                                         BE_IF_FLAGS_BROADCAST |
3188                                         BE_IF_FLAGS_MULTICAST);
3189                 status =
3190                     be_cmd_if_create(adapter, cap_flags, en_flags,
3191                                      &vf_cfg->if_handle, vf + 1);
3192                 if (status)
3193                         goto err;
3194         }
3195 err:
3196         return status;
3197 }
3198
3199 static int be_vf_setup_init(struct be_adapter *adapter)
3200 {
3201         struct be_vf_cfg *vf_cfg;
3202         int vf;
3203
3204         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3205                                   GFP_KERNEL);
3206         if (!adapter->vf_cfg)
3207                 return -ENOMEM;
3208
3209         for_all_vfs(adapter, vf_cfg, vf) {
3210                 vf_cfg->if_handle = -1;
3211                 vf_cfg->pmac_id = -1;
3212         }
3213         return 0;
3214 }
3215
3216 static int be_vf_setup(struct be_adapter *adapter)
3217 {
3218         struct device *dev = &adapter->pdev->dev;
3219         struct be_vf_cfg *vf_cfg;
3220         int status, old_vfs, vf;
3221         u32 privileges;
3222
3223         old_vfs = pci_num_vf(adapter->pdev);
3224
3225         status = be_vf_setup_init(adapter);
3226         if (status)
3227                 goto err;
3228
3229         if (old_vfs) {
3230                 for_all_vfs(adapter, vf_cfg, vf) {
3231                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3232                         if (status)
3233                                 goto err;
3234                 }
3235
3236                 status = be_vfs_mac_query(adapter);
3237                 if (status)
3238                         goto err;
3239         } else {
3240                 status = be_vfs_if_create(adapter);
3241                 if (status)
3242                         goto err;
3243
3244                 status = be_vf_eth_addr_config(adapter);
3245                 if (status)
3246                         goto err;
3247         }
3248
3249         for_all_vfs(adapter, vf_cfg, vf) {
3250                 /* Allow VFs to programs MAC/VLAN filters */
3251                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3252                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3253                         status = be_cmd_set_fn_privileges(adapter,
3254                                                           privileges |
3255                                                           BE_PRIV_FILTMGMT,
3256                                                           vf + 1);
3257                         if (!status)
3258                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3259                                          vf);
3260                 }
3261
3262                 /* Allow full available bandwidth */
3263                 if (!old_vfs)
3264                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3265
3266                 if (!old_vfs) {
3267                         be_cmd_enable_vf(adapter, vf + 1);
3268                         be_cmd_set_logical_link_config(adapter,
3269                                                        IFLA_VF_LINK_STATE_AUTO,
3270                                                        vf+1);
3271                 }
3272         }
3273
3274         if (!old_vfs) {
3275                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3276                 if (status) {
3277                         dev_err(dev, "SRIOV enable failed\n");
3278                         adapter->num_vfs = 0;
3279                         goto err;
3280                 }
3281         }
3282
3283         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3284         return 0;
3285 err:
3286         dev_err(dev, "VF setup failed\n");
3287         be_vf_clear(adapter);
3288         return status;
3289 }
3290
3291 /* Converting function_mode bits on BE3 to SH mc_type enums */
3292
3293 static u8 be_convert_mc_type(u32 function_mode)
3294 {
3295         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3296                 return vNIC1;
3297         else if (function_mode & QNQ_MODE)
3298                 return FLEX10;
3299         else if (function_mode & VNIC_MODE)
3300                 return vNIC2;
3301         else if (function_mode & UMC_ENABLED)
3302                 return UMC;
3303         else
3304                 return MC_NONE;
3305 }
3306
3307 /* On BE2/BE3 FW does not suggest the supported limits */
3308 static void BEx_get_resources(struct be_adapter *adapter,
3309                               struct be_resources *res)
3310 {
3311         bool use_sriov = adapter->num_vfs ? 1 : 0;
3312
3313         if (be_physfn(adapter))
3314                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3315         else
3316                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3317
3318         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3319
3320         if (be_is_mc(adapter)) {
3321                 /* Assuming that there are 4 channels per port,
3322                  * when multi-channel is enabled
3323                  */
3324                 if (be_is_qnq_mode(adapter))
3325                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3326                 else
3327                         /* In a non-qnq multichannel mode, the pvid
3328                          * takes up one vlan entry
3329                          */
3330                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3331         } else {
3332                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3333         }
3334
3335         res->max_mcast_mac = BE_MAX_MC;
3336
3337         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3338          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3339          *    *only* if it is RSS-capable.
3340          */
3341         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3342             !be_physfn(adapter) || (be_is_mc(adapter) &&
3343             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3344                 res->max_tx_qs = 1;
3345         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3346                 struct be_resources super_nic_res = {0};
3347
3348                 /* On a SuperNIC profile, the driver needs to use the
3349                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3350                  */
3351                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3352                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3353                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3354         } else {
3355                 res->max_tx_qs = BE3_MAX_TX_QS;
3356         }
3357
3358         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3359             !use_sriov && be_physfn(adapter))
3360                 res->max_rss_qs = (adapter->be3_native) ?
3361                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3362         res->max_rx_qs = res->max_rss_qs + 1;
3363
3364         if (be_physfn(adapter))
3365                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3366                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3367         else
3368                 res->max_evt_qs = 1;
3369
3370         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3371         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3372                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3373 }
3374
3375 static void be_setup_init(struct be_adapter *adapter)
3376 {
3377         adapter->vlan_prio_bmap = 0xff;
3378         adapter->phy.link_speed = -1;
3379         adapter->if_handle = -1;
3380         adapter->be3_native = false;
3381         adapter->promiscuous = false;
3382         if (be_physfn(adapter))
3383                 adapter->cmd_privileges = MAX_PRIVILEGES;
3384         else
3385                 adapter->cmd_privileges = MIN_PRIVILEGES;
3386 }
3387
3388 static int be_get_sriov_config(struct be_adapter *adapter)
3389 {
3390         struct device *dev = &adapter->pdev->dev;
3391         struct be_resources res = {0};
3392         int max_vfs, old_vfs;
3393
3394         /* Some old versions of BE3 FW don't report max_vfs value */
3395         be_cmd_get_profile_config(adapter, &res, 0);
3396
3397         if (BE3_chip(adapter) && !res.max_vfs) {
3398                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3399                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3400         }
3401
3402         adapter->pool_res = res;
3403
3404         if (!be_max_vfs(adapter)) {
3405                 if (num_vfs)
3406                         dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n");
3407                 adapter->num_vfs = 0;
3408                 return 0;
3409         }
3410
3411         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3412
3413         /* validate num_vfs module param */
3414         old_vfs = pci_num_vf(adapter->pdev);
3415         if (old_vfs) {
3416                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3417                 if (old_vfs != num_vfs)
3418                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3419                 adapter->num_vfs = old_vfs;
3420         } else {
3421                 if (num_vfs > be_max_vfs(adapter)) {
3422                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3423                                  num_vfs);
3424                         dev_info(dev, "Limiting to %d VFs\n",
3425                                  be_max_vfs(adapter));
3426                 }
3427                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3428         }
3429
3430         return 0;
3431 }
3432
3433 static int be_get_resources(struct be_adapter *adapter)
3434 {
3435         struct device *dev = &adapter->pdev->dev;
3436         struct be_resources res = {0};
3437         int status;
3438
3439         if (BEx_chip(adapter)) {
3440                 BEx_get_resources(adapter, &res);
3441                 adapter->res = res;
3442         }
3443
3444         /* For Lancer, SH etc read per-function resource limits from FW.
3445          * GET_FUNC_CONFIG returns per function guaranteed limits.
3446          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3447          */
3448         if (!BEx_chip(adapter)) {
3449                 status = be_cmd_get_func_config(adapter, &res);
3450                 if (status)
3451                         return status;
3452
3453                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3454                 if (be_roce_supported(adapter))
3455                         res.max_evt_qs /= 2;
3456                 adapter->res = res;
3457         }
3458
3459         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3460                  be_max_txqs(adapter), be_max_rxqs(adapter),
3461                  be_max_rss(adapter), be_max_eqs(adapter),
3462                  be_max_vfs(adapter));
3463         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3464                  be_max_uc(adapter), be_max_mc(adapter),
3465                  be_max_vlans(adapter));
3466
3467         return 0;
3468 }
3469
3470 static void be_sriov_config(struct be_adapter *adapter)
3471 {
3472         struct device *dev = &adapter->pdev->dev;
3473         int status;
3474
3475         status = be_get_sriov_config(adapter);
3476         if (status) {
3477                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3478                 dev_err(dev, "SR-IOV cannot be enabled\n");
3479                 return;
3480         }
3481
3482         /* When the HW is in SRIOV capable configuration, the PF-pool
3483          * resources are equally distributed across the max-number of
3484          * VFs. The user may request only a subset of the max-vfs to be
3485          * enabled. Based on num_vfs, redistribute the resources across
3486          * num_vfs so that each VF will have access to more number of
3487          * resources. This facility is not available in BE3 FW.
3488          * Also, this is done by FW in Lancer chip.
3489          */
3490         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3491                 status = be_cmd_set_sriov_config(adapter,
3492                                                  adapter->pool_res,
3493                                                  adapter->num_vfs);
3494                 if (status)
3495                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3496         }
3497 }
3498
3499 static int be_get_config(struct be_adapter *adapter)
3500 {
3501         u16 profile_id;
3502         int status;
3503
3504         status = be_cmd_query_fw_cfg(adapter);
3505         if (status)
3506                 return status;
3507
3508          if (be_physfn(adapter)) {
3509                 status = be_cmd_get_active_profile(adapter, &profile_id);
3510                 if (!status)
3511                         dev_info(&adapter->pdev->dev,
3512                                  "Using profile 0x%x\n", profile_id);
3513         }
3514
3515         if (!BE2_chip(adapter) && be_physfn(adapter))
3516                 be_sriov_config(adapter);
3517
3518         status = be_get_resources(adapter);
3519         if (status)
3520                 return status;
3521
3522         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3523                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3524         if (!adapter->pmac_id)
3525                 return -ENOMEM;
3526
3527         /* Sanitize cfg_num_qs based on HW and platform limits */
3528         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3529
3530         return 0;
3531 }
3532
3533 static int be_mac_setup(struct be_adapter *adapter)
3534 {
3535         u8 mac[ETH_ALEN];
3536         int status;
3537
3538         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3539                 status = be_cmd_get_perm_mac(adapter, mac);
3540                 if (status)
3541                         return status;
3542
3543                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3544                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3545         } else {
3546                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3547                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3548         }
3549
3550         /* For BE3-R VFs, the PF programs the initial MAC address */
3551         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3552                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3553                                 &adapter->pmac_id[0], 0);
3554         return 0;
3555 }
3556
3557 static void be_schedule_worker(struct be_adapter *adapter)
3558 {
3559         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3560         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3561 }
3562
3563 static int be_setup_queues(struct be_adapter *adapter)
3564 {
3565         struct net_device *netdev = adapter->netdev;
3566         int status;
3567
3568         status = be_evt_queues_create(adapter);
3569         if (status)
3570                 goto err;
3571
3572         status = be_tx_qs_create(adapter);
3573         if (status)
3574                 goto err;
3575
3576         status = be_rx_cqs_create(adapter);
3577         if (status)
3578                 goto err;
3579
3580         status = be_mcc_queues_create(adapter);
3581         if (status)
3582                 goto err;
3583
3584         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3585         if (status)
3586                 goto err;
3587
3588         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3589         if (status)
3590                 goto err;
3591
3592         return 0;
3593 err:
3594         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3595         return status;
3596 }
3597
3598 int be_update_queues(struct be_adapter *adapter)
3599 {
3600         struct net_device *netdev = adapter->netdev;
3601         int status;
3602
3603         if (netif_running(netdev))
3604                 be_close(netdev);
3605
3606         be_cancel_worker(adapter);
3607
3608         /* If any vectors have been shared with RoCE we cannot re-program
3609          * the MSIx table.
3610          */
3611         if (!adapter->num_msix_roce_vec)
3612                 be_msix_disable(adapter);
3613
3614         be_clear_queues(adapter);
3615
3616         if (!msix_enabled(adapter)) {
3617                 status = be_msix_enable(adapter);
3618                 if (status)
3619                         return status;
3620         }
3621
3622         status = be_setup_queues(adapter);
3623         if (status)
3624                 return status;
3625
3626         be_schedule_worker(adapter);
3627
3628         if (netif_running(netdev))
3629                 status = be_open(netdev);
3630
3631         return status;
3632 }
3633
3634 static int be_setup(struct be_adapter *adapter)
3635 {
3636         struct device *dev = &adapter->pdev->dev;
3637         u32 tx_fc, rx_fc, en_flags;
3638         int status;
3639
3640         be_setup_init(adapter);
3641
3642         if (!lancer_chip(adapter))
3643                 be_cmd_req_native_mode(adapter);
3644
3645         status = be_get_config(adapter);
3646         if (status)
3647                 goto err;
3648
3649         status = be_msix_enable(adapter);
3650         if (status)
3651                 goto err;
3652
3653         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3654                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS;
3655         if (adapter->function_caps & BE_FUNCTION_CAPS_RSS)
3656                 en_flags |= BE_IF_FLAGS_RSS;
3657         en_flags = en_flags & be_if_cap_flags(adapter);
3658         status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
3659                                   &adapter->if_handle, 0);
3660         if (status)
3661                 goto err;
3662
3663         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3664         rtnl_lock();
3665         status = be_setup_queues(adapter);
3666         rtnl_unlock();
3667         if (status)
3668                 goto err;
3669
3670         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3671
3672         status = be_mac_setup(adapter);
3673         if (status)
3674                 goto err;
3675
3676         be_cmd_get_fw_ver(adapter);
3677         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3678
3679         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3680                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
3681                         adapter->fw_ver);
3682                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3683         }
3684
3685         if (adapter->vlans_added)
3686                 be_vid_config(adapter);
3687
3688         be_set_rx_mode(adapter->netdev);
3689
3690         be_cmd_get_acpi_wol_cap(adapter);
3691
3692         be_cmd_get_flow_control(adapter, &tx_fc, &rx_fc);
3693
3694         if (rx_fc != adapter->rx_fc || tx_fc != adapter->tx_fc)
3695                 be_cmd_set_flow_control(adapter, adapter->tx_fc,
3696                                         adapter->rx_fc);
3697
3698         if (be_physfn(adapter))
3699                 be_cmd_set_logical_link_config(adapter,
3700                                                IFLA_VF_LINK_STATE_AUTO, 0);
3701
3702         if (adapter->num_vfs)
3703                 be_vf_setup(adapter);
3704
3705         status = be_cmd_get_phy_info(adapter);
3706         if (!status && be_pause_supported(adapter))
3707                 adapter->phy.fc_autoneg = 1;
3708
3709         be_schedule_worker(adapter);
3710         adapter->flags |= BE_FLAGS_SETUP_DONE;
3711         return 0;
3712 err:
3713         be_clear(adapter);
3714         return status;
3715 }
3716
3717 #ifdef CONFIG_NET_POLL_CONTROLLER
3718 static void be_netpoll(struct net_device *netdev)
3719 {
3720         struct be_adapter *adapter = netdev_priv(netdev);
3721         struct be_eq_obj *eqo;
3722         int i;
3723
3724         for_all_evt_queues(adapter, eqo, i) {
3725                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3726                 napi_schedule(&eqo->napi);
3727         }
3728 }
3729 #endif
3730
3731 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3732
3733 static bool phy_flashing_required(struct be_adapter *adapter)
3734 {
3735         return (adapter->phy.phy_type == TN_8022 &&
3736                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3737 }
3738
3739 static bool is_comp_in_ufi(struct be_adapter *adapter,
3740                            struct flash_section_info *fsec, int type)
3741 {
3742         int i = 0, img_type = 0;
3743         struct flash_section_info_g2 *fsec_g2 = NULL;
3744
3745         if (BE2_chip(adapter))
3746                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3747
3748         for (i = 0; i < MAX_FLASH_COMP; i++) {
3749                 if (fsec_g2)
3750                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3751                 else
3752                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3753
3754                 if (img_type == type)
3755                         return true;
3756         }
3757         return false;
3758
3759 }
3760
3761 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3762                                                 int header_size,
3763                                                 const struct firmware *fw)
3764 {
3765         struct flash_section_info *fsec = NULL;
3766         const u8 *p = fw->data;
3767
3768         p += header_size;
3769         while (p < (fw->data + fw->size)) {
3770                 fsec = (struct flash_section_info *)p;
3771                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3772                         return fsec;
3773                 p += 32;
3774         }
3775         return NULL;
3776 }
3777
3778 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3779                               u32 img_offset, u32 img_size, int hdr_size,
3780                               u16 img_optype, bool *crc_match)
3781 {
3782         u32 crc_offset;
3783         int status;
3784         u8 crc[4];
3785
3786         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_size - 4);
3787         if (status)
3788                 return status;
3789
3790         crc_offset = hdr_size + img_offset + img_size - 4;
3791
3792         /* Skip flashing, if crc of flashed region matches */
3793         if (!memcmp(crc, p + crc_offset, 4))
3794                 *crc_match = true;
3795         else
3796                 *crc_match = false;
3797
3798         return status;
3799 }
3800
3801 static int be_flash(struct be_adapter *adapter, const u8 *img,
3802                     struct be_dma_mem *flash_cmd, int optype, int img_size)
3803 {
3804         struct be_cmd_write_flashrom *req = flash_cmd->va;
3805         u32 total_bytes, flash_op, num_bytes;
3806         int status;
3807
3808         total_bytes = img_size;
3809         while (total_bytes) {
3810                 num_bytes = min_t(u32, 32*1024, total_bytes);
3811
3812                 total_bytes -= num_bytes;
3813
3814                 if (!total_bytes) {
3815                         if (optype == OPTYPE_PHY_FW)
3816                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3817                         else
3818                                 flash_op = FLASHROM_OPER_FLASH;
3819                 } else {
3820                         if (optype == OPTYPE_PHY_FW)
3821                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3822                         else
3823                                 flash_op = FLASHROM_OPER_SAVE;
3824                 }
3825
3826                 memcpy(req->data_buf, img, num_bytes);
3827                 img += num_bytes;
3828                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3829                                                flash_op, num_bytes);
3830                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
3831                     optype == OPTYPE_PHY_FW)
3832                         break;
3833                 else if (status)
3834                         return status;
3835         }
3836         return 0;
3837 }
3838
3839 /* For BE2, BE3 and BE3-R */
3840 static int be_flash_BEx(struct be_adapter *adapter,
3841                         const struct firmware *fw,
3842                         struct be_dma_mem *flash_cmd, int num_of_images)
3843 {
3844         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3845         struct device *dev = &adapter->pdev->dev;
3846         struct flash_section_info *fsec = NULL;
3847         int status, i, filehdr_size, num_comp;
3848         const struct flash_comp *pflashcomp;
3849         bool crc_match;
3850         const u8 *p;
3851
3852         struct flash_comp gen3_flash_types[] = {
3853                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3854                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3855                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3856                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3857                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3858                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3859                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3860                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3861                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3862                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3863                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3864                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3865                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3866                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3867                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3868                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3869                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3870                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3871                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3872                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3873         };
3874
3875         struct flash_comp gen2_flash_types[] = {
3876                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3877                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3878                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3879                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3880                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3881                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3882                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3883                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3884                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3885                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3886                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3887                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3888                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3889                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3890                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3891                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3892         };
3893
3894         if (BE3_chip(adapter)) {
3895                 pflashcomp = gen3_flash_types;
3896                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3897                 num_comp = ARRAY_SIZE(gen3_flash_types);
3898         } else {
3899                 pflashcomp = gen2_flash_types;
3900                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3901                 num_comp = ARRAY_SIZE(gen2_flash_types);
3902         }
3903
3904         /* Get flash section info*/
3905         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3906         if (!fsec) {
3907                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3908                 return -1;
3909         }
3910         for (i = 0; i < num_comp; i++) {
3911                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3912                         continue;
3913
3914                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3915                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3916                         continue;
3917
3918                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3919                     !phy_flashing_required(adapter))
3920                                 continue;
3921
3922                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3923                         status = be_check_flash_crc(adapter, fw->data,
3924                                                     pflashcomp[i].offset,
3925                                                     pflashcomp[i].size,
3926                                                     filehdr_size +
3927                                                     img_hdrs_size,
3928                                                     OPTYPE_REDBOOT, &crc_match);
3929                         if (status) {
3930                                 dev_err(dev,
3931                                         "Could not get CRC for 0x%x region\n",
3932                                         pflashcomp[i].optype);
3933                                 continue;
3934                         }
3935
3936                         if (crc_match)
3937                                 continue;
3938                 }
3939
3940                 p = fw->data + filehdr_size + pflashcomp[i].offset +
3941                         img_hdrs_size;
3942                 if (p + pflashcomp[i].size > fw->data + fw->size)
3943                         return -1;
3944
3945                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3946                                   pflashcomp[i].size);
3947                 if (status) {
3948                         dev_err(dev, "Flashing section type 0x%x failed\n",
3949                                 pflashcomp[i].img_type);
3950                         return status;
3951                 }
3952         }
3953         return 0;
3954 }
3955
3956 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
3957 {
3958         u32 img_type = le32_to_cpu(fsec_entry.type);
3959         u16 img_optype = le16_to_cpu(fsec_entry.optype);
3960
3961         if (img_optype != 0xFFFF)
3962                 return img_optype;
3963
3964         switch (img_type) {
3965         case IMAGE_FIRMWARE_iSCSI:
3966                 img_optype = OPTYPE_ISCSI_ACTIVE;
3967                 break;
3968         case IMAGE_BOOT_CODE:
3969                 img_optype = OPTYPE_REDBOOT;
3970                 break;
3971         case IMAGE_OPTION_ROM_ISCSI:
3972                 img_optype = OPTYPE_BIOS;
3973                 break;
3974         case IMAGE_OPTION_ROM_PXE:
3975                 img_optype = OPTYPE_PXE_BIOS;
3976                 break;
3977         case IMAGE_OPTION_ROM_FCoE:
3978                 img_optype = OPTYPE_FCOE_BIOS;
3979                 break;
3980         case IMAGE_FIRMWARE_BACKUP_iSCSI:
3981                 img_optype = OPTYPE_ISCSI_BACKUP;
3982                 break;
3983         case IMAGE_NCSI:
3984                 img_optype = OPTYPE_NCSI_FW;
3985                 break;
3986         case IMAGE_FLASHISM_JUMPVECTOR:
3987                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
3988                 break;
3989         case IMAGE_FIRMWARE_PHY:
3990                 img_optype = OPTYPE_SH_PHY_FW;
3991                 break;
3992         case IMAGE_REDBOOT_DIR:
3993                 img_optype = OPTYPE_REDBOOT_DIR;
3994                 break;
3995         case IMAGE_REDBOOT_CONFIG:
3996                 img_optype = OPTYPE_REDBOOT_CONFIG;
3997                 break;
3998         case IMAGE_UFI_DIR:
3999                 img_optype = OPTYPE_UFI_DIR;
4000                 break;
4001         default:
4002                 break;
4003         }
4004
4005         return img_optype;
4006 }
4007
4008 static int be_flash_skyhawk(struct be_adapter *adapter,
4009                             const struct firmware *fw,
4010                             struct be_dma_mem *flash_cmd, int num_of_images)
4011 {
4012         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4013         struct device *dev = &adapter->pdev->dev;
4014         struct flash_section_info *fsec = NULL;
4015         u32 img_offset, img_size, img_type;
4016         int status, i, filehdr_size;
4017         bool crc_match, old_fw_img;
4018         u16 img_optype;
4019         const u8 *p;
4020
4021         filehdr_size = sizeof(struct flash_file_hdr_g3);
4022         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4023         if (!fsec) {
4024                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4025                 return -EINVAL;
4026         }
4027
4028         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4029                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4030                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4031                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4032                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4033                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4034
4035                 if (img_optype == 0xFFFF)
4036                         continue;
4037                 /* Don't bother verifying CRC if an old FW image is being
4038                  * flashed
4039                  */
4040                 if (old_fw_img)
4041                         goto flash;
4042
4043                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4044                                             img_size, filehdr_size +
4045                                             img_hdrs_size, img_optype,
4046                                             &crc_match);
4047                 /* The current FW image on the card does not recognize the new
4048                  * FLASH op_type. The FW download is partially complete.
4049                  * Reboot the server now to enable FW image to recognize the
4050                  * new FLASH op_type. To complete the remaining process,
4051                  * download the same FW again after the reboot.
4052                  */
4053                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4054                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4055                         dev_err(dev, "Flash incomplete. Reset the server\n");
4056                         dev_err(dev, "Download FW image again after reset\n");
4057                         return -EAGAIN;
4058                 } else if (status) {
4059                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4060                                 img_optype);
4061                         return -EFAULT;
4062                 }
4063
4064                 if (crc_match)
4065                         continue;
4066
4067 flash:
4068                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4069                 if (p + img_size > fw->data + fw->size)
4070                         return -1;
4071
4072                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
4073                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4074                  * UFI_DIR region
4075                  */
4076                 if (old_fw_img &&
4077                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4078                      (img_optype == OPTYPE_UFI_DIR &&
4079                       base_status(status) == MCC_STATUS_FAILED))) {
4080                         continue;
4081                 } else if (status) {
4082                         dev_err(dev, "Flashing section type 0x%x failed\n",
4083                                 img_type);
4084                         return -EFAULT;
4085                 }
4086         }
4087         return 0;
4088 }
4089
4090 static int lancer_fw_download(struct be_adapter *adapter,
4091                               const struct firmware *fw)
4092 {
4093 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4094 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4095         struct device *dev = &adapter->pdev->dev;
4096         struct be_dma_mem flash_cmd;
4097         const u8 *data_ptr = NULL;
4098         u8 *dest_image_ptr = NULL;
4099         size_t image_size = 0;
4100         u32 chunk_size = 0;
4101         u32 data_written = 0;
4102         u32 offset = 0;
4103         int status = 0;
4104         u8 add_status = 0;
4105         u8 change_status;
4106
4107         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4108                 dev_err(dev, "FW image size should be multiple of 4\n");
4109                 return -EINVAL;
4110         }
4111
4112         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4113                                 + LANCER_FW_DOWNLOAD_CHUNK;
4114         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4115                                           &flash_cmd.dma, GFP_KERNEL);
4116         if (!flash_cmd.va)
4117                 return -ENOMEM;
4118
4119         dest_image_ptr = flash_cmd.va +
4120                                 sizeof(struct lancer_cmd_req_write_object);
4121         image_size = fw->size;
4122         data_ptr = fw->data;
4123
4124         while (image_size) {
4125                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4126
4127                 /* Copy the image chunk content. */
4128                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4129
4130                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4131                                                  chunk_size, offset,
4132                                                  LANCER_FW_DOWNLOAD_LOCATION,
4133                                                  &data_written, &change_status,
4134                                                  &add_status);
4135                 if (status)
4136                         break;
4137
4138                 offset += data_written;
4139                 data_ptr += data_written;
4140                 image_size -= data_written;
4141         }
4142
4143         if (!status) {
4144                 /* Commit the FW written */
4145                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4146                                                  0, offset,
4147                                                  LANCER_FW_DOWNLOAD_LOCATION,
4148                                                  &data_written, &change_status,
4149                                                  &add_status);
4150         }
4151
4152         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4153         if (status) {
4154                 dev_err(dev, "Firmware load error\n");
4155                 return be_cmd_status(status);
4156         }
4157
4158         dev_info(dev, "Firmware flashed successfully\n");
4159
4160         if (change_status == LANCER_FW_RESET_NEEDED) {
4161                 dev_info(dev, "Resetting adapter to activate new FW\n");
4162                 status = lancer_physdev_ctrl(adapter,
4163                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4164                 if (status) {
4165                         dev_err(dev, "Adapter busy, could not reset FW\n");
4166                         dev_err(dev, "Reboot server to activate new FW\n");
4167                 }
4168         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4169                 dev_info(dev, "Reboot server to activate new FW\n");
4170         }
4171
4172         return 0;
4173 }
4174
4175 #define UFI_TYPE2               2
4176 #define UFI_TYPE3               3
4177 #define UFI_TYPE3R              10
4178 #define UFI_TYPE4               4
4179 static int be_get_ufi_type(struct be_adapter *adapter,
4180                            struct flash_file_hdr_g3 *fhdr)
4181 {
4182         if (!fhdr)
4183                 goto be_get_ufi_exit;
4184
4185         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
4186                 return UFI_TYPE4;
4187         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
4188                 if (fhdr->asic_type_rev == 0x10)
4189                         return UFI_TYPE3R;
4190                 else
4191                         return UFI_TYPE3;
4192         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
4193                 return UFI_TYPE2;
4194
4195 be_get_ufi_exit:
4196         dev_err(&adapter->pdev->dev,
4197                 "UFI and Interface are not compatible for flashing\n");
4198         return -1;
4199 }
4200
4201 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4202 {
4203         struct flash_file_hdr_g3 *fhdr3;
4204         struct image_hdr *img_hdr_ptr = NULL;
4205         struct be_dma_mem flash_cmd;
4206         const u8 *p;
4207         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
4208
4209         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4210         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4211                                           &flash_cmd.dma, GFP_KERNEL);
4212         if (!flash_cmd.va) {
4213                 status = -ENOMEM;
4214                 goto be_fw_exit;
4215         }
4216
4217         p = fw->data;
4218         fhdr3 = (struct flash_file_hdr_g3 *)p;
4219
4220         ufi_type = be_get_ufi_type(adapter, fhdr3);
4221
4222         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4223         for (i = 0; i < num_imgs; i++) {
4224                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4225                                 (sizeof(struct flash_file_hdr_g3) +
4226                                  i * sizeof(struct image_hdr)));
4227                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
4228                         switch (ufi_type) {
4229                         case UFI_TYPE4:
4230                                 status = be_flash_skyhawk(adapter, fw,
4231                                                           &flash_cmd, num_imgs);
4232                                 break;
4233                         case UFI_TYPE3R:
4234                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
4235                                                       num_imgs);
4236                                 break;
4237                         case UFI_TYPE3:
4238                                 /* Do not flash this ufi on BE3-R cards */
4239                                 if (adapter->asic_rev < 0x10)
4240                                         status = be_flash_BEx(adapter, fw,
4241                                                               &flash_cmd,
4242                                                               num_imgs);
4243                                 else {
4244                                         status = -EINVAL;
4245                                         dev_err(&adapter->pdev->dev,
4246                                                 "Can't load BE3 UFI on BE3R\n");
4247                                 }
4248                         }
4249                 }
4250         }
4251
4252         if (ufi_type == UFI_TYPE2)
4253                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
4254         else if (ufi_type == -1)
4255                 status = -EINVAL;
4256
4257         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4258                           flash_cmd.dma);
4259         if (status) {
4260                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
4261                 goto be_fw_exit;
4262         }
4263
4264         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4265
4266 be_fw_exit:
4267         return status;
4268 }
4269
4270 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4271 {
4272         const struct firmware *fw;
4273         int status;
4274
4275         if (!netif_running(adapter->netdev)) {
4276                 dev_err(&adapter->pdev->dev,
4277                         "Firmware load not allowed (interface is down)\n");
4278                 return -ENETDOWN;
4279         }
4280
4281         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4282         if (status)
4283                 goto fw_exit;
4284
4285         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4286
4287         if (lancer_chip(adapter))
4288                 status = lancer_fw_download(adapter, fw);
4289         else
4290                 status = be_fw_download(adapter, fw);
4291
4292         if (!status)
4293                 be_cmd_get_fw_ver(adapter);
4294
4295 fw_exit:
4296         release_firmware(fw);
4297         return status;
4298 }
4299
4300 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh)
4301 {
4302         struct be_adapter *adapter = netdev_priv(dev);
4303         struct nlattr *attr, *br_spec;
4304         int rem;
4305         int status = 0;
4306         u16 mode = 0;
4307
4308         if (!sriov_enabled(adapter))
4309                 return -EOPNOTSUPP;
4310
4311         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4312
4313         nla_for_each_nested(attr, br_spec, rem) {
4314                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4315                         continue;
4316
4317                 mode = nla_get_u16(attr);
4318                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4319                         return -EINVAL;
4320
4321                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4322                                                adapter->if_handle,
4323                                                mode == BRIDGE_MODE_VEPA ?
4324                                                PORT_FWD_TYPE_VEPA :
4325                                                PORT_FWD_TYPE_VEB);
4326                 if (status)
4327                         goto err;
4328
4329                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4330                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4331
4332                 return status;
4333         }
4334 err:
4335         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4336                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4337
4338         return status;
4339 }
4340
4341 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4342                                  struct net_device *dev, u32 filter_mask)
4343 {
4344         struct be_adapter *adapter = netdev_priv(dev);
4345         int status = 0;
4346         u8 hsw_mode;
4347
4348         if (!sriov_enabled(adapter))
4349                 return 0;
4350
4351         /* BE and Lancer chips support VEB mode only */
4352         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4353                 hsw_mode = PORT_FWD_TYPE_VEB;
4354         } else {
4355                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4356                                                adapter->if_handle, &hsw_mode);
4357                 if (status)
4358                         return 0;
4359         }
4360
4361         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4362                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4363                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB);
4364 }
4365
4366 #ifdef CONFIG_BE2NET_VXLAN
4367 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4368                               __be16 port)
4369 {
4370         struct be_adapter *adapter = netdev_priv(netdev);
4371         struct device *dev = &adapter->pdev->dev;
4372         int status;
4373
4374         if (lancer_chip(adapter) || BEx_chip(adapter))
4375                 return;
4376
4377         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4378                 dev_warn(dev, "Cannot add UDP port %d for VxLAN offloads\n",
4379                          be16_to_cpu(port));
4380                 dev_info(dev,
4381                          "Only one UDP port supported for VxLAN offloads\n");
4382                 return;
4383         }
4384
4385         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4386                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4387         if (status) {
4388                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4389                 goto err;
4390         }
4391
4392         status = be_cmd_set_vxlan_port(adapter, port);
4393         if (status) {
4394                 dev_warn(dev, "Failed to add VxLAN port\n");
4395                 goto err;
4396         }
4397         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4398         adapter->vxlan_port = port;
4399
4400         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4401                  be16_to_cpu(port));
4402         return;
4403 err:
4404         be_disable_vxlan_offloads(adapter);
4405 }
4406
4407 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4408                               __be16 port)
4409 {
4410         struct be_adapter *adapter = netdev_priv(netdev);
4411
4412         if (lancer_chip(adapter) || BEx_chip(adapter))
4413                 return;
4414
4415         if (adapter->vxlan_port != port)
4416                 return;
4417
4418         be_disable_vxlan_offloads(adapter);
4419
4420         dev_info(&adapter->pdev->dev,
4421                  "Disabled VxLAN offloads for UDP port %d\n",
4422                  be16_to_cpu(port));
4423 }
4424 #endif
4425
4426 static const struct net_device_ops be_netdev_ops = {
4427         .ndo_open               = be_open,
4428         .ndo_stop               = be_close,
4429         .ndo_start_xmit         = be_xmit,
4430         .ndo_set_rx_mode        = be_set_rx_mode,
4431         .ndo_set_mac_address    = be_mac_addr_set,
4432         .ndo_change_mtu         = be_change_mtu,
4433         .ndo_get_stats64        = be_get_stats64,
4434         .ndo_validate_addr      = eth_validate_addr,
4435         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4436         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4437         .ndo_set_vf_mac         = be_set_vf_mac,
4438         .ndo_set_vf_vlan        = be_set_vf_vlan,
4439         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4440         .ndo_get_vf_config      = be_get_vf_config,
4441         .ndo_set_vf_link_state  = be_set_vf_link_state,
4442 #ifdef CONFIG_NET_POLL_CONTROLLER
4443         .ndo_poll_controller    = be_netpoll,
4444 #endif
4445         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4446         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4447 #ifdef CONFIG_NET_RX_BUSY_POLL
4448         .ndo_busy_poll          = be_busy_poll,
4449 #endif
4450 #ifdef CONFIG_BE2NET_VXLAN
4451         .ndo_add_vxlan_port     = be_add_vxlan_port,
4452         .ndo_del_vxlan_port     = be_del_vxlan_port,
4453 #endif
4454 };
4455
4456 static void be_netdev_init(struct net_device *netdev)
4457 {
4458         struct be_adapter *adapter = netdev_priv(netdev);
4459
4460         if (skyhawk_chip(adapter)) {
4461                 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4462                                            NETIF_F_TSO | NETIF_F_TSO6 |
4463                                            NETIF_F_GSO_UDP_TUNNEL;
4464                 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4465         }
4466         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4467                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4468                 NETIF_F_HW_VLAN_CTAG_TX;
4469         if (be_multi_rxq(adapter))
4470                 netdev->hw_features |= NETIF_F_RXHASH;
4471
4472         netdev->features |= netdev->hw_features |
4473                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4474
4475         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4476                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4477
4478         netdev->priv_flags |= IFF_UNICAST_FLT;
4479
4480         netdev->flags |= IFF_MULTICAST;
4481
4482         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4483
4484         netdev->netdev_ops = &be_netdev_ops;
4485
4486         netdev->ethtool_ops = &be_ethtool_ops;
4487 }
4488
4489 static void be_unmap_pci_bars(struct be_adapter *adapter)
4490 {
4491         if (adapter->csr)
4492                 pci_iounmap(adapter->pdev, adapter->csr);
4493         if (adapter->db)
4494                 pci_iounmap(adapter->pdev, adapter->db);
4495 }
4496
4497 static int db_bar(struct be_adapter *adapter)
4498 {
4499         if (lancer_chip(adapter) || !be_physfn(adapter))
4500                 return 0;
4501         else
4502                 return 4;
4503 }
4504
4505 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4506 {
4507         if (skyhawk_chip(adapter)) {
4508                 adapter->roce_db.size = 4096;
4509                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4510                                                               db_bar(adapter));
4511                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4512                                                                db_bar(adapter));
4513         }
4514         return 0;
4515 }
4516
4517 static int be_map_pci_bars(struct be_adapter *adapter)
4518 {
4519         u8 __iomem *addr;
4520
4521         if (BEx_chip(adapter) && be_physfn(adapter)) {
4522                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4523                 if (!adapter->csr)
4524                         return -ENOMEM;
4525         }
4526
4527         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4528         if (!addr)
4529                 goto pci_map_err;
4530         adapter->db = addr;
4531
4532         be_roce_map_pci_bars(adapter);
4533         return 0;
4534
4535 pci_map_err:
4536         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4537         be_unmap_pci_bars(adapter);
4538         return -ENOMEM;
4539 }
4540
4541 static void be_ctrl_cleanup(struct be_adapter *adapter)
4542 {
4543         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4544
4545         be_unmap_pci_bars(adapter);
4546
4547         if (mem->va)
4548                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4549                                   mem->dma);
4550
4551         mem = &adapter->rx_filter;
4552         if (mem->va)
4553                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4554                                   mem->dma);
4555 }
4556
4557 static int be_ctrl_init(struct be_adapter *adapter)
4558 {
4559         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4560         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4561         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4562         u32 sli_intf;
4563         int status;
4564
4565         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4566         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4567                                  SLI_INTF_FAMILY_SHIFT;
4568         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4569
4570         status = be_map_pci_bars(adapter);
4571         if (status)
4572                 goto done;
4573
4574         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4575         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4576                                                 mbox_mem_alloc->size,
4577                                                 &mbox_mem_alloc->dma,
4578                                                 GFP_KERNEL);
4579         if (!mbox_mem_alloc->va) {
4580                 status = -ENOMEM;
4581                 goto unmap_pci_bars;
4582         }
4583         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4584         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4585         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4586         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4587
4588         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4589         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4590                                             rx_filter->size, &rx_filter->dma,
4591                                             GFP_KERNEL);
4592         if (!rx_filter->va) {
4593                 status = -ENOMEM;
4594                 goto free_mbox;
4595         }
4596
4597         mutex_init(&adapter->mbox_lock);
4598         spin_lock_init(&adapter->mcc_lock);
4599         spin_lock_init(&adapter->mcc_cq_lock);
4600
4601         init_completion(&adapter->et_cmd_compl);
4602         pci_save_state(adapter->pdev);
4603         return 0;
4604
4605 free_mbox:
4606         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4607                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4608
4609 unmap_pci_bars:
4610         be_unmap_pci_bars(adapter);
4611
4612 done:
4613         return status;
4614 }
4615
4616 static void be_stats_cleanup(struct be_adapter *adapter)
4617 {
4618         struct be_dma_mem *cmd = &adapter->stats_cmd;
4619
4620         if (cmd->va)
4621                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4622                                   cmd->va, cmd->dma);
4623 }
4624
4625 static int be_stats_init(struct be_adapter *adapter)
4626 {
4627         struct be_dma_mem *cmd = &adapter->stats_cmd;
4628
4629         if (lancer_chip(adapter))
4630                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4631         else if (BE2_chip(adapter))
4632                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4633         else if (BE3_chip(adapter))
4634                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4635         else
4636                 /* ALL non-BE ASICs */
4637                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4638
4639         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4640                                       GFP_KERNEL);
4641         if (!cmd->va)
4642                 return -ENOMEM;
4643         return 0;
4644 }
4645
4646 static void be_remove(struct pci_dev *pdev)
4647 {
4648         struct be_adapter *adapter = pci_get_drvdata(pdev);
4649
4650         if (!adapter)
4651                 return;
4652
4653         be_roce_dev_remove(adapter);
4654         be_intr_set(adapter, false);
4655
4656         cancel_delayed_work_sync(&adapter->func_recovery_work);
4657
4658         unregister_netdev(adapter->netdev);
4659
4660         be_clear(adapter);
4661
4662         /* tell fw we're done with firing cmds */
4663         be_cmd_fw_clean(adapter);
4664
4665         be_stats_cleanup(adapter);
4666
4667         be_ctrl_cleanup(adapter);
4668
4669         pci_disable_pcie_error_reporting(pdev);
4670
4671         pci_release_regions(pdev);
4672         pci_disable_device(pdev);
4673
4674         free_netdev(adapter->netdev);
4675 }
4676
4677 static int be_get_initial_config(struct be_adapter *adapter)
4678 {
4679         int status, level;
4680
4681         status = be_cmd_get_cntl_attributes(adapter);
4682         if (status)
4683                 return status;
4684
4685         /* Must be a power of 2 or else MODULO will BUG_ON */
4686         adapter->be_get_temp_freq = 64;
4687
4688         if (BEx_chip(adapter)) {
4689                 level = be_cmd_get_fw_log_level(adapter);
4690                 adapter->msg_enable =
4691                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4692         }
4693
4694         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4695         return 0;
4696 }
4697
4698 static int lancer_recover_func(struct be_adapter *adapter)
4699 {
4700         struct device *dev = &adapter->pdev->dev;
4701         int status;
4702
4703         status = lancer_test_and_set_rdy_state(adapter);
4704         if (status)
4705                 goto err;
4706
4707         if (netif_running(adapter->netdev))
4708                 be_close(adapter->netdev);
4709
4710         be_clear(adapter);
4711
4712         be_clear_all_error(adapter);
4713
4714         status = be_setup(adapter);
4715         if (status)
4716                 goto err;
4717
4718         if (netif_running(adapter->netdev)) {
4719                 status = be_open(adapter->netdev);
4720                 if (status)
4721                         goto err;
4722         }
4723
4724         dev_err(dev, "Adapter recovery successful\n");
4725         return 0;
4726 err:
4727         if (status == -EAGAIN)
4728                 dev_err(dev, "Waiting for resource provisioning\n");
4729         else
4730                 dev_err(dev, "Adapter recovery failed\n");
4731
4732         return status;
4733 }
4734
4735 static void be_func_recovery_task(struct work_struct *work)
4736 {
4737         struct be_adapter *adapter =
4738                 container_of(work, struct be_adapter,  func_recovery_work.work);
4739         int status = 0;
4740
4741         be_detect_error(adapter);
4742
4743         if (adapter->hw_error && lancer_chip(adapter)) {
4744
4745                 rtnl_lock();
4746                 netif_device_detach(adapter->netdev);
4747                 rtnl_unlock();
4748
4749                 status = lancer_recover_func(adapter);
4750                 if (!status)
4751                         netif_device_attach(adapter->netdev);
4752         }
4753
4754         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4755          * no need to attempt further recovery.
4756          */
4757         if (!status || status == -EAGAIN)
4758                 schedule_delayed_work(&adapter->func_recovery_work,
4759                                       msecs_to_jiffies(1000));
4760 }
4761
4762 static void be_worker(struct work_struct *work)
4763 {
4764         struct be_adapter *adapter =
4765                 container_of(work, struct be_adapter, work.work);
4766         struct be_rx_obj *rxo;
4767         int i;
4768
4769         /* when interrupts are not yet enabled, just reap any pending
4770         * mcc completions */
4771         if (!netif_running(adapter->netdev)) {
4772                 local_bh_disable();
4773                 be_process_mcc(adapter);
4774                 local_bh_enable();
4775                 goto reschedule;
4776         }
4777
4778         if (!adapter->stats_cmd_sent) {
4779                 if (lancer_chip(adapter))
4780                         lancer_cmd_get_pport_stats(adapter,
4781                                                 &adapter->stats_cmd);
4782                 else
4783                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4784         }
4785
4786         if (be_physfn(adapter) &&
4787             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4788                 be_cmd_get_die_temperature(adapter);
4789
4790         for_all_rx_queues(adapter, rxo, i) {
4791                 /* Replenish RX-queues starved due to memory
4792                  * allocation failures.
4793                  */
4794                 if (rxo->rx_post_starved)
4795                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
4796         }
4797
4798         be_eqd_update(adapter);
4799
4800 reschedule:
4801         adapter->work_counter++;
4802         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4803 }
4804
4805 /* If any VFs are already enabled don't FLR the PF */
4806 static bool be_reset_required(struct be_adapter *adapter)
4807 {
4808         return pci_num_vf(adapter->pdev) ? false : true;
4809 }
4810
4811 static char *mc_name(struct be_adapter *adapter)
4812 {
4813         char *str = ""; /* default */
4814
4815         switch (adapter->mc_type) {
4816         case UMC:
4817                 str = "UMC";
4818                 break;
4819         case FLEX10:
4820                 str = "FLEX10";
4821                 break;
4822         case vNIC1:
4823                 str = "vNIC-1";
4824                 break;
4825         case nPAR:
4826                 str = "nPAR";
4827                 break;
4828         case UFP:
4829                 str = "UFP";
4830                 break;
4831         case vNIC2:
4832                 str = "vNIC-2";
4833                 break;
4834         default:
4835                 str = "";
4836         }
4837
4838         return str;
4839 }
4840
4841 static inline char *func_name(struct be_adapter *adapter)
4842 {
4843         return be_physfn(adapter) ? "PF" : "VF";
4844 }
4845
4846 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4847 {
4848         int status = 0;
4849         struct be_adapter *adapter;
4850         struct net_device *netdev;
4851         char port_name;
4852
4853         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
4854
4855         status = pci_enable_device(pdev);
4856         if (status)
4857                 goto do_none;
4858
4859         status = pci_request_regions(pdev, DRV_NAME);
4860         if (status)
4861                 goto disable_dev;
4862         pci_set_master(pdev);
4863
4864         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4865         if (!netdev) {
4866                 status = -ENOMEM;
4867                 goto rel_reg;
4868         }
4869         adapter = netdev_priv(netdev);
4870         adapter->pdev = pdev;
4871         pci_set_drvdata(pdev, adapter);
4872         adapter->netdev = netdev;
4873         SET_NETDEV_DEV(netdev, &pdev->dev);
4874
4875         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4876         if (!status) {
4877                 netdev->features |= NETIF_F_HIGHDMA;
4878         } else {
4879                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4880                 if (status) {
4881                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4882                         goto free_netdev;
4883                 }
4884         }
4885
4886         status = pci_enable_pcie_error_reporting(pdev);
4887         if (!status)
4888                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4889
4890         status = be_ctrl_init(adapter);
4891         if (status)
4892                 goto free_netdev;
4893
4894         /* sync up with fw's ready state */
4895         if (be_physfn(adapter)) {
4896                 status = be_fw_wait_ready(adapter);
4897                 if (status)
4898                         goto ctrl_clean;
4899         }
4900
4901         if (be_reset_required(adapter)) {
4902                 status = be_cmd_reset_function(adapter);
4903                 if (status)
4904                         goto ctrl_clean;
4905
4906                 /* Wait for interrupts to quiesce after an FLR */
4907                 msleep(100);
4908         }
4909
4910         /* Allow interrupts for other ULPs running on NIC function */
4911         be_intr_set(adapter, true);
4912
4913         /* tell fw we're ready to fire cmds */
4914         status = be_cmd_fw_init(adapter);
4915         if (status)
4916                 goto ctrl_clean;
4917
4918         status = be_stats_init(adapter);
4919         if (status)
4920                 goto ctrl_clean;
4921
4922         status = be_get_initial_config(adapter);
4923         if (status)
4924                 goto stats_clean;
4925
4926         INIT_DELAYED_WORK(&adapter->work, be_worker);
4927         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
4928         adapter->rx_fc = adapter->tx_fc = true;
4929
4930         status = be_setup(adapter);
4931         if (status)
4932                 goto stats_clean;
4933
4934         be_netdev_init(netdev);
4935         status = register_netdev(netdev);
4936         if (status != 0)
4937                 goto unsetup;
4938
4939         be_roce_dev_add(adapter);
4940
4941         schedule_delayed_work(&adapter->func_recovery_work,
4942                               msecs_to_jiffies(1000));
4943
4944         be_cmd_query_port_name(adapter, &port_name);
4945
4946         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
4947                  func_name(adapter), mc_name(adapter), port_name);
4948
4949         return 0;
4950
4951 unsetup:
4952         be_clear(adapter);
4953 stats_clean:
4954         be_stats_cleanup(adapter);
4955 ctrl_clean:
4956         be_ctrl_cleanup(adapter);
4957 free_netdev:
4958         free_netdev(netdev);
4959 rel_reg:
4960         pci_release_regions(pdev);
4961 disable_dev:
4962         pci_disable_device(pdev);
4963 do_none:
4964         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
4965         return status;
4966 }
4967
4968 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
4969 {
4970         struct be_adapter *adapter = pci_get_drvdata(pdev);
4971         struct net_device *netdev =  adapter->netdev;
4972
4973         if (adapter->wol_en)
4974                 be_setup_wol(adapter, true);
4975
4976         be_intr_set(adapter, false);
4977         cancel_delayed_work_sync(&adapter->func_recovery_work);
4978
4979         netif_device_detach(netdev);
4980         if (netif_running(netdev)) {
4981                 rtnl_lock();
4982                 be_close(netdev);
4983                 rtnl_unlock();
4984         }
4985         be_clear(adapter);
4986
4987         pci_save_state(pdev);
4988         pci_disable_device(pdev);
4989         pci_set_power_state(pdev, pci_choose_state(pdev, state));
4990         return 0;
4991 }
4992
4993 static int be_resume(struct pci_dev *pdev)
4994 {
4995         int status = 0;
4996         struct be_adapter *adapter = pci_get_drvdata(pdev);
4997         struct net_device *netdev =  adapter->netdev;
4998
4999         netif_device_detach(netdev);
5000
5001         status = pci_enable_device(pdev);
5002         if (status)
5003                 return status;
5004
5005         pci_set_power_state(pdev, PCI_D0);
5006         pci_restore_state(pdev);
5007
5008         status = be_fw_wait_ready(adapter);
5009         if (status)
5010                 return status;
5011
5012         be_intr_set(adapter, true);
5013         /* tell fw we're ready to fire cmds */
5014         status = be_cmd_fw_init(adapter);
5015         if (status)
5016                 return status;
5017
5018         be_setup(adapter);
5019         if (netif_running(netdev)) {
5020                 rtnl_lock();
5021                 be_open(netdev);
5022                 rtnl_unlock();
5023         }
5024
5025         schedule_delayed_work(&adapter->func_recovery_work,
5026                               msecs_to_jiffies(1000));
5027         netif_device_attach(netdev);
5028
5029         if (adapter->wol_en)
5030                 be_setup_wol(adapter, false);
5031
5032         return 0;
5033 }
5034
5035 /*
5036  * An FLR will stop BE from DMAing any data.
5037  */
5038 static void be_shutdown(struct pci_dev *pdev)
5039 {
5040         struct be_adapter *adapter = pci_get_drvdata(pdev);
5041
5042         if (!adapter)
5043                 return;
5044
5045         be_roce_dev_shutdown(adapter);
5046         cancel_delayed_work_sync(&adapter->work);
5047         cancel_delayed_work_sync(&adapter->func_recovery_work);
5048
5049         netif_device_detach(adapter->netdev);
5050
5051         be_cmd_reset_function(adapter);
5052
5053         pci_disable_device(pdev);
5054 }
5055
5056 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5057                                             pci_channel_state_t state)
5058 {
5059         struct be_adapter *adapter = pci_get_drvdata(pdev);
5060         struct net_device *netdev =  adapter->netdev;
5061
5062         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5063
5064         if (!adapter->eeh_error) {
5065                 adapter->eeh_error = true;
5066
5067                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5068
5069                 rtnl_lock();
5070                 netif_device_detach(netdev);
5071                 if (netif_running(netdev))
5072                         be_close(netdev);
5073                 rtnl_unlock();
5074
5075                 be_clear(adapter);
5076         }
5077
5078         if (state == pci_channel_io_perm_failure)
5079                 return PCI_ERS_RESULT_DISCONNECT;
5080
5081         pci_disable_device(pdev);
5082
5083         /* The error could cause the FW to trigger a flash debug dump.
5084          * Resetting the card while flash dump is in progress
5085          * can cause it not to recover; wait for it to finish.
5086          * Wait only for first function as it is needed only once per
5087          * adapter.
5088          */
5089         if (pdev->devfn == 0)
5090                 ssleep(30);
5091
5092         return PCI_ERS_RESULT_NEED_RESET;
5093 }
5094
5095 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5096 {
5097         struct be_adapter *adapter = pci_get_drvdata(pdev);
5098         int status;
5099
5100         dev_info(&adapter->pdev->dev, "EEH reset\n");
5101
5102         status = pci_enable_device(pdev);
5103         if (status)
5104                 return PCI_ERS_RESULT_DISCONNECT;
5105
5106         pci_set_master(pdev);
5107         pci_set_power_state(pdev, PCI_D0);
5108         pci_restore_state(pdev);
5109
5110         /* Check if card is ok and fw is ready */
5111         dev_info(&adapter->pdev->dev,
5112                  "Waiting for FW to be ready after EEH reset\n");
5113         status = be_fw_wait_ready(adapter);
5114         if (status)
5115                 return PCI_ERS_RESULT_DISCONNECT;
5116
5117         pci_cleanup_aer_uncorrect_error_status(pdev);
5118         be_clear_all_error(adapter);
5119         return PCI_ERS_RESULT_RECOVERED;
5120 }
5121
5122 static void be_eeh_resume(struct pci_dev *pdev)
5123 {
5124         int status = 0;
5125         struct be_adapter *adapter = pci_get_drvdata(pdev);
5126         struct net_device *netdev =  adapter->netdev;
5127
5128         dev_info(&adapter->pdev->dev, "EEH resume\n");
5129
5130         pci_save_state(pdev);
5131
5132         status = be_cmd_reset_function(adapter);
5133         if (status)
5134                 goto err;
5135
5136         /* On some BE3 FW versions, after a HW reset,
5137          * interrupts will remain disabled for each function.
5138          * So, explicitly enable interrupts
5139          */
5140         be_intr_set(adapter, true);
5141
5142         /* tell fw we're ready to fire cmds */
5143         status = be_cmd_fw_init(adapter);
5144         if (status)
5145                 goto err;
5146
5147         status = be_setup(adapter);
5148         if (status)
5149                 goto err;
5150
5151         if (netif_running(netdev)) {
5152                 status = be_open(netdev);
5153                 if (status)
5154                         goto err;
5155         }
5156
5157         schedule_delayed_work(&adapter->func_recovery_work,
5158                               msecs_to_jiffies(1000));
5159         netif_device_attach(netdev);
5160         return;
5161 err:
5162         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5163 }
5164
5165 static const struct pci_error_handlers be_eeh_handlers = {
5166         .error_detected = be_eeh_err_detected,
5167         .slot_reset = be_eeh_reset,
5168         .resume = be_eeh_resume,
5169 };
5170
5171 static struct pci_driver be_driver = {
5172         .name = DRV_NAME,
5173         .id_table = be_dev_ids,
5174         .probe = be_probe,
5175         .remove = be_remove,
5176         .suspend = be_suspend,
5177         .resume = be_resume,
5178         .shutdown = be_shutdown,
5179         .err_handler = &be_eeh_handlers
5180 };
5181
5182 static int __init be_init_module(void)
5183 {
5184         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5185             rx_frag_size != 2048) {
5186                 printk(KERN_WARNING DRV_NAME
5187                         " : Module param rx_frag_size must be 2048/4096/8192."
5188                         " Using 2048\n");
5189                 rx_frag_size = 2048;
5190         }
5191
5192         return pci_register_driver(&be_driver);
5193 }
5194 module_init(be_init_module);
5195
5196 static void __exit be_exit_module(void)
5197 {
5198         pci_unregister_driver(&be_driver);
5199 }
5200 module_exit(be_exit_module);