drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2017 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 #include <linux/etherdevice.h>
  28 #include <linux/of_net.h>
  29 #include <linux/pci.h>
  30 #include <linux/bpf.h>
  31
  32 /* Local includes */
  33 #include "i40e.h"
  34 #include "i40e_diag.h"
  35 #include <net/udp_tunnel.h>
  36 /* All i40e tracepoints are defined by the include below, which
  37  * must be included exactly once across the whole kernel with
  38  * CREATE_TRACE_POINTS defined
  39  */
  40 #define CREATE_TRACE_POINTS
  41 #include "i40e_trace.h"
  42
  43 const char i40e_driver_name[] = "i40e";
  44 static const char i40e_driver_string[] =
  45                         "Intel(R) Ethernet Connection XL710 Network Driver";
  46
  47 #define DRV_KERN "-k"
  48
  49 #define DRV_VERSION_MAJOR 2
  50 #define DRV_VERSION_MINOR 1
  51 #define DRV_VERSION_BUILD 14
  52 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  53              __stringify(DRV_VERSION_MINOR) "." \
  54              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  55 const char i40e_driver_version_str[] = DRV_VERSION;
  56 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  57
  58 /* a bit of forward declarations */
  59 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  60 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  61 static int i40e_add_vsi(struct i40e_vsi *vsi);
  62 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  63 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  64 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  65 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  66 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  67 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
  68 static int i40e_reset(struct i40e_pf *pf);
  69 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  70 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  71 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  72
  73 /* i40e_pci_tbl - PCI Device ID Table
  74  *
  75  * Last entry must be all 0s
  76  *
  77  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  78  *   Class, Class Mask, private data (not used) }
  79  */
  80 static const struct pci_device_id i40e_pci_tbl[] = {
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  86         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  87         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  88         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  89         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  90         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  91         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  92         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  93         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  94         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  95         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  96         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  97         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  98         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  99         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
 100         /* required last entry */
 101         {0, }
 102 };
 103 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
 104
 105 #define I40E_MAX_VF_COUNT 128
 106 static int debug = -1;
 107 module_param(debug, uint, 0);
 108 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
 109
 110 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 111 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
 112 MODULE_LICENSE("GPL");
 113 MODULE_VERSION(DRV_VERSION);
 114
 115 static struct workqueue_struct *i40e_wq;
 116
 117 /**
 118  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 119  * @hw:   pointer to the HW structure
 120  * @mem:  ptr to mem struct to fill out
 121  * @size: size of memory requested
 122  * @alignment: what to align the allocation to
 123  **/
 124 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 125                             u64 size, u32 alignment)
 126 {
 127         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 128
 129         mem->size = ALIGN(size, alignment);
 130         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 131                                       &mem->pa, GFP_KERNEL);
 132         if (!mem->va)
 133                 return -ENOMEM;
 134
 135         return 0;
 136 }
 137
 138 /**
 139  * i40e_free_dma_mem_d - OS specific memory free for shared code
 140  * @hw:   pointer to the HW structure
 141  * @mem:  ptr to mem struct to free
 142  **/
 143 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 144 {
 145         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 146
 147         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 148         mem->va = NULL;
 149         mem->pa = 0;
 150         mem->size = 0;
 151
 152         return 0;
 153 }
 154
 155 /**
 156  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 157  * @hw:   pointer to the HW structure
 158  * @mem:  ptr to mem struct to fill out
 159  * @size: size of memory requested
 160  **/
 161 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 162                              u32 size)
 163 {
 164         mem->size = size;
 165         mem->va = kzalloc(size, GFP_KERNEL);
 166
 167         if (!mem->va)
 168                 return -ENOMEM;
 169
 170         return 0;
 171 }
 172
 173 /**
 174  * i40e_free_virt_mem_d - OS specific memory free for shared code
 175  * @hw:   pointer to the HW structure
 176  * @mem:  ptr to mem struct to free
 177  **/
 178 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 179 {
 180         /* it's ok to kfree a NULL pointer */
 181         kfree(mem->va);
 182         mem->va = NULL;
 183         mem->size = 0;
 184
 185         return 0;
 186 }
 187
 188 /**
 189  * i40e_get_lump - find a lump of free generic resource
 190  * @pf: board private structure
 191  * @pile: the pile of resource to search
 192  * @needed: the number of items needed
 193  * @id: an owner id to stick on the items assigned
 194  *
 195  * Returns the base item index of the lump, or negative for error
 196  *
 197  * The search_hint trick and lack of advanced fit-finding only work
 198  * because we're highly likely to have all the same size lump requests.
 199  * Linear search time and any fragmentation should be minimal.
 200  **/
 201 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 202                          u16 needed, u16 id)
 203 {
 204         int ret = -ENOMEM;
 205         int i, j;
 206
 207         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 208                 dev_info(&pf->pdev->dev,
 209                          "param err: pile=%p needed=%d id=0x%04x\n",
 210                          pile, needed, id);
 211                 return -EINVAL;
 212         }
 213
 214         /* start the linear search with an imperfect hint */
 215         i = pile->search_hint;
 216         while (i < pile->num_entries) {
 217                 /* skip already allocated entries */
 218                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 219                         i++;
 220                         continue;
 221                 }
 222
 223                 /* do we have enough in this lump? */
 224                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 225                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 226                                 break;
 227                 }
 228
 229                 if (j == needed) {
 230                         /* there was enough, so assign it to the requestor */
 231                         for (j = 0; j < needed; j++)
 232                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 233                         ret = i;
 234                         pile->search_hint = i + j;
 235                         break;
 236                 }
 237
 238                 /* not enough, so skip over it and continue looking */
 239                 i += j;
 240         }
 241
 242         return ret;
 243 }
 244
 245 /**
 246  * i40e_put_lump - return a lump of generic resource
 247  * @pile: the pile of resource to search
 248  * @index: the base item index
 249  * @id: the owner id of the items assigned
 250  *
 251  * Returns the count of items in the lump
 252  **/
 253 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 254 {
 255         int valid_id = (id | I40E_PILE_VALID_BIT);
 256         int count = 0;
 257         int i;
 258
 259         if (!pile || index >= pile->num_entries)
 260                 return -EINVAL;
 261
 262         for (i = index;
 263              i < pile->num_entries && pile->list[i] == valid_id;
 264              i++) {
 265                 pile->list[i] = 0;
 266                 count++;
 267         }
 268
 269         if (count && index < pile->search_hint)
 270                 pile->search_hint = index;
 271
 272         return count;
 273 }
 274
 275 /**
 276  * i40e_find_vsi_from_id - searches for the vsi with the given id
 277  * @pf - the pf structure to search for the vsi
 278  * @id - id of the vsi it is searching for
 279  **/
 280 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 281 {
 282         int i;
 283
 284         for (i = 0; i < pf->num_alloc_vsi; i++)
 285                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 286                         return pf->vsi[i];
 287
 288         return NULL;
 289 }
 290
 291 /**
 292  * i40e_service_event_schedule - Schedule the service task to wake up
 293  * @pf: board private structure
 294  *
 295  * If not already scheduled, this puts the task into the work queue
 296  **/
 297 void i40e_service_event_schedule(struct i40e_pf *pf)
 298 {
 299         if (!test_bit(__I40E_DOWN, pf->state) &&
 300             !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
 301                 queue_work(i40e_wq, &pf->service_task);
 302 }
 303
 304 /**
 305  * i40e_tx_timeout - Respond to a Tx Hang
 306  * @netdev: network interface device structure
 307  *
 308  * If any port has noticed a Tx timeout, it is likely that the whole
 309  * device is munged, not just the one netdev port, so go for the full
 310  * reset.
 311  **/
 312 static void i40e_tx_timeout(struct net_device *netdev)
 313 {
 314         struct i40e_netdev_priv *np = netdev_priv(netdev);
 315         struct i40e_vsi *vsi = np->vsi;
 316         struct i40e_pf *pf = vsi->back;
 317         struct i40e_ring *tx_ring = NULL;
 318         unsigned int i, hung_queue = 0;
 319         u32 head, val;
 320
 321         pf->tx_timeout_count++;
 322
 323         /* find the stopped queue the same way the stack does */
 324         for (i = 0; i < netdev->num_tx_queues; i++) {
 325                 struct netdev_queue *q;
 326                 unsigned long trans_start;
 327
 328                 q = netdev_get_tx_queue(netdev, i);
 329                 trans_start = q->trans_start;
 330                 if (netif_xmit_stopped(q) &&
 331                     time_after(jiffies,
 332                                (trans_start + netdev->watchdog_timeo))) {
 333                         hung_queue = i;
 334                         break;
 335                 }
 336         }
 337
 338         if (i == netdev->num_tx_queues) {
 339                 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
 340         } else {
 341                 /* now that we have an index, find the tx_ring struct */
 342                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 343                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 344                                 if (hung_queue ==
 345                                     vsi->tx_rings[i]->queue_index) {
 346                                         tx_ring = vsi->tx_rings[i];
 347                                         break;
 348                                 }
 349                         }
 350                 }
 351         }
 352
 353         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 354                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 355         else if (time_before(jiffies,
 356                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 357                 return;   /* don't do any new action before the next timeout */
 358
 359         if (tx_ring) {
 360                 head = i40e_get_head(tx_ring);
 361                 /* Read interrupt register */
 362                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 363                         val = rd32(&pf->hw,
 364                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 365                                                 tx_ring->vsi->base_vector - 1));
 366                 else
 367                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 368
 369                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 370                             vsi->seid, hung_queue, tx_ring->next_to_clean,
 371                             head, tx_ring->next_to_use,
 372                             readl(tx_ring->tail), val);
 373         }
 374
 375         pf->tx_timeout_last_recovery = jiffies;
 376         netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
 377                     pf->tx_timeout_recovery_level, hung_queue);
 378
 379         switch (pf->tx_timeout_recovery_level) {
 380         case 1:
 381                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
 382                 break;
 383         case 2:
 384                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
 385                 break;
 386         case 3:
 387                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
 388                 break;
 389         default:
 390                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 391                 break;
 392         }
 393
 394         i40e_service_event_schedule(pf);
 395         pf->tx_timeout_recovery_level++;
 396 }
 397
 398 /**
 399  * i40e_get_vsi_stats_struct - Get System Network Statistics
 400  * @vsi: the VSI we care about
 401  *
 402  * Returns the address of the device statistics structure.
 403  * The statistics are actually updated from the service task.
 404  **/
 405 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 406 {
 407         return &vsi->net_stats;
 408 }
 409
 410 /**
 411  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
 412  * @ring: Tx ring to get statistics from
 413  * @stats: statistics entry to be updated
 414  **/
 415 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 416                                             struct rtnl_link_stats64 *stats)
 417 {
 418         u64 bytes, packets;
 419         unsigned int start;
 420
 421         do {
 422                 start = u64_stats_fetch_begin_irq(&ring->syncp);
 423                 packets = ring->stats.packets;
 424                 bytes   = ring->stats.bytes;
 425         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 426
 427         stats->tx_packets += packets;
 428         stats->tx_bytes   += bytes;
 429 }
 430
 431 /**
 432  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 433  * @netdev: network interface device structure
 434  *
 435  * Returns the address of the device statistics structure.
 436  * The statistics are actually updated from the service task.
 437  **/
 438 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 439                                   struct rtnl_link_stats64 *stats)
 440 {
 441         struct i40e_netdev_priv *np = netdev_priv(netdev);
 442         struct i40e_ring *tx_ring, *rx_ring;
 443         struct i40e_vsi *vsi = np->vsi;
 444         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 445         int i;
 446
 447         if (test_bit(__I40E_VSI_DOWN, vsi->state))
 448                 return;
 449
 450         if (!vsi->tx_rings)
 451                 return;
 452
 453         rcu_read_lock();
 454         for (i = 0; i < vsi->num_queue_pairs; i++) {
 455                 u64 bytes, packets;
 456                 unsigned int start;
 457
 458                 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
 459                 if (!tx_ring)
 460                         continue;
 461                 i40e_get_netdev_stats_struct_tx(tx_ring, stats);
 462
 463                 rx_ring = &tx_ring[1];
 464
 465                 do {
 466                         start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
 467                         packets = rx_ring->stats.packets;
 468                         bytes   = rx_ring->stats.bytes;
 469                 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
 470
 471                 stats->rx_packets += packets;
 472                 stats->rx_bytes   += bytes;
 473
 474                 if (i40e_enabled_xdp_vsi(vsi))
 475                         i40e_get_netdev_stats_struct_tx(&rx_ring[1], stats);
 476         }
 477         rcu_read_unlock();
 478
 479         /* following stats updated by i40e_watchdog_subtask() */
 480         stats->multicast        = vsi_stats->multicast;
 481         stats->tx_errors        = vsi_stats->tx_errors;
 482         stats->tx_dropped       = vsi_stats->tx_dropped;
 483         stats->rx_errors        = vsi_stats->rx_errors;
 484         stats->rx_dropped       = vsi_stats->rx_dropped;
 485         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 486         stats->rx_length_errors = vsi_stats->rx_length_errors;
 487 }
 488
 489 /**
 490  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 491  * @vsi: the VSI to have its stats reset
 492  **/
 493 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 494 {
 495         struct rtnl_link_stats64 *ns;
 496         int i;
 497
 498         if (!vsi)
 499                 return;
 500
 501         ns = i40e_get_vsi_stats_struct(vsi);
 502         memset(ns, 0, sizeof(*ns));
 503         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 504         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 505         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 506         if (vsi->rx_rings && vsi->rx_rings[0]) {
 507                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 508                         memset(&vsi->rx_rings[i]->stats, 0,
 509                                sizeof(vsi->rx_rings[i]->stats));
 510                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 511                                sizeof(vsi->rx_rings[i]->rx_stats));
 512                         memset(&vsi->tx_rings[i]->stats, 0,
 513                                sizeof(vsi->tx_rings[i]->stats));
 514                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 515                                sizeof(vsi->tx_rings[i]->tx_stats));
 516                 }
 517         }
 518         vsi->stat_offsets_loaded = false;
 519 }
 520
 521 /**
 522  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 523  * @pf: the PF to be reset
 524  **/
 525 void i40e_pf_reset_stats(struct i40e_pf *pf)
 526 {
 527         int i;
 528
 529         memset(&pf->stats, 0, sizeof(pf->stats));
 530         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 531         pf->stat_offsets_loaded = false;
 532
 533         for (i = 0; i < I40E_MAX_VEB; i++) {
 534                 if (pf->veb[i]) {
 535                         memset(&pf->veb[i]->stats, 0,
 536                                sizeof(pf->veb[i]->stats));
 537                         memset(&pf->veb[i]->stats_offsets, 0,
 538                                sizeof(pf->veb[i]->stats_offsets));
 539                         pf->veb[i]->stat_offsets_loaded = false;
 540                 }
 541         }
 542         pf->hw_csum_rx_error = 0;
 543 }
 544
 545 /**
 546  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 547  * @hw: ptr to the hardware info
 548  * @hireg: the high 32 bit reg to read
 549  * @loreg: the low 32 bit reg to read
 550  * @offset_loaded: has the initial offset been loaded yet
 551  * @offset: ptr to current offset value
 552  * @stat: ptr to the stat
 553  *
 554  * Since the device stats are not reset at PFReset, they likely will not
 555  * be zeroed when the driver starts.  We'll save the first values read
 556  * and use them as offsets to be subtracted from the raw values in order
 557  * to report stats that count from zero.  In the process, we also manage
 558  * the potential roll-over.
 559  **/
 560 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 561                                bool offset_loaded, u64 *offset, u64 *stat)
 562 {
 563         u64 new_data;
 564
 565         if (hw->device_id == I40E_DEV_ID_QEMU) {
 566                 new_data = rd32(hw, loreg);
 567                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 568         } else {
 569                 new_data = rd64(hw, loreg);
 570         }
 571         if (!offset_loaded)
 572                 *offset = new_data;
 573         if (likely(new_data >= *offset))
 574                 *stat = new_data - *offset;
 575         else
 576                 *stat = (new_data + BIT_ULL(48)) - *offset;
 577         *stat &= 0xFFFFFFFFFFFFULL;
 578 }
 579
 580 /**
 581  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 582  * @hw: ptr to the hardware info
 583  * @reg: the hw reg to read
 584  * @offset_loaded: has the initial offset been loaded yet
 585  * @offset: ptr to current offset value
 586  * @stat: ptr to the stat
 587  **/
 588 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 589                                bool offset_loaded, u64 *offset, u64 *stat)
 590 {
 591         u32 new_data;
 592
 593         new_data = rd32(hw, reg);
 594         if (!offset_loaded)
 595                 *offset = new_data;
 596         if (likely(new_data >= *offset))
 597                 *stat = (u32)(new_data - *offset);
 598         else
 599                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 600 }
 601
 602 /**
 603  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
 604  * @hw: ptr to the hardware info
 605  * @reg: the hw reg to read and clear
 606  * @stat: ptr to the stat
 607  **/
 608 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
 609 {
 610         u32 new_data = rd32(hw, reg);
 611
 612         wr32(hw, reg, 1); /* must write a nonzero value to clear register */
 613         *stat += new_data;
 614 }
 615
 616 /**
 617  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 618  * @vsi: the VSI to be updated
 619  **/
 620 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 621 {
 622         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 623         struct i40e_pf *pf = vsi->back;
 624         struct i40e_hw *hw = &pf->hw;
 625         struct i40e_eth_stats *oes;
 626         struct i40e_eth_stats *es;     /* device's eth stats */
 627
 628         es = &vsi->eth_stats;
 629         oes = &vsi->eth_stats_offsets;
 630
 631         /* Gather up the stats that the hw collects */
 632         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 633                            vsi->stat_offsets_loaded,
 634                            &oes->tx_errors, &es->tx_errors);
 635         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 636                            vsi->stat_offsets_loaded,
 637                            &oes->rx_discards, &es->rx_discards);
 638         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 639                            vsi->stat_offsets_loaded,
 640                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 641         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 642                            vsi->stat_offsets_loaded,
 643                            &oes->tx_errors, &es->tx_errors);
 644
 645         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 646                            I40E_GLV_GORCL(stat_idx),
 647                            vsi->stat_offsets_loaded,
 648                            &oes->rx_bytes, &es->rx_bytes);
 649         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 650                            I40E_GLV_UPRCL(stat_idx),
 651                            vsi->stat_offsets_loaded,
 652                            &oes->rx_unicast, &es->rx_unicast);
 653         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 654                            I40E_GLV_MPRCL(stat_idx),
 655                            vsi->stat_offsets_loaded,
 656                            &oes->rx_multicast, &es->rx_multicast);
 657         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 658                            I40E_GLV_BPRCL(stat_idx),
 659                            vsi->stat_offsets_loaded,
 660                            &oes->rx_broadcast, &es->rx_broadcast);
 661
 662         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 663                            I40E_GLV_GOTCL(stat_idx),
 664                            vsi->stat_offsets_loaded,
 665                            &oes->tx_bytes, &es->tx_bytes);
 666         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 667                            I40E_GLV_UPTCL(stat_idx),
 668                            vsi->stat_offsets_loaded,
 669                            &oes->tx_unicast, &es->tx_unicast);
 670         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 671                            I40E_GLV_MPTCL(stat_idx),
 672                            vsi->stat_offsets_loaded,
 673                            &oes->tx_multicast, &es->tx_multicast);
 674         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 675                            I40E_GLV_BPTCL(stat_idx),
 676                            vsi->stat_offsets_loaded,
 677                            &oes->tx_broadcast, &es->tx_broadcast);
 678         vsi->stat_offsets_loaded = true;
 679 }
 680
 681 /**
 682  * i40e_update_veb_stats - Update Switch component statistics
 683  * @veb: the VEB being updated
 684  **/
 685 static void i40e_update_veb_stats(struct i40e_veb *veb)
 686 {
 687         struct i40e_pf *pf = veb->pf;
 688         struct i40e_hw *hw = &pf->hw;
 689         struct i40e_eth_stats *oes;
 690         struct i40e_eth_stats *es;     /* device's eth stats */
 691         struct i40e_veb_tc_stats *veb_oes;
 692         struct i40e_veb_tc_stats *veb_es;
 693         int i, idx = 0;
 694
 695         idx = veb->stats_idx;
 696         es = &veb->stats;
 697         oes = &veb->stats_offsets;
 698         veb_es = &veb->tc_stats;
 699         veb_oes = &veb->tc_stats_offsets;
 700
 701         /* Gather up the stats that the hw collects */
 702         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 703                            veb->stat_offsets_loaded,
 704                            &oes->tx_discards, &es->tx_discards);
 705         if (hw->revision_id > 0)
 706                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 707                                    veb->stat_offsets_loaded,
 708                                    &oes->rx_unknown_protocol,
 709                                    &es->rx_unknown_protocol);
 710         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 711                            veb->stat_offsets_loaded,
 712                            &oes->rx_bytes, &es->rx_bytes);
 713         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 714                            veb->stat_offsets_loaded,
 715                            &oes->rx_unicast, &es->rx_unicast);
 716         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 717                            veb->stat_offsets_loaded,
 718                            &oes->rx_multicast, &es->rx_multicast);
 719         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 720                            veb->stat_offsets_loaded,
 721                            &oes->rx_broadcast, &es->rx_broadcast);
 722
 723         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 724                            veb->stat_offsets_loaded,
 725                            &oes->tx_bytes, &es->tx_bytes);
 726         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 727                            veb->stat_offsets_loaded,
 728                            &oes->tx_unicast, &es->tx_unicast);
 729         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 730                            veb->stat_offsets_loaded,
 731                            &oes->tx_multicast, &es->tx_multicast);
 732         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 733                            veb->stat_offsets_loaded,
 734                            &oes->tx_broadcast, &es->tx_broadcast);
 735         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 736                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 737                                    I40E_GLVEBTC_RPCL(i, idx),
 738                                    veb->stat_offsets_loaded,
 739                                    &veb_oes->tc_rx_packets[i],
 740                                    &veb_es->tc_rx_packets[i]);
 741                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 742                                    I40E_GLVEBTC_RBCL(i, idx),
 743                                    veb->stat_offsets_loaded,
 744                                    &veb_oes->tc_rx_bytes[i],
 745                                    &veb_es->tc_rx_bytes[i]);
 746                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 747                                    I40E_GLVEBTC_TPCL(i, idx),
 748                                    veb->stat_offsets_loaded,
 749                                    &veb_oes->tc_tx_packets[i],
 750                                    &veb_es->tc_tx_packets[i]);
 751                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 752                                    I40E_GLVEBTC_TBCL(i, idx),
 753                                    veb->stat_offsets_loaded,
 754                                    &veb_oes->tc_tx_bytes[i],
 755                                    &veb_es->tc_tx_bytes[i]);
 756         }
 757         veb->stat_offsets_loaded = true;
 758 }
 759
 760 /**
 761  * i40e_update_vsi_stats - Update the vsi statistics counters.
 762  * @vsi: the VSI to be updated
 763  *
 764  * There are a few instances where we store the same stat in a
 765  * couple of different structs.  This is partly because we have
 766  * the netdev stats that need to be filled out, which is slightly
 767  * different from the "eth_stats" defined by the chip and used in
 768  * VF communications.  We sort it out here.
 769  **/
 770 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 771 {
 772         struct i40e_pf *pf = vsi->back;
 773         struct rtnl_link_stats64 *ons;
 774         struct rtnl_link_stats64 *ns;   /* netdev stats */
 775         struct i40e_eth_stats *oes;
 776         struct i40e_eth_stats *es;     /* device's eth stats */
 777         u32 tx_restart, tx_busy;
 778         struct i40e_ring *p;
 779         u32 rx_page, rx_buf;
 780         u64 bytes, packets;
 781         unsigned int start;
 782         u64 tx_linearize;
 783         u64 tx_force_wb;
 784         u64 rx_p, rx_b;
 785         u64 tx_p, tx_b;
 786         u16 q;
 787
 788         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
 789             test_bit(__I40E_CONFIG_BUSY, pf->state))
 790                 return;
 791
 792         ns = i40e_get_vsi_stats_struct(vsi);
 793         ons = &vsi->net_stats_offsets;
 794         es = &vsi->eth_stats;
 795         oes = &vsi->eth_stats_offsets;
 796
 797         /* Gather up the netdev and vsi stats that the driver collects
 798          * on the fly during packet processing
 799          */
 800         rx_b = rx_p = 0;
 801         tx_b = tx_p = 0;
 802         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 803         rx_page = 0;
 804         rx_buf = 0;
 805         rcu_read_lock();
 806         for (q = 0; q < vsi->num_queue_pairs; q++) {
 807                 /* locate Tx ring */
 808                 p = ACCESS_ONCE(vsi->tx_rings[q]);
 809
 810                 do {
 811                         start = u64_stats_fetch_begin_irq(&p->syncp);
 812                         packets = p->stats.packets;
 813                         bytes = p->stats.bytes;
 814                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 815                 tx_b += bytes;
 816                 tx_p += packets;
 817                 tx_restart += p->tx_stats.restart_queue;
 818                 tx_busy += p->tx_stats.tx_busy;
 819                 tx_linearize += p->tx_stats.tx_linearize;
 820                 tx_force_wb += p->tx_stats.tx_force_wb;
 821
 822                 /* Rx queue is part of the same block as Tx queue */
 823                 p = &p[1];
 824                 do {
 825                         start = u64_stats_fetch_begin_irq(&p->syncp);
 826                         packets = p->stats.packets;
 827                         bytes = p->stats.bytes;
 828                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 829                 rx_b += bytes;
 830                 rx_p += packets;
 831                 rx_buf += p->rx_stats.alloc_buff_failed;
 832                 rx_page += p->rx_stats.alloc_page_failed;
 833         }
 834         rcu_read_unlock();
 835         vsi->tx_restart = tx_restart;
 836         vsi->tx_busy = tx_busy;
 837         vsi->tx_linearize = tx_linearize;
 838         vsi->tx_force_wb = tx_force_wb;
 839         vsi->rx_page_failed = rx_page;
 840         vsi->rx_buf_failed = rx_buf;
 841
 842         ns->rx_packets = rx_p;
 843         ns->rx_bytes = rx_b;
 844         ns->tx_packets = tx_p;
 845         ns->tx_bytes = tx_b;
 846
 847         /* update netdev stats from eth stats */
 848         i40e_update_eth_stats(vsi);
 849         ons->tx_errors = oes->tx_errors;
 850         ns->tx_errors = es->tx_errors;
 851         ons->multicast = oes->rx_multicast;
 852         ns->multicast = es->rx_multicast;
 853         ons->rx_dropped = oes->rx_discards;
 854         ns->rx_dropped = es->rx_discards;
 855         ons->tx_dropped = oes->tx_discards;
 856         ns->tx_dropped = es->tx_discards;
 857
 858         /* pull in a couple PF stats if this is the main vsi */
 859         if (vsi == pf->vsi[pf->lan_vsi]) {
 860                 ns->rx_crc_errors = pf->stats.crc_errors;
 861                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 862                 ns->rx_length_errors = pf->stats.rx_length_errors;
 863         }
 864 }
 865
 866 /**
 867  * i40e_update_pf_stats - Update the PF statistics counters.
 868  * @pf: the PF to be updated
 869  **/
 870 static void i40e_update_pf_stats(struct i40e_pf *pf)
 871 {
 872         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 873         struct i40e_hw_port_stats *nsd = &pf->stats;
 874         struct i40e_hw *hw = &pf->hw;
 875         u32 val;
 876         int i;
 877
 878         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 879                            I40E_GLPRT_GORCL(hw->port),
 880                            pf->stat_offsets_loaded,
 881                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 882         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 883                            I40E_GLPRT_GOTCL(hw->port),
 884                            pf->stat_offsets_loaded,
 885                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 886         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 887                            pf->stat_offsets_loaded,
 888                            &osd->eth.rx_discards,
 889                            &nsd->eth.rx_discards);
 890         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 891                            I40E_GLPRT_UPRCL(hw->port),
 892                            pf->stat_offsets_loaded,
 893                            &osd->eth.rx_unicast,
 894                            &nsd->eth.rx_unicast);
 895         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 896                            I40E_GLPRT_MPRCL(hw->port),
 897                            pf->stat_offsets_loaded,
 898                            &osd->eth.rx_multicast,
 899                            &nsd->eth.rx_multicast);
 900         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 901                            I40E_GLPRT_BPRCL(hw->port),
 902                            pf->stat_offsets_loaded,
 903                            &osd->eth.rx_broadcast,
 904                            &nsd->eth.rx_broadcast);
 905         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 906                            I40E_GLPRT_UPTCL(hw->port),
 907                            pf->stat_offsets_loaded,
 908                            &osd->eth.tx_unicast,
 909                            &nsd->eth.tx_unicast);
 910         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 911                            I40E_GLPRT_MPTCL(hw->port),
 912                            pf->stat_offsets_loaded,
 913                            &osd->eth.tx_multicast,
 914                            &nsd->eth.tx_multicast);
 915         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 916                            I40E_GLPRT_BPTCL(hw->port),
 917                            pf->stat_offsets_loaded,
 918                            &osd->eth.tx_broadcast,
 919                            &nsd->eth.tx_broadcast);
 920
 921         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 922                            pf->stat_offsets_loaded,
 923                            &osd->tx_dropped_link_down,
 924                            &nsd->tx_dropped_link_down);
 925
 926         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 927                            pf->stat_offsets_loaded,
 928                            &osd->crc_errors, &nsd->crc_errors);
 929
 930         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 931                            pf->stat_offsets_loaded,
 932                            &osd->illegal_bytes, &nsd->illegal_bytes);
 933
 934         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 935                            pf->stat_offsets_loaded,
 936                            &osd->mac_local_faults,
 937                            &nsd->mac_local_faults);
 938         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 939                            pf->stat_offsets_loaded,
 940                            &osd->mac_remote_faults,
 941                            &nsd->mac_remote_faults);
 942
 943         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 944                            pf->stat_offsets_loaded,
 945                            &osd->rx_length_errors,
 946                            &nsd->rx_length_errors);
 947
 948         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 949                            pf->stat_offsets_loaded,
 950                            &osd->link_xon_rx, &nsd->link_xon_rx);
 951         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 952                            pf->stat_offsets_loaded,
 953                            &osd->link_xon_tx, &nsd->link_xon_tx);
 954         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 955                            pf->stat_offsets_loaded,
 956                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 957         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 958                            pf->stat_offsets_loaded,
 959                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 960
 961         for (i = 0; i < 8; i++) {
 962                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 963                                    pf->stat_offsets_loaded,
 964                                    &osd->priority_xoff_rx[i],
 965                                    &nsd->priority_xoff_rx[i]);
 966                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 967                                    pf->stat_offsets_loaded,
 968                                    &osd->priority_xon_rx[i],
 969                                    &nsd->priority_xon_rx[i]);
 970                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 971                                    pf->stat_offsets_loaded,
 972                                    &osd->priority_xon_tx[i],
 973                                    &nsd->priority_xon_tx[i]);
 974                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 975                                    pf->stat_offsets_loaded,
 976                                    &osd->priority_xoff_tx[i],
 977                                    &nsd->priority_xoff_tx[i]);
 978                 i40e_stat_update32(hw,
 979                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 980                                    pf->stat_offsets_loaded,
 981                                    &osd->priority_xon_2_xoff[i],
 982                                    &nsd->priority_xon_2_xoff[i]);
 983         }
 984
 985         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 986                            I40E_GLPRT_PRC64L(hw->port),
 987                            pf->stat_offsets_loaded,
 988                            &osd->rx_size_64, &nsd->rx_size_64);
 989         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 990                            I40E_GLPRT_PRC127L(hw->port),
 991                            pf->stat_offsets_loaded,
 992                            &osd->rx_size_127, &nsd->rx_size_127);
 993         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 994                            I40E_GLPRT_PRC255L(hw->port),
 995                            pf->stat_offsets_loaded,
 996                            &osd->rx_size_255, &nsd->rx_size_255);
 997         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
 998                            I40E_GLPRT_PRC511L(hw->port),
 999                            pf->stat_offsets_loaded,
1000                            &osd->rx_size_511, &nsd->rx_size_511);
1001         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1002                            I40E_GLPRT_PRC1023L(hw->port),
1003                            pf->stat_offsets_loaded,
1004                            &osd->rx_size_1023, &nsd->rx_size_1023);
1005         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1006                            I40E_GLPRT_PRC1522L(hw->port),
1007                            pf->stat_offsets_loaded,
1008                            &osd->rx_size_1522, &nsd->rx_size_1522);
1009         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1010                            I40E_GLPRT_PRC9522L(hw->port),
1011                            pf->stat_offsets_loaded,
1012                            &osd->rx_size_big, &nsd->rx_size_big);
1013
1014         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1015                            I40E_GLPRT_PTC64L(hw->port),
1016                            pf->stat_offsets_loaded,
1017                            &osd->tx_size_64, &nsd->tx_size_64);
1018         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1019                            I40E_GLPRT_PTC127L(hw->port),
1020                            pf->stat_offsets_loaded,
1021                            &osd->tx_size_127, &nsd->tx_size_127);
1022         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1023                            I40E_GLPRT_PTC255L(hw->port),
1024                            pf->stat_offsets_loaded,
1025                            &osd->tx_size_255, &nsd->tx_size_255);
1026         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1027                            I40E_GLPRT_PTC511L(hw->port),
1028                            pf->stat_offsets_loaded,
1029                            &osd->tx_size_511, &nsd->tx_size_511);
1030         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1031                            I40E_GLPRT_PTC1023L(hw->port),
1032                            pf->stat_offsets_loaded,
1033                            &osd->tx_size_1023, &nsd->tx_size_1023);
1034         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1035                            I40E_GLPRT_PTC1522L(hw->port),
1036                            pf->stat_offsets_loaded,
1037                            &osd->tx_size_1522, &nsd->tx_size_1522);
1038         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1039                            I40E_GLPRT_PTC9522L(hw->port),
1040                            pf->stat_offsets_loaded,
1041                            &osd->tx_size_big, &nsd->tx_size_big);
1042
1043         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1044                            pf->stat_offsets_loaded,
1045                            &osd->rx_undersize, &nsd->rx_undersize);
1046         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1047                            pf->stat_offsets_loaded,
1048                            &osd->rx_fragments, &nsd->rx_fragments);
1049         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1050                            pf->stat_offsets_loaded,
1051                            &osd->rx_oversize, &nsd->rx_oversize);
1052         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1053                            pf->stat_offsets_loaded,
1054                            &osd->rx_jabber, &nsd->rx_jabber);
1055
1056         /* FDIR stats */
1057         i40e_stat_update_and_clear32(hw,
1058                         I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1059                         &nsd->fd_atr_match);
1060         i40e_stat_update_and_clear32(hw,
1061                         I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1062                         &nsd->fd_sb_match);
1063         i40e_stat_update_and_clear32(hw,
1064                         I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1065                         &nsd->fd_atr_tunnel_match);
1066
1067         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1068         nsd->tx_lpi_status =
1069                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1070                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1071         nsd->rx_lpi_status =
1072                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1073                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1074         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1075                            pf->stat_offsets_loaded,
1076                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1077         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1078                            pf->stat_offsets_loaded,
1079                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1080
1081         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1082             !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED))
1083                 nsd->fd_sb_status = true;
1084         else
1085                 nsd->fd_sb_status = false;
1086
1087         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1088             !(pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED))
1089                 nsd->fd_atr_status = true;
1090         else
1091                 nsd->fd_atr_status = false;
1092
1093         pf->stat_offsets_loaded = true;
1094 }
1095
1096 /**
1097  * i40e_update_stats - Update the various statistics counters.
1098  * @vsi: the VSI to be updated
1099  *
1100  * Update the various stats for this VSI and its related entities.
1101  **/
1102 void i40e_update_stats(struct i40e_vsi *vsi)
1103 {
1104         struct i40e_pf *pf = vsi->back;
1105
1106         if (vsi == pf->vsi[pf->lan_vsi])
1107                 i40e_update_pf_stats(pf);
1108
1109         i40e_update_vsi_stats(vsi);
1110 }
1111
1112 /**
1113  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1114  * @vsi: the VSI to be searched
1115  * @macaddr: the MAC address
1116  * @vlan: the vlan
1117  *
1118  * Returns ptr to the filter object or NULL
1119  **/
1120 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1121                                                 const u8 *macaddr, s16 vlan)
1122 {
1123         struct i40e_mac_filter *f;
1124         u64 key;
1125
1126         if (!vsi || !macaddr)
1127                 return NULL;
1128
1129         key = i40e_addr_to_hkey(macaddr);
1130         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1131                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1132                     (vlan == f->vlan))
1133                         return f;
1134         }
1135         return NULL;
1136 }
1137
1138 /**
1139  * i40e_find_mac - Find a mac addr in the macvlan filters list
1140  * @vsi: the VSI to be searched
1141  * @macaddr: the MAC address we are searching for
1142  *
1143  * Returns the first filter with the provided MAC address or NULL if
1144  * MAC address was not found
1145  **/
1146 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1147 {
1148         struct i40e_mac_filter *f;
1149         u64 key;
1150
1151         if (!vsi || !macaddr)
1152                 return NULL;
1153
1154         key = i40e_addr_to_hkey(macaddr);
1155         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1156                 if ((ether_addr_equal(macaddr, f->macaddr)))
1157                         return f;
1158         }
1159         return NULL;
1160 }
1161
1162 /**
1163  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1164  * @vsi: the VSI to be searched
1165  *
1166  * Returns true if VSI is in vlan mode or false otherwise
1167  **/
1168 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1169 {
1170         /* If we have a PVID, always operate in VLAN mode */
1171         if (vsi->info.pvid)
1172                 return true;
1173
1174         /* We need to operate in VLAN mode whenever we have any filters with
1175          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1176          * time, incurring search cost repeatedly. However, we can notice two
1177          * things:
1178          *
1179          * 1) the only place where we can gain a VLAN filter is in
1180          *    i40e_add_filter.
1181          *
1182          * 2) the only place where filters are actually removed is in
1183          *    i40e_sync_filters_subtask.
1184          *
1185          * Thus, we can simply use a boolean value, has_vlan_filters which we
1186          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1187          * we have to perform the full search after deleting filters in
1188          * i40e_sync_filters_subtask, but we already have to search
1189          * filters here and can perform the check at the same time. This
1190          * results in avoiding embedding a loop for VLAN mode inside another
1191          * loop over all the filters, and should maintain correctness as noted
1192          * above.
1193          */
1194         return vsi->has_vlan_filter;
1195 }
1196
1197 /**
1198  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1199  * @vsi: the VSI to configure
1200  * @tmp_add_list: list of filters ready to be added
1201  * @tmp_del_list: list of filters ready to be deleted
1202  * @vlan_filters: the number of active VLAN filters
1203  *
1204  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1205  * behave as expected. If we have any active VLAN filters remaining or about
1206  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1207  * so that they only match against untagged traffic. If we no longer have any
1208  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1209  * so that they match against both tagged and untagged traffic. In this way,
1210  * we ensure that we correctly receive the desired traffic. This ensures that
1211  * when we have an active VLAN we will receive only untagged traffic and
1212  * traffic matching active VLANs. If we have no active VLANs then we will
1213  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1214  *
1215  * Finally, in a similar fashion, this function also corrects filters when
1216  * there is an active PVID assigned to this VSI.
1217  *
1218  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1219  *
1220  * This function is only expected to be called from within
1221  * i40e_sync_vsi_filters.
1222  *
1223  * NOTE: This function expects to be called while under the
1224  * mac_filter_hash_lock
1225  */
1226 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1227                                          struct hlist_head *tmp_add_list,
1228                                          struct hlist_head *tmp_del_list,
1229                                          int vlan_filters)
1230 {
1231         s16 pvid = le16_to_cpu(vsi->info.pvid);
1232         struct i40e_mac_filter *f, *add_head;
1233         struct i40e_new_mac_filter *new;
1234         struct hlist_node *h;
1235         int bkt, new_vlan;
1236
1237         /* To determine if a particular filter needs to be replaced we
1238          * have the three following conditions:
1239          *
1240          * a) if we have a PVID assigned, then all filters which are
1241          *    not marked as VLAN=PVID must be replaced with filters that
1242          *    are.
1243          * b) otherwise, if we have any active VLANS, all filters
1244          *    which are marked as VLAN=-1 must be replaced with
1245          *    filters marked as VLAN=0
1246          * c) finally, if we do not have any active VLANS, all filters
1247          *    which are marked as VLAN=0 must be replaced with filters
1248          *    marked as VLAN=-1
1249          */
1250
1251         /* Update the filters about to be added in place */
1252         hlist_for_each_entry(new, tmp_add_list, hlist) {
1253                 if (pvid && new->f->vlan != pvid)
1254                         new->f->vlan = pvid;
1255                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1256                         new->f->vlan = 0;
1257                 else if (!vlan_filters && new->f->vlan == 0)
1258                         new->f->vlan = I40E_VLAN_ANY;
1259         }
1260
1261         /* Update the remaining active filters */
1262         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1263                 /* Combine the checks for whether a filter needs to be changed
1264                  * and then determine the new VLAN inside the if block, in
1265                  * order to avoid duplicating code for adding the new filter
1266                  * then deleting the old filter.
1267                  */
1268                 if ((pvid && f->vlan != pvid) ||
1269                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1270                     (!vlan_filters && f->vlan == 0)) {
1271                         /* Determine the new vlan we will be adding */
1272                         if (pvid)
1273                                 new_vlan = pvid;
1274                         else if (vlan_filters)
1275                                 new_vlan = 0;
1276                         else
1277                                 new_vlan = I40E_VLAN_ANY;
1278
1279                         /* Create the new filter */
1280                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1281                         if (!add_head)
1282                                 return -ENOMEM;
1283
1284                         /* Create a temporary i40e_new_mac_filter */
1285                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1286                         if (!new)
1287                                 return -ENOMEM;
1288
1289                         new->f = add_head;
1290                         new->state = add_head->state;
1291
1292                         /* Add the new filter to the tmp list */
1293                         hlist_add_head(&new->hlist, tmp_add_list);
1294
1295                         /* Put the original filter into the delete list */
1296                         f->state = I40E_FILTER_REMOVE;
1297                         hash_del(&f->hlist);
1298                         hlist_add_head(&f->hlist, tmp_del_list);
1299                 }
1300         }
1301
1302         vsi->has_vlan_filter = !!vlan_filters;
1303
1304         return 0;
1305 }
1306
1307 /**
1308  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1309  * @vsi: the PF Main VSI - inappropriate for any other VSI
1310  * @macaddr: the MAC address
1311  *
1312  * Remove whatever filter the firmware set up so the driver can manage
1313  * its own filtering intelligently.
1314  **/
1315 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1316 {
1317         struct i40e_aqc_remove_macvlan_element_data element;
1318         struct i40e_pf *pf = vsi->back;
1319
1320         /* Only appropriate for the PF main VSI */
1321         if (vsi->type != I40E_VSI_MAIN)
1322                 return;
1323
1324         memset(&element, 0, sizeof(element));
1325         ether_addr_copy(element.mac_addr, macaddr);
1326         element.vlan_tag = 0;
1327         /* Ignore error returns, some firmware does it this way... */
1328         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1329         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1330
1331         memset(&element, 0, sizeof(element));
1332         ether_addr_copy(element.mac_addr, macaddr);
1333         element.vlan_tag = 0;
1334         /* ...and some firmware does it this way. */
1335         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1336                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1337         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1338 }
1339
1340 /**
1341  * i40e_add_filter - Add a mac/vlan filter to the VSI
1342  * @vsi: the VSI to be searched
1343  * @macaddr: the MAC address
1344  * @vlan: the vlan
1345  *
1346  * Returns ptr to the filter object or NULL when no memory available.
1347  *
1348  * NOTE: This function is expected to be called with mac_filter_hash_lock
1349  * being held.
1350  **/
1351 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1352                                         const u8 *macaddr, s16 vlan)
1353 {
1354         struct i40e_mac_filter *f;
1355         u64 key;
1356
1357         if (!vsi || !macaddr)
1358                 return NULL;
1359
1360         f = i40e_find_filter(vsi, macaddr, vlan);
1361         if (!f) {
1362                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1363                 if (!f)
1364                         return NULL;
1365
1366                 /* Update the boolean indicating if we need to function in
1367                  * VLAN mode.
1368                  */
1369                 if (vlan >= 0)
1370                         vsi->has_vlan_filter = true;
1371
1372                 ether_addr_copy(f->macaddr, macaddr);
1373                 f->vlan = vlan;
1374                 /* If we're in overflow promisc mode, set the state directly
1375                  * to failed, so we don't bother to try sending the filter
1376                  * to the hardware.
1377                  */
1378                 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state))
1379                         f->state = I40E_FILTER_FAILED;
1380                 else
1381                         f->state = I40E_FILTER_NEW;
1382                 INIT_HLIST_NODE(&f->hlist);
1383
1384                 key = i40e_addr_to_hkey(macaddr);
1385                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1386
1387                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1388                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1389         }
1390
1391         /* If we're asked to add a filter that has been marked for removal, it
1392          * is safe to simply restore it to active state. __i40e_del_filter
1393          * will have simply deleted any filters which were previously marked
1394          * NEW or FAILED, so if it is currently marked REMOVE it must have
1395          * previously been ACTIVE. Since we haven't yet run the sync filters
1396          * task, just restore this filter to the ACTIVE state so that the
1397          * sync task leaves it in place
1398          */
1399         if (f->state == I40E_FILTER_REMOVE)
1400                 f->state = I40E_FILTER_ACTIVE;
1401
1402         return f;
1403 }
1404
1405 /**
1406  * __i40e_del_filter - Remove a specific filter from the VSI
1407  * @vsi: VSI to remove from
1408  * @f: the filter to remove from the list
1409  *
1410  * This function should be called instead of i40e_del_filter only if you know
1411  * the exact filter you will remove already, such as via i40e_find_filter or
1412  * i40e_find_mac.
1413  *
1414  * NOTE: This function is expected to be called with mac_filter_hash_lock
1415  * being held.
1416  * ANOTHER NOTE: This function MUST be called from within the context of
1417  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1418  * instead of list_for_each_entry().
1419  **/
1420 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1421 {
1422         if (!f)
1423                 return;
1424
1425         /* If the filter was never added to firmware then we can just delete it
1426          * directly and we don't want to set the status to remove or else an
1427          * admin queue command will unnecessarily fire.
1428          */
1429         if ((f->state == I40E_FILTER_FAILED) ||
1430             (f->state == I40E_FILTER_NEW)) {
1431                 hash_del(&f->hlist);
1432                 kfree(f);
1433         } else {
1434                 f->state = I40E_FILTER_REMOVE;
1435         }
1436
1437         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1438         vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1439 }
1440
1441 /**
1442  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1443  * @vsi: the VSI to be searched
1444  * @macaddr: the MAC address
1445  * @vlan: the VLAN
1446  *
1447  * NOTE: This function is expected to be called with mac_filter_hash_lock
1448  * being held.
1449  * ANOTHER NOTE: This function MUST be called from within the context of
1450  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1451  * instead of list_for_each_entry().
1452  **/
1453 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1454 {
1455         struct i40e_mac_filter *f;
1456
1457         if (!vsi || !macaddr)
1458                 return;
1459
1460         f = i40e_find_filter(vsi, macaddr, vlan);
1461         __i40e_del_filter(vsi, f);
1462 }
1463
1464 /**
1465  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1466  * @vsi: the VSI to be searched
1467  * @macaddr: the mac address to be filtered
1468  *
1469  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1470  * go through all the macvlan filters and add a macvlan filter for each
1471  * unique vlan that already exists. If a PVID has been assigned, instead only
1472  * add the macaddr to that VLAN.
1473  *
1474  * Returns last filter added on success, else NULL
1475  **/
1476 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1477                                             const u8 *macaddr)
1478 {
1479         struct i40e_mac_filter *f, *add = NULL;
1480         struct hlist_node *h;
1481         int bkt;
1482
1483         if (vsi->info.pvid)
1484                 return i40e_add_filter(vsi, macaddr,
1485                                        le16_to_cpu(vsi->info.pvid));
1486
1487         if (!i40e_is_vsi_in_vlan(vsi))
1488                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1489
1490         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1491                 if (f->state == I40E_FILTER_REMOVE)
1492                         continue;
1493                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1494                 if (!add)
1495                         return NULL;
1496         }
1497
1498         return add;
1499 }
1500
1501 /**
1502  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1503  * @vsi: the VSI to be searched
1504  * @macaddr: the mac address to be removed
1505  *
1506  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1507  * associated with.
1508  *
1509  * Returns 0 for success, or error
1510  **/
1511 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1512 {
1513         struct i40e_mac_filter *f;
1514         struct hlist_node *h;
1515         bool found = false;
1516         int bkt;
1517
1518         WARN(!spin_is_locked(&vsi->mac_filter_hash_lock),
1519              "Missing mac_filter_hash_lock\n");
1520         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1521                 if (ether_addr_equal(macaddr, f->macaddr)) {
1522                         __i40e_del_filter(vsi, f);
1523                         found = true;
1524                 }
1525         }
1526
1527         if (found)
1528                 return 0;
1529         else
1530                 return -ENOENT;
1531 }
1532
1533 /**
1534  * i40e_set_mac - NDO callback to set mac address
1535  * @netdev: network interface device structure
1536  * @p: pointer to an address structure
1537  *
1538  * Returns 0 on success, negative on failure
1539  **/
1540 static int i40e_set_mac(struct net_device *netdev, void *p)
1541 {
1542         struct i40e_netdev_priv *np = netdev_priv(netdev);
1543         struct i40e_vsi *vsi = np->vsi;
1544         struct i40e_pf *pf = vsi->back;
1545         struct i40e_hw *hw = &pf->hw;
1546         struct sockaddr *addr = p;
1547
1548         if (!is_valid_ether_addr(addr->sa_data))
1549                 return -EADDRNOTAVAIL;
1550
1551         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1552                 netdev_info(netdev, "already using mac address %pM\n",
1553                             addr->sa_data);
1554                 return 0;
1555         }
1556
1557         if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
1558             test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
1559                 return -EADDRNOTAVAIL;
1560
1561         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1562                 netdev_info(netdev, "returning to hw mac address %pM\n",
1563                             hw->mac.addr);
1564         else
1565                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1566
1567         spin_lock_bh(&vsi->mac_filter_hash_lock);
1568         i40e_del_mac_filter(vsi, netdev->dev_addr);
1569         i40e_add_mac_filter(vsi, addr->sa_data);
1570         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1571         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1572         if (vsi->type == I40E_VSI_MAIN) {
1573                 i40e_status ret;
1574
1575                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1576                                                 I40E_AQC_WRITE_TYPE_LAA_WOL,
1577                                                 addr->sa_data, NULL);
1578                 if (ret)
1579                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1580                                     i40e_stat_str(hw, ret),
1581                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1582         }
1583
1584         /* schedule our worker thread which will take care of
1585          * applying the new filter changes
1586          */
1587         i40e_service_event_schedule(vsi->back);
1588         return 0;
1589 }
1590
1591 /**
1592  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1593  * @vsi: the VSI being setup
1594  * @ctxt: VSI context structure
1595  * @enabled_tc: Enabled TCs bitmap
1596  * @is_add: True if called before Add VSI
1597  *
1598  * Setup VSI queue mapping for enabled traffic classes.
1599  **/
1600 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1601                                      struct i40e_vsi_context *ctxt,
1602                                      u8 enabled_tc,
1603                                      bool is_add)
1604 {
1605         struct i40e_pf *pf = vsi->back;
1606         u16 sections = 0;
1607         u8 netdev_tc = 0;
1608         u16 numtc = 0;
1609         u16 qcount;
1610         u8 offset;
1611         u16 qmap;
1612         int i;
1613         u16 num_tc_qps = 0;
1614
1615         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1616         offset = 0;
1617
1618         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1619                 /* Find numtc from enabled TC bitmap */
1620                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1621                         if (enabled_tc & BIT(i)) /* TC is enabled */
1622                                 numtc++;
1623                 }
1624                 if (!numtc) {
1625                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1626                         numtc = 1;
1627                 }
1628         } else {
1629                 /* At least TC0 is enabled in case of non-DCB case */
1630                 numtc = 1;
1631         }
1632
1633         vsi->tc_config.numtc = numtc;
1634         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1635         /* Number of queues per enabled TC */
1636         qcount = vsi->alloc_queue_pairs;
1637
1638         num_tc_qps = qcount / numtc;
1639         num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf));
1640
1641         /* Setup queue offset/count for all TCs for given VSI */
1642         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1643                 /* See if the given TC is enabled for the given VSI */
1644                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1645                         /* TC is enabled */
1646                         int pow, num_qps;
1647
1648                         switch (vsi->type) {
1649                         case I40E_VSI_MAIN:
1650                                 qcount = min_t(int, pf->alloc_rss_size,
1651                                                num_tc_qps);
1652                                 break;
1653                         case I40E_VSI_FDIR:
1654                         case I40E_VSI_SRIOV:
1655                         case I40E_VSI_VMDQ2:
1656                         default:
1657                                 qcount = num_tc_qps;
1658                                 WARN_ON(i != 0);
1659                                 break;
1660                         }
1661                         vsi->tc_config.tc_info[i].qoffset = offset;
1662                         vsi->tc_config.tc_info[i].qcount = qcount;
1663
1664                         /* find the next higher power-of-2 of num queue pairs */
1665                         num_qps = qcount;
1666                         pow = 0;
1667                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1668                                 pow++;
1669                                 num_qps >>= 1;
1670                         }
1671
1672                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1673                         qmap =
1674                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1675                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1676
1677                         offset += qcount;
1678                 } else {
1679                         /* TC is not enabled so set the offset to
1680                          * default queue and allocate one queue
1681                          * for the given TC.
1682                          */
1683                         vsi->tc_config.tc_info[i].qoffset = 0;
1684                         vsi->tc_config.tc_info[i].qcount = 1;
1685                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1686
1687                         qmap = 0;
1688                 }
1689                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1690         }
1691
1692         /* Set actual Tx/Rx queue pairs */
1693         vsi->num_queue_pairs = offset;
1694         if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1695                 if (vsi->req_queue_pairs > 0)
1696                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1697                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1698                         vsi->num_queue_pairs = pf->num_lan_msix;
1699         }
1700
1701         /* Scheduler section valid can only be set for ADD VSI */
1702         if (is_add) {
1703                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1704
1705                 ctxt->info.up_enable_bits = enabled_tc;
1706         }
1707         if (vsi->type == I40E_VSI_SRIOV) {
1708                 ctxt->info.mapping_flags |=
1709                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1710                 for (i = 0; i < vsi->num_queue_pairs; i++)
1711                         ctxt->info.queue_mapping[i] =
1712                                                cpu_to_le16(vsi->base_queue + i);
1713         } else {
1714                 ctxt->info.mapping_flags |=
1715                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1716                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1717         }
1718         ctxt->info.valid_sections |= cpu_to_le16(sections);
1719 }
1720
1721 /**
1722  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1723  * @netdev: the netdevice
1724  * @addr: address to add
1725  *
1726  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1727  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1728  */
1729 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1730 {
1731         struct i40e_netdev_priv *np = netdev_priv(netdev);
1732         struct i40e_vsi *vsi = np->vsi;
1733
1734         if (i40e_add_mac_filter(vsi, addr))
1735                 return 0;
1736         else
1737                 return -ENOMEM;
1738 }
1739
1740 /**
1741  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1742  * @netdev: the netdevice
1743  * @addr: address to add
1744  *
1745  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1746  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1747  */
1748 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1749 {
1750         struct i40e_netdev_priv *np = netdev_priv(netdev);
1751         struct i40e_vsi *vsi = np->vsi;
1752
1753         i40e_del_mac_filter(vsi, addr);
1754
1755         return 0;
1756 }
1757
1758 /**
1759  * i40e_set_rx_mode - NDO callback to set the netdev filters
1760  * @netdev: network interface device structure
1761  **/
1762 static void i40e_set_rx_mode(struct net_device *netdev)
1763 {
1764         struct i40e_netdev_priv *np = netdev_priv(netdev);
1765         struct i40e_vsi *vsi = np->vsi;
1766
1767         spin_lock_bh(&vsi->mac_filter_hash_lock);
1768
1769         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1770         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1771
1772         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1773
1774         /* check for other flag changes */
1775         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1776                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1777                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1778         }
1779
1780         /* schedule our worker thread which will take care of
1781          * applying the new filter changes
1782          */
1783         i40e_service_event_schedule(vsi->back);
1784 }
1785
1786 /**
1787  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
1788  * @vsi: Pointer to VSI struct
1789  * @from: Pointer to list which contains MAC filter entries - changes to
1790  *        those entries needs to be undone.
1791  *
1792  * MAC filter entries from this list were slated for deletion.
1793  **/
1794 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
1795                                          struct hlist_head *from)
1796 {
1797         struct i40e_mac_filter *f;
1798         struct hlist_node *h;
1799
1800         hlist_for_each_entry_safe(f, h, from, hlist) {
1801                 u64 key = i40e_addr_to_hkey(f->macaddr);
1802
1803                 /* Move the element back into MAC filter list*/
1804                 hlist_del(&f->hlist);
1805                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1806         }
1807 }
1808
1809 /**
1810  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
1811  * @vsi: Pointer to vsi struct
1812  * @from: Pointer to list which contains MAC filter entries - changes to
1813  *        those entries needs to be undone.
1814  *
1815  * MAC filter entries from this list were slated for addition.
1816  **/
1817 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
1818                                          struct hlist_head *from)
1819 {
1820         struct i40e_new_mac_filter *new;
1821         struct hlist_node *h;
1822
1823         hlist_for_each_entry_safe(new, h, from, hlist) {
1824                 /* We can simply free the wrapper structure */
1825                 hlist_del(&new->hlist);
1826                 kfree(new);
1827         }
1828 }
1829
1830 /**
1831  * i40e_next_entry - Get the next non-broadcast filter from a list
1832  * @next: pointer to filter in list
1833  *
1834  * Returns the next non-broadcast filter in the list. Required so that we
1835  * ignore broadcast filters within the list, since these are not handled via
1836  * the normal firmware update path.
1837  */
1838 static
1839 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
1840 {
1841         hlist_for_each_entry_continue(next, hlist) {
1842                 if (!is_broadcast_ether_addr(next->f->macaddr))
1843                         return next;
1844         }
1845
1846         return NULL;
1847 }
1848
1849 /**
1850  * i40e_update_filter_state - Update filter state based on return data
1851  * from firmware
1852  * @count: Number of filters added
1853  * @add_list: return data from fw
1854  * @head: pointer to first filter in current batch
1855  *
1856  * MAC filter entries from list were slated to be added to device. Returns
1857  * number of successful filters. Note that 0 does NOT mean success!
1858  **/
1859 static int
1860 i40e_update_filter_state(int count,
1861                          struct i40e_aqc_add_macvlan_element_data *add_list,
1862                          struct i40e_new_mac_filter *add_head)
1863 {
1864         int retval = 0;
1865         int i;
1866
1867         for (i = 0; i < count; i++) {
1868                 /* Always check status of each filter. We don't need to check
1869                  * the firmware return status because we pre-set the filter
1870                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
1871                  * request to the adminq. Thus, if it no longer matches then
1872                  * we know the filter is active.
1873                  */
1874                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
1875                         add_head->state = I40E_FILTER_FAILED;
1876                 } else {
1877                         add_head->state = I40E_FILTER_ACTIVE;
1878                         retval++;
1879                 }
1880
1881                 add_head = i40e_next_filter(add_head);
1882                 if (!add_head)
1883                         break;
1884         }
1885
1886         return retval;
1887 }
1888
1889 /**
1890  * i40e_aqc_del_filters - Request firmware to delete a set of filters
1891  * @vsi: ptr to the VSI
1892  * @vsi_name: name to display in messages
1893  * @list: the list of filters to send to firmware
1894  * @num_del: the number of filters to delete
1895  * @retval: Set to -EIO on failure to delete
1896  *
1897  * Send a request to firmware via AdminQ to delete a set of filters. Uses
1898  * *retval instead of a return value so that success does not force ret_val to
1899  * be set to 0. This ensures that a sequence of calls to this function
1900  * preserve the previous value of *retval on successful delete.
1901  */
1902 static
1903 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
1904                           struct i40e_aqc_remove_macvlan_element_data *list,
1905                           int num_del, int *retval)
1906 {
1907         struct i40e_hw *hw = &vsi->back->hw;
1908         i40e_status aq_ret;
1909         int aq_err;
1910
1911         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
1912         aq_err = hw->aq.asq_last_status;
1913
1914         /* Explicitly ignore and do not report when firmware returns ENOENT */
1915         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
1916                 *retval = -EIO;
1917                 dev_info(&vsi->back->pdev->dev,
1918                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
1919                          vsi_name, i40e_stat_str(hw, aq_ret),
1920                          i40e_aq_str(hw, aq_err));
1921         }
1922 }
1923
1924 /**
1925  * i40e_aqc_add_filters - Request firmware to add a set of filters
1926  * @vsi: ptr to the VSI
1927  * @vsi_name: name to display in messages
1928  * @list: the list of filters to send to firmware
1929  * @add_head: Position in the add hlist
1930  * @num_add: the number of filters to add
1931  * @promisc_change: set to true on exit if promiscuous mode was forced on
1932  *
1933  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
1934  * promisc_changed to true if the firmware has run out of space for more
1935  * filters.
1936  */
1937 static
1938 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
1939                           struct i40e_aqc_add_macvlan_element_data *list,
1940                           struct i40e_new_mac_filter *add_head,
1941                           int num_add, bool *promisc_changed)
1942 {
1943         struct i40e_hw *hw = &vsi->back->hw;
1944         int aq_err, fcnt;
1945
1946         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
1947         aq_err = hw->aq.asq_last_status;
1948         fcnt = i40e_update_filter_state(num_add, list, add_head);
1949
1950         if (fcnt != num_add) {
1951                 *promisc_changed = true;
1952                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
1953                 dev_warn(&vsi->back->pdev->dev,
1954                          "Error %s adding RX filters on %s, promiscuous mode forced on\n",
1955                          i40e_aq_str(hw, aq_err),
1956                          vsi_name);
1957         }
1958 }
1959
1960 /**
1961  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
1962  * @vsi: pointer to the VSI
1963  * @f: filter data
1964  *
1965  * This function sets or clears the promiscuous broadcast flags for VLAN
1966  * filters in order to properly receive broadcast frames. Assumes that only
1967  * broadcast filters are passed.
1968  *
1969  * Returns status indicating success or failure;
1970  **/
1971 static i40e_status
1972 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
1973                           struct i40e_mac_filter *f)
1974 {
1975         bool enable = f->state == I40E_FILTER_NEW;
1976         struct i40e_hw *hw = &vsi->back->hw;
1977         i40e_status aq_ret;
1978
1979         if (f->vlan == I40E_VLAN_ANY) {
1980                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
1981                                                    vsi->seid,
1982                                                    enable,
1983                                                    NULL);
1984         } else {
1985                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
1986                                                             vsi->seid,
1987                                                             enable,
1988                                                             f->vlan,
1989                                                             NULL);
1990         }
1991
1992         if (aq_ret)
1993                 dev_warn(&vsi->back->pdev->dev,
1994                          "Error %s setting broadcast promiscuous mode on %s\n",
1995                          i40e_aq_str(hw, hw->aq.asq_last_status),
1996                          vsi_name);
1997
1998         return aq_ret;
1999 }
2000
2001 /**
2002  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2003  * @vsi: ptr to the VSI
2004  *
2005  * Push any outstanding VSI filter changes through the AdminQ.
2006  *
2007  * Returns 0 or error value
2008  **/
2009 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2010 {
2011         struct hlist_head tmp_add_list, tmp_del_list;
2012         struct i40e_mac_filter *f;
2013         struct i40e_new_mac_filter *new, *add_head = NULL;
2014         struct i40e_hw *hw = &vsi->back->hw;
2015         unsigned int failed_filters = 0;
2016         unsigned int vlan_filters = 0;
2017         bool promisc_changed = false;
2018         char vsi_name[16] = "PF";
2019         int filter_list_len = 0;
2020         i40e_status aq_ret = 0;
2021         u32 changed_flags = 0;
2022         struct hlist_node *h;
2023         struct i40e_pf *pf;
2024         int num_add = 0;
2025         int num_del = 0;
2026         int retval = 0;
2027         u16 cmd_flags;
2028         int list_size;
2029         int bkt;
2030
2031         /* empty array typed pointers, kcalloc later */
2032         struct i40e_aqc_add_macvlan_element_data *add_list;
2033         struct i40e_aqc_remove_macvlan_element_data *del_list;
2034
2035         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2036                 usleep_range(1000, 2000);
2037         pf = vsi->back;
2038
2039         if (vsi->netdev) {
2040                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2041                 vsi->current_netdev_flags = vsi->netdev->flags;
2042         }
2043
2044         INIT_HLIST_HEAD(&tmp_add_list);
2045         INIT_HLIST_HEAD(&tmp_del_list);
2046
2047         if (vsi->type == I40E_VSI_SRIOV)
2048                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2049         else if (vsi->type != I40E_VSI_MAIN)
2050                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2051
2052         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2053                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2054
2055                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2056                 /* Create a list of filters to delete. */
2057                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2058                         if (f->state == I40E_FILTER_REMOVE) {
2059                                 /* Move the element into temporary del_list */
2060                                 hash_del(&f->hlist);
2061                                 hlist_add_head(&f->hlist, &tmp_del_list);
2062
2063                                 /* Avoid counting removed filters */
2064                                 continue;
2065                         }
2066                         if (f->state == I40E_FILTER_NEW) {
2067                                 /* Create a temporary i40e_new_mac_filter */
2068                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2069                                 if (!new)
2070                                         goto err_no_memory_locked;
2071
2072                                 /* Store pointer to the real filter */
2073                                 new->f = f;
2074                                 new->state = f->state;
2075
2076                                 /* Add it to the hash list */
2077                                 hlist_add_head(&new->hlist, &tmp_add_list);
2078                         }
2079
2080                         /* Count the number of active (current and new) VLAN
2081                          * filters we have now. Does not count filters which
2082                          * are marked for deletion.
2083                          */
2084                         if (f->vlan > 0)
2085                                 vlan_filters++;
2086                 }
2087
2088                 retval = i40e_correct_mac_vlan_filters(vsi,
2089                                                        &tmp_add_list,
2090                                                        &tmp_del_list,
2091                                                        vlan_filters);
2092                 if (retval)
2093                         goto err_no_memory_locked;
2094
2095                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2096         }
2097
2098         /* Now process 'del_list' outside the lock */
2099         if (!hlist_empty(&tmp_del_list)) {
2100                 filter_list_len = hw->aq.asq_buf_size /
2101                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2102                 list_size = filter_list_len *
2103                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2104                 del_list = kzalloc(list_size, GFP_ATOMIC);
2105                 if (!del_list)
2106                         goto err_no_memory;
2107
2108                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2109                         cmd_flags = 0;
2110
2111                         /* handle broadcast filters by updating the broadcast
2112                          * promiscuous flag and release filter list.
2113                          */
2114                         if (is_broadcast_ether_addr(f->macaddr)) {
2115                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2116
2117                                 hlist_del(&f->hlist);
2118                                 kfree(f);
2119                                 continue;
2120                         }
2121
2122                         /* add to delete list */
2123                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2124                         if (f->vlan == I40E_VLAN_ANY) {
2125                                 del_list[num_del].vlan_tag = 0;
2126                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2127                         } else {
2128                                 del_list[num_del].vlan_tag =
2129                                         cpu_to_le16((u16)(f->vlan));
2130                         }
2131
2132                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2133                         del_list[num_del].flags = cmd_flags;
2134                         num_del++;
2135
2136                         /* flush a full buffer */
2137                         if (num_del == filter_list_len) {
2138                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2139                                                      num_del, &retval);
2140                                 memset(del_list, 0, list_size);
2141                                 num_del = 0;
2142                         }
2143                         /* Release memory for MAC filter entries which were
2144                          * synced up with HW.
2145                          */
2146                         hlist_del(&f->hlist);
2147                         kfree(f);
2148                 }
2149
2150                 if (num_del) {
2151                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2152                                              num_del, &retval);
2153                 }
2154
2155                 kfree(del_list);
2156                 del_list = NULL;
2157         }
2158
2159         if (!hlist_empty(&tmp_add_list)) {
2160                 /* Do all the adds now. */
2161                 filter_list_len = hw->aq.asq_buf_size /
2162                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2163                 list_size = filter_list_len *
2164                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2165                 add_list = kzalloc(list_size, GFP_ATOMIC);
2166                 if (!add_list)
2167                         goto err_no_memory;
2168
2169                 num_add = 0;
2170                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2171                         if (test_bit(__I40E_VSI_OVERFLOW_PROMISC,
2172                                      vsi->state)) {
2173                                 new->state = I40E_FILTER_FAILED;
2174                                 continue;
2175                         }
2176
2177                         /* handle broadcast filters by updating the broadcast
2178                          * promiscuous flag instead of adding a MAC filter.
2179                          */
2180                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2181                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2182                                                               new->f))
2183                                         new->state = I40E_FILTER_FAILED;
2184                                 else
2185                                         new->state = I40E_FILTER_ACTIVE;
2186                                 continue;
2187                         }
2188
2189                         /* add to add array */
2190                         if (num_add == 0)
2191                                 add_head = new;
2192                         cmd_flags = 0;
2193                         ether_addr_copy(add_list[num_add].mac_addr,
2194                                         new->f->macaddr);
2195                         if (new->f->vlan == I40E_VLAN_ANY) {
2196                                 add_list[num_add].vlan_tag = 0;
2197                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2198                         } else {
2199                                 add_list[num_add].vlan_tag =
2200                                         cpu_to_le16((u16)(new->f->vlan));
2201                         }
2202                         add_list[num_add].queue_number = 0;
2203                         /* set invalid match method for later detection */
2204                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2205                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2206                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2207                         num_add++;
2208
2209                         /* flush a full buffer */
2210                         if (num_add == filter_list_len) {
2211                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2212                                                      add_head, num_add,
2213                                                      &promisc_changed);
2214                                 memset(add_list, 0, list_size);
2215                                 num_add = 0;
2216                         }
2217                 }
2218                 if (num_add) {
2219                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2220                                              num_add, &promisc_changed);
2221                 }
2222                 /* Now move all of the filters from the temp add list back to
2223                  * the VSI's list.
2224                  */
2225                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2226                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2227                         /* Only update the state if we're still NEW */
2228                         if (new->f->state == I40E_FILTER_NEW)
2229                                 new->f->state = new->state;
2230                         hlist_del(&new->hlist);
2231                         kfree(new);
2232                 }
2233                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2234                 kfree(add_list);
2235                 add_list = NULL;
2236         }
2237
2238         /* Determine the number of active and failed filters. */
2239         spin_lock_bh(&vsi->mac_filter_hash_lock);
2240         vsi->active_filters = 0;
2241         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2242                 if (f->state == I40E_FILTER_ACTIVE)
2243                         vsi->active_filters++;
2244                 else if (f->state == I40E_FILTER_FAILED)
2245                         failed_filters++;
2246         }
2247         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2248
2249         /* If promiscuous mode has changed, we need to calculate a new
2250          * threshold for when we are safe to exit
2251          */
2252         if (promisc_changed)
2253                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2254
2255         /* Check if we are able to exit overflow promiscuous mode. We can
2256          * safely exit if we didn't just enter, we no longer have any failed
2257          * filters, and we have reduced filters below the threshold value.
2258          */
2259         if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state) &&
2260             !promisc_changed && !failed_filters &&
2261             (vsi->active_filters < vsi->promisc_threshold)) {
2262                 dev_info(&pf->pdev->dev,
2263                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2264                          vsi_name);
2265                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2266                 promisc_changed = true;
2267                 vsi->promisc_threshold = 0;
2268         }
2269
2270         /* if the VF is not trusted do not do promisc */
2271         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2272                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2273                 goto out;
2274         }
2275
2276         /* check for changes in promiscuous modes */
2277         if (changed_flags & IFF_ALLMULTI) {
2278                 bool cur_multipromisc;
2279
2280                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2281                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2282                                                                vsi->seid,
2283                                                                cur_multipromisc,
2284                                                                NULL);
2285                 if (aq_ret) {
2286                         retval = i40e_aq_rc_to_posix(aq_ret,
2287                                                      hw->aq.asq_last_status);
2288                         dev_info(&pf->pdev->dev,
2289                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2290                                  vsi_name,
2291                                  i40e_stat_str(hw, aq_ret),
2292                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2293                 }
2294         }
2295
2296         if ((changed_flags & IFF_PROMISC) || promisc_changed) {
2297                 bool cur_promisc;
2298
2299                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2300                                test_bit(__I40E_VSI_OVERFLOW_PROMISC,
2301                                         vsi->state));
2302                 if ((vsi->type == I40E_VSI_MAIN) &&
2303                     (pf->lan_veb != I40E_NO_VEB) &&
2304                     !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2305                         /* set defport ON for Main VSI instead of true promisc
2306                          * this way we will get all unicast/multicast and VLAN
2307                          * promisc behavior but will not get VF or VMDq traffic
2308                          * replicated on the Main VSI.
2309                          */
2310                         if (pf->cur_promisc != cur_promisc) {
2311                                 pf->cur_promisc = cur_promisc;
2312                                 if (cur_promisc)
2313                                         aq_ret =
2314                                               i40e_aq_set_default_vsi(hw,
2315                                                                       vsi->seid,
2316                                                                       NULL);
2317                                 else
2318                                         aq_ret =
2319                                             i40e_aq_clear_default_vsi(hw,
2320                                                                       vsi->seid,
2321                                                                       NULL);
2322                                 if (aq_ret) {
2323                                         retval = i40e_aq_rc_to_posix(aq_ret,
2324                                                         hw->aq.asq_last_status);
2325                                         dev_info(&pf->pdev->dev,
2326                                                  "Set default VSI failed on %s, err %s, aq_err %s\n",
2327                                                  vsi_name,
2328                                                  i40e_stat_str(hw, aq_ret),
2329                                                  i40e_aq_str(hw,
2330                                                      hw->aq.asq_last_status));
2331                                 }
2332                         }
2333                 } else {
2334                         aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2335                                                           hw,
2336                                                           vsi->seid,
2337                                                           cur_promisc, NULL,
2338                                                           true);
2339                         if (aq_ret) {
2340                                 retval =
2341                                 i40e_aq_rc_to_posix(aq_ret,
2342                                                     hw->aq.asq_last_status);
2343                                 dev_info(&pf->pdev->dev,
2344                                          "set unicast promisc failed on %s, err %s, aq_err %s\n",
2345                                          vsi_name,
2346                                          i40e_stat_str(hw, aq_ret),
2347                                          i40e_aq_str(hw,
2348                                                      hw->aq.asq_last_status));
2349                         }
2350                         aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2351                                                           hw,
2352                                                           vsi->seid,
2353                                                           cur_promisc, NULL);
2354                         if (aq_ret) {
2355                                 retval =
2356                                 i40e_aq_rc_to_posix(aq_ret,
2357                                                     hw->aq.asq_last_status);
2358                                 dev_info(&pf->pdev->dev,
2359                                          "set multicast promisc failed on %s, err %s, aq_err %s\n",
2360                                          vsi_name,
2361                                          i40e_stat_str(hw, aq_ret),
2362                                          i40e_aq_str(hw,
2363                                                      hw->aq.asq_last_status));
2364                         }
2365                 }
2366                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
2367                                                    vsi->seid,
2368                                                    cur_promisc, NULL);
2369                 if (aq_ret) {
2370                         retval = i40e_aq_rc_to_posix(aq_ret,
2371                                                      pf->hw.aq.asq_last_status);
2372                         dev_info(&pf->pdev->dev,
2373                                  "set brdcast promisc failed, err %s, aq_err %s\n",
2374                                          i40e_stat_str(hw, aq_ret),
2375                                          i40e_aq_str(hw,
2376                                                      hw->aq.asq_last_status));
2377                 }
2378         }
2379 out:
2380         /* if something went wrong then set the changed flag so we try again */
2381         if (retval)
2382                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2383
2384         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2385         return retval;
2386
2387 err_no_memory:
2388         /* Restore elements on the temporary add and delete lists */
2389         spin_lock_bh(&vsi->mac_filter_hash_lock);
2390 err_no_memory_locked:
2391         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2392         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2393         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2394
2395         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2396         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2397         return -ENOMEM;
2398 }
2399
2400 /**
2401  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2402  * @pf: board private structure
2403  **/
2404 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2405 {
2406         int v;
2407
2408         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
2409                 return;
2410         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
2411
2412         for (v = 0; v < pf->num_alloc_vsi; v++) {
2413                 if (pf->vsi[v] &&
2414                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2415                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2416
2417                         if (ret) {
2418                                 /* come back and try again later */
2419                                 pf->flags |= I40E_FLAG_FILTER_SYNC;
2420                                 break;
2421                         }
2422                 }
2423         }
2424 }
2425
2426 /**
2427  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2428  * @vsi: the vsi
2429  **/
2430 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2431 {
2432         if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2433                 return I40E_RXBUFFER_2048;
2434         else
2435                 return I40E_RXBUFFER_3072;
2436 }
2437
2438 /**
2439  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2440  * @netdev: network interface device structure
2441  * @new_mtu: new value for maximum frame size
2442  *
2443  * Returns 0 on success, negative on failure
2444  **/
2445 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2446 {
2447         struct i40e_netdev_priv *np = netdev_priv(netdev);
2448         struct i40e_vsi *vsi = np->vsi;
2449         struct i40e_pf *pf = vsi->back;
2450
2451         if (i40e_enabled_xdp_vsi(vsi)) {
2452                 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2453
2454                 if (frame_size > i40e_max_xdp_frame_size(vsi))
2455                         return -EINVAL;
2456         }
2457
2458         netdev_info(netdev, "changing MTU from %d to %d\n",
2459                     netdev->mtu, new_mtu);
2460         netdev->mtu = new_mtu;
2461         if (netif_running(netdev))
2462                 i40e_vsi_reinit_locked(vsi);
2463         pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
2464                       I40E_FLAG_CLIENT_L2_CHANGE);
2465         return 0;
2466 }
2467
2468 /**
2469  * i40e_ioctl - Access the hwtstamp interface
2470  * @netdev: network interface device structure
2471  * @ifr: interface request data
2472  * @cmd: ioctl command
2473  **/
2474 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2475 {
2476         struct i40e_netdev_priv *np = netdev_priv(netdev);
2477         struct i40e_pf *pf = np->vsi->back;
2478
2479         switch (cmd) {
2480         case SIOCGHWTSTAMP:
2481                 return i40e_ptp_get_ts_config(pf, ifr);
2482         case SIOCSHWTSTAMP:
2483                 return i40e_ptp_set_ts_config(pf, ifr);
2484         default:
2485                 return -EOPNOTSUPP;
2486         }
2487 }
2488
2489 /**
2490  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2491  * @vsi: the vsi being adjusted
2492  **/
2493 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2494 {
2495         struct i40e_vsi_context ctxt;
2496         i40e_status ret;
2497
2498         if ((vsi->info.valid_sections &
2499              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2500             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2501                 return;  /* already enabled */
2502
2503         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2504         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2505                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2506
2507         ctxt.seid = vsi->seid;
2508         ctxt.info = vsi->info;
2509         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2510         if (ret) {
2511                 dev_info(&vsi->back->pdev->dev,
2512                          "update vlan stripping failed, err %s aq_err %s\n",
2513                          i40e_stat_str(&vsi->back->hw, ret),
2514                          i40e_aq_str(&vsi->back->hw,
2515                                      vsi->back->hw.aq.asq_last_status));
2516         }
2517 }
2518
2519 /**
2520  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2521  * @vsi: the vsi being adjusted
2522  **/
2523 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2524 {
2525         struct i40e_vsi_context ctxt;
2526         i40e_status ret;
2527
2528         if ((vsi->info.valid_sections &
2529              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2530             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2531              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2532                 return;  /* already disabled */
2533
2534         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2535         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2536                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2537
2538         ctxt.seid = vsi->seid;
2539         ctxt.info = vsi->info;
2540         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2541         if (ret) {
2542                 dev_info(&vsi->back->pdev->dev,
2543                          "update vlan stripping failed, err %s aq_err %s\n",
2544                          i40e_stat_str(&vsi->back->hw, ret),
2545                          i40e_aq_str(&vsi->back->hw,
2546                                      vsi->back->hw.aq.asq_last_status));
2547         }
2548 }
2549
2550 /**
2551  * i40e_vlan_rx_register - Setup or shutdown vlan offload
2552  * @netdev: network interface to be adjusted
2553  * @features: netdev features to test if VLAN offload is enabled or not
2554  **/
2555 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
2556 {
2557         struct i40e_netdev_priv *np = netdev_priv(netdev);
2558         struct i40e_vsi *vsi = np->vsi;
2559
2560         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2561                 i40e_vlan_stripping_enable(vsi);
2562         else
2563                 i40e_vlan_stripping_disable(vsi);
2564 }
2565
2566 /**
2567  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2568  * @vsi: the vsi being configured
2569  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2570  *
2571  * This is a helper function for adding a new MAC/VLAN filter with the
2572  * specified VLAN for each existing MAC address already in the hash table.
2573  * This function does *not* perform any accounting to update filters based on
2574  * VLAN mode.
2575  *
2576  * NOTE: this function expects to be called while under the
2577  * mac_filter_hash_lock
2578  **/
2579 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2580 {
2581         struct i40e_mac_filter *f, *add_f;
2582         struct hlist_node *h;
2583         int bkt;
2584
2585         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2586                 if (f->state == I40E_FILTER_REMOVE)
2587                         continue;
2588                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2589                 if (!add_f) {
2590                         dev_info(&vsi->back->pdev->dev,
2591                                  "Could not add vlan filter %d for %pM\n",
2592                                  vid, f->macaddr);
2593                         return -ENOMEM;
2594                 }
2595         }
2596
2597         return 0;
2598 }
2599
2600 /**
2601  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2602  * @vsi: the VSI being configured
2603  * @vid: VLAN id to be added
2604  **/
2605 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2606 {
2607         int err;
2608
2609         if (vsi->info.pvid)
2610                 return -EINVAL;
2611
2612         /* The network stack will attempt to add VID=0, with the intention to
2613          * receive priority tagged packets with a VLAN of 0. Our HW receives
2614          * these packets by default when configured to receive untagged
2615          * packets, so we don't need to add a filter for this case.
2616          * Additionally, HW interprets adding a VID=0 filter as meaning to
2617          * receive *only* tagged traffic and stops receiving untagged traffic.
2618          * Thus, we do not want to actually add a filter for VID=0
2619          */
2620         if (!vid)
2621                 return 0;
2622
2623         /* Locked once because all functions invoked below iterates list*/
2624         spin_lock_bh(&vsi->mac_filter_hash_lock);
2625         err = i40e_add_vlan_all_mac(vsi, vid);
2626         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2627         if (err)
2628                 return err;
2629
2630         /* schedule our worker thread which will take care of
2631          * applying the new filter changes
2632          */
2633         i40e_service_event_schedule(vsi->back);
2634         return 0;
2635 }
2636
2637 /**
2638  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2639  * @vsi: the vsi being configured
2640  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2641  *
2642  * This function should be used to remove all VLAN filters which match the
2643  * given VID. It does not schedule the service event and does not take the
2644  * mac_filter_hash_lock so it may be combined with other operations under
2645  * a single invocation of the mac_filter_hash_lock.
2646  *
2647  * NOTE: this function expects to be called while under the
2648  * mac_filter_hash_lock
2649  */
2650 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2651 {
2652         struct i40e_mac_filter *f;
2653         struct hlist_node *h;
2654         int bkt;
2655
2656         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2657                 if (f->vlan == vid)
2658                         __i40e_del_filter(vsi, f);
2659         }
2660 }
2661
2662 /**
2663  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2664  * @vsi: the VSI being configured
2665  * @vid: VLAN id to be removed
2666  **/
2667 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2668 {
2669         if (!vid || vsi->info.pvid)
2670                 return;
2671
2672         spin_lock_bh(&vsi->mac_filter_hash_lock);
2673         i40e_rm_vlan_all_mac(vsi, vid);
2674         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2675
2676         /* schedule our worker thread which will take care of
2677          * applying the new filter changes
2678          */
2679         i40e_service_event_schedule(vsi->back);
2680 }
2681
2682 /**
2683  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2684  * @netdev: network interface to be adjusted
2685  * @vid: vlan id to be added
2686  *
2687  * net_device_ops implementation for adding vlan ids
2688  **/
2689 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2690                                 __always_unused __be16 proto, u16 vid)
2691 {
2692         struct i40e_netdev_priv *np = netdev_priv(netdev);
2693         struct i40e_vsi *vsi = np->vsi;
2694         int ret = 0;
2695
2696         if (vid >= VLAN_N_VID)
2697                 return -EINVAL;
2698
2699         ret = i40e_vsi_add_vlan(vsi, vid);
2700         if (!ret)
2701                 set_bit(vid, vsi->active_vlans);
2702
2703         return ret;
2704 }
2705
2706 /**
2707  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2708  * @netdev: network interface to be adjusted
2709  * @vid: vlan id to be removed
2710  *
2711  * net_device_ops implementation for removing vlan ids
2712  **/
2713 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2714                                  __always_unused __be16 proto, u16 vid)
2715 {
2716         struct i40e_netdev_priv *np = netdev_priv(netdev);
2717         struct i40e_vsi *vsi = np->vsi;
2718
2719         /* return code is ignored as there is nothing a user
2720          * can do about failure to remove and a log message was
2721          * already printed from the other function
2722          */
2723         i40e_vsi_kill_vlan(vsi, vid);
2724
2725         clear_bit(vid, vsi->active_vlans);
2726
2727         return 0;
2728 }
2729
2730 /**
2731  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2732  * @vsi: the vsi being brought back up
2733  **/
2734 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2735 {
2736         u16 vid;
2737
2738         if (!vsi->netdev)
2739                 return;
2740
2741         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2742
2743         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2744                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2745                                      vid);
2746 }
2747
2748 /**
2749  * i40e_vsi_add_pvid - Add pvid for the VSI
2750  * @vsi: the vsi being adjusted
2751  * @vid: the vlan id to set as a PVID
2752  **/
2753 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2754 {
2755         struct i40e_vsi_context ctxt;
2756         i40e_status ret;
2757
2758         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2759         vsi->info.pvid = cpu_to_le16(vid);
2760         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2761                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2762                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2763
2764         ctxt.seid = vsi->seid;
2765         ctxt.info = vsi->info;
2766         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2767         if (ret) {
2768                 dev_info(&vsi->back->pdev->dev,
2769                          "add pvid failed, err %s aq_err %s\n",
2770                          i40e_stat_str(&vsi->back->hw, ret),
2771                          i40e_aq_str(&vsi->back->hw,
2772                                      vsi->back->hw.aq.asq_last_status));
2773                 return -ENOENT;
2774         }
2775
2776         return 0;
2777 }
2778
2779 /**
2780  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2781  * @vsi: the vsi being adjusted
2782  *
2783  * Just use the vlan_rx_register() service to put it back to normal
2784  **/
2785 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2786 {
2787         i40e_vlan_stripping_disable(vsi);
2788
2789         vsi->info.pvid = 0;
2790 }
2791
2792 /**
2793  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2794  * @vsi: ptr to the VSI
2795  *
2796  * If this function returns with an error, then it's possible one or
2797  * more of the rings is populated (while the rest are not).  It is the
2798  * callers duty to clean those orphaned rings.
2799  *
2800  * Return 0 on success, negative on failure
2801  **/
2802 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2803 {
2804         int i, err = 0;
2805
2806         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2807                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2808
2809         if (!i40e_enabled_xdp_vsi(vsi))
2810                 return err;
2811
2812         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2813                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
2814
2815         return err;
2816 }
2817
2818 /**
2819  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2820  * @vsi: ptr to the VSI
2821  *
2822  * Free VSI's transmit software resources
2823  **/
2824 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2825 {
2826         int i;
2827
2828         if (vsi->tx_rings) {
2829                 for (i = 0; i < vsi->num_queue_pairs; i++)
2830                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2831                                 i40e_free_tx_resources(vsi->tx_rings[i]);
2832         }
2833
2834         if (vsi->xdp_rings) {
2835                 for (i = 0; i < vsi->num_queue_pairs; i++)
2836                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
2837                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
2838         }
2839 }
2840
2841 /**
2842  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2843  * @vsi: ptr to the VSI
2844  *
2845  * If this function returns with an error, then it's possible one or
2846  * more of the rings is populated (while the rest are not).  It is the
2847  * callers duty to clean those orphaned rings.
2848  *
2849  * Return 0 on success, negative on failure
2850  **/
2851 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2852 {
2853         int i, err = 0;
2854
2855         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2856                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2857         return err;
2858 }
2859
2860 /**
2861  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2862  * @vsi: ptr to the VSI
2863  *
2864  * Free all receive software resources
2865  **/
2866 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2867 {
2868         int i;
2869
2870         if (!vsi->rx_rings)
2871                 return;
2872
2873         for (i = 0; i < vsi->num_queue_pairs; i++)
2874                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2875                         i40e_free_rx_resources(vsi->rx_rings[i]);
2876 }
2877
2878 /**
2879  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
2880  * @ring: The Tx ring to configure
2881  *
2882  * This enables/disables XPS for a given Tx descriptor ring
2883  * based on the TCs enabled for the VSI that ring belongs to.
2884  **/
2885 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
2886 {
2887         struct i40e_vsi *vsi = ring->vsi;
2888
2889         if (!ring->q_vector || !ring->netdev)
2890                 return;
2891
2892         if ((vsi->tc_config.numtc <= 1) &&
2893             !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state)) {
2894                 netif_set_xps_queue(ring->netdev,
2895                                     get_cpu_mask(ring->q_vector->v_idx),
2896                                     ring->queue_index);
2897         }
2898
2899         /* schedule our worker thread which will take care of
2900          * applying the new filter changes
2901          */
2902         i40e_service_event_schedule(vsi->back);
2903 }
2904
2905 /**
2906  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2907  * @ring: The Tx ring to configure
2908  *
2909  * Configure the Tx descriptor ring in the HMC context.
2910  **/
2911 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2912 {
2913         struct i40e_vsi *vsi = ring->vsi;
2914         u16 pf_q = vsi->base_queue + ring->queue_index;
2915         struct i40e_hw *hw = &vsi->back->hw;
2916         struct i40e_hmc_obj_txq tx_ctx;
2917         i40e_status err = 0;
2918         u32 qtx_ctl = 0;
2919
2920         /* some ATR related tx ring init */
2921         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
2922                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2923                 ring->atr_count = 0;
2924         } else {
2925                 ring->atr_sample_rate = 0;
2926         }
2927
2928         /* configure XPS */
2929         i40e_config_xps_tx_ring(ring);
2930
2931         /* clear the context structure first */
2932         memset(&tx_ctx, 0, sizeof(tx_ctx));
2933
2934         tx_ctx.new_context = 1;
2935         tx_ctx.base = (ring->dma / 128);
2936         tx_ctx.qlen = ring->count;
2937         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
2938                                                I40E_FLAG_FD_ATR_ENABLED));
2939         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2940         /* FDIR VSI tx ring can still use RS bit and writebacks */
2941         if (vsi->type != I40E_VSI_FDIR)
2942                 tx_ctx.head_wb_ena = 1;
2943         tx_ctx.head_wb_addr = ring->dma +
2944                               (ring->count * sizeof(struct i40e_tx_desc));
2945
2946         /* As part of VSI creation/update, FW allocates certain
2947          * Tx arbitration queue sets for each TC enabled for
2948          * the VSI. The FW returns the handles to these queue
2949          * sets as part of the response buffer to Add VSI,
2950          * Update VSI, etc. AQ commands. It is expected that
2951          * these queue set handles be associated with the Tx
2952          * queues by the driver as part of the TX queue context
2953          * initialization. This has to be done regardless of
2954          * DCB as by default everything is mapped to TC0.
2955          */
2956         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2957         tx_ctx.rdylist_act = 0;
2958
2959         /* clear the context in the HMC */
2960         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2961         if (err) {
2962                 dev_info(&vsi->back->pdev->dev,
2963                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2964                          ring->queue_index, pf_q, err);
2965                 return -ENOMEM;
2966         }
2967
2968         /* set the context in the HMC */
2969         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2970         if (err) {
2971                 dev_info(&vsi->back->pdev->dev,
2972                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2973                          ring->queue_index, pf_q, err);
2974                 return -ENOMEM;
2975         }
2976
2977         /* Now associate this queue with this PCI function */
2978         if (vsi->type == I40E_VSI_VMDQ2) {
2979                 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
2980                 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
2981                            I40E_QTX_CTL_VFVM_INDX_MASK;
2982         } else {
2983                 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2984         }
2985
2986         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2987                     I40E_QTX_CTL_PF_INDX_MASK);
2988         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2989         i40e_flush(hw);
2990
2991         /* cache tail off for easier writes later */
2992         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2993
2994         return 0;
2995 }
2996
2997 /**
2998  * i40e_configure_rx_ring - Configure a receive ring context
2999  * @ring: The Rx ring to configure
3000  *
3001  * Configure the Rx descriptor ring in the HMC context.
3002  **/
3003 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3004 {
3005         struct i40e_vsi *vsi = ring->vsi;
3006         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3007         u16 pf_q = vsi->base_queue + ring->queue_index;
3008         struct i40e_hw *hw = &vsi->back->hw;
3009         struct i40e_hmc_obj_rxq rx_ctx;
3010         i40e_status err = 0;
3011
3012         ring->state = 0;
3013
3014         /* clear the context structure first */
3015         memset(&rx_ctx, 0, sizeof(rx_ctx));
3016
3017         ring->rx_buf_len = vsi->rx_buf_len;
3018
3019         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3020                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3021
3022         rx_ctx.base = (ring->dma / 128);
3023         rx_ctx.qlen = ring->count;
3024
3025         /* use 32 byte descriptors */
3026         rx_ctx.dsize = 1;
3027
3028         /* descriptor type is always zero
3029          * rx_ctx.dtype = 0;
3030          */
3031         rx_ctx.hsplit_0 = 0;
3032
3033         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3034         if (hw->revision_id == 0)
3035                 rx_ctx.lrxqthresh = 0;
3036         else
3037                 rx_ctx.lrxqthresh = 2;
3038         rx_ctx.crcstrip = 1;
3039         rx_ctx.l2tsel = 1;
3040         /* this controls whether VLAN is stripped from inner headers */
3041         rx_ctx.showiv = 0;
3042         /* set the prefena field to 1 because the manual says to */
3043         rx_ctx.prefena = 1;
3044
3045         /* clear the context in the HMC */
3046         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3047         if (err) {
3048                 dev_info(&vsi->back->pdev->dev,
3049                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3050                          ring->queue_index, pf_q, err);
3051                 return -ENOMEM;
3052         }
3053
3054         /* set the context in the HMC */
3055         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3056         if (err) {
3057                 dev_info(&vsi->back->pdev->dev,
3058                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3059                          ring->queue_index, pf_q, err);
3060                 return -ENOMEM;
3061         }
3062
3063         /* configure Rx buffer alignment */
3064         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3065                 clear_ring_build_skb_enabled(ring);
3066         else
3067                 set_ring_build_skb_enabled(ring);
3068
3069         /* cache tail for quicker writes, and clear the reg before use */
3070         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3071         writel(0, ring->tail);
3072
3073         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3074
3075         return 0;
3076 }
3077
3078 /**
3079  * i40e_vsi_configure_tx - Configure the VSI for Tx
3080  * @vsi: VSI structure describing this set of rings and resources
3081  *
3082  * Configure the Tx VSI for operation.
3083  **/
3084 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3085 {
3086         int err = 0;
3087         u16 i;
3088
3089         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3090                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3091
3092         if (!i40e_enabled_xdp_vsi(vsi))
3093                 return err;
3094
3095         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3096                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3097
3098         return err;
3099 }
3100
3101 /**
3102  * i40e_vsi_configure_rx - Configure the VSI for Rx
3103  * @vsi: the VSI being configured
3104  *
3105  * Configure the Rx VSI for operation.
3106  **/
3107 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3108 {
3109         int err = 0;
3110         u16 i;
3111
3112         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3113                 vsi->max_frame = I40E_MAX_RXBUFFER;
3114                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3115 #if (PAGE_SIZE < 8192)
3116         } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3117                    (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3118                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3119                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3120 #endif
3121         } else {
3122                 vsi->max_frame = I40E_MAX_RXBUFFER;
3123                 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3124                                                        I40E_RXBUFFER_2048;
3125         }
3126
3127         /* set up individual rings */
3128         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3129                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3130
3131         return err;
3132 }
3133
3134 /**
3135  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3136  * @vsi: ptr to the VSI
3137  **/
3138 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3139 {
3140         struct i40e_ring *tx_ring, *rx_ring;
3141         u16 qoffset, qcount;
3142         int i, n;
3143
3144         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3145                 /* Reset the TC information */
3146                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3147                         rx_ring = vsi->rx_rings[i];
3148                         tx_ring = vsi->tx_rings[i];
3149                         rx_ring->dcb_tc = 0;
3150                         tx_ring->dcb_tc = 0;
3151                 }
3152         }
3153
3154         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3155                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3156                         continue;
3157
3158                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3159                 qcount = vsi->tc_config.tc_info[n].qcount;
3160                 for (i = qoffset; i < (qoffset + qcount); i++) {
3161                         rx_ring = vsi->rx_rings[i];
3162                         tx_ring = vsi->tx_rings[i];
3163                         rx_ring->dcb_tc = n;
3164                         tx_ring->dcb_tc = n;
3165                 }
3166         }
3167 }
3168
3169 /**
3170  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3171  * @vsi: ptr to the VSI
3172  **/
3173 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3174 {
3175         if (vsi->netdev)
3176                 i40e_set_rx_mode(vsi->netdev);
3177 }
3178
3179 /**
3180  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3181  * @vsi: Pointer to the targeted VSI
3182  *
3183  * This function replays the hlist on the hw where all the SB Flow Director
3184  * filters were saved.
3185  **/
3186 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3187 {
3188         struct i40e_fdir_filter *filter;
3189         struct i40e_pf *pf = vsi->back;
3190         struct hlist_node *node;
3191
3192         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3193                 return;
3194
3195         /* Reset FDir counters as we're replaying all existing filters */
3196         pf->fd_tcp4_filter_cnt = 0;
3197         pf->fd_udp4_filter_cnt = 0;
3198         pf->fd_sctp4_filter_cnt = 0;
3199         pf->fd_ip4_filter_cnt = 0;
3200
3201         hlist_for_each_entry_safe(filter, node,
3202                                   &pf->fdir_filter_list, fdir_node) {
3203                 i40e_add_del_fdir(vsi, filter, true);
3204         }
3205 }
3206
3207 /**
3208  * i40e_vsi_configure - Set up the VSI for action
3209  * @vsi: the VSI being configured
3210  **/
3211 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3212 {
3213         int err;
3214
3215         i40e_set_vsi_rx_mode(vsi);
3216         i40e_restore_vlan(vsi);
3217         i40e_vsi_config_dcb_rings(vsi);
3218         err = i40e_vsi_configure_tx(vsi);
3219         if (!err)
3220                 err = i40e_vsi_configure_rx(vsi);
3221
3222         return err;
3223 }
3224
3225 /**
3226  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3227  * @vsi: the VSI being configured
3228  **/
3229 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3230 {
3231         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3232         struct i40e_pf *pf = vsi->back;
3233         struct i40e_hw *hw = &pf->hw;
3234         u16 vector;
3235         int i, q;
3236         u32 qp;
3237
3238         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3239          * and PFINT_LNKLSTn registers, e.g.:
3240          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3241          */
3242         qp = vsi->base_queue;
3243         vector = vsi->base_vector;
3244         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3245                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3246
3247                 q_vector->itr_countdown = ITR_COUNTDOWN_START;
3248                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting);
3249                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
3250                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3251                      q_vector->rx.itr);
3252                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting);
3253                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
3254                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3255                      q_vector->tx.itr);
3256                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3257                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3258
3259                 /* Linked list for the queuepairs assigned to this vector */
3260                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3261                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3262                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3263                         u32 val;
3264
3265                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3266                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3267                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3268                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3269                               (I40E_QUEUE_TYPE_TX <<
3270                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3271
3272                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3273
3274                         if (has_xdp) {
3275                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3276                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3277                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3278                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3279                                       (I40E_QUEUE_TYPE_TX <<
3280                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3281
3282                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3283                         }
3284
3285                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3286                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3287                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3288                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3289                               (I40E_QUEUE_TYPE_RX <<
3290                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3291
3292                         /* Terminate the linked list */
3293                         if (q == (q_vector->num_ringpairs - 1))
3294                                 val |= (I40E_QUEUE_END_OF_LIST <<
3295                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3296
3297                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3298                         qp++;
3299                 }
3300         }
3301
3302         i40e_flush(hw);
3303 }
3304
3305 /**
3306  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3307  * @hw: ptr to the hardware info
3308  **/
3309 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3310 {
3311         struct i40e_hw *hw = &pf->hw;
3312         u32 val;
3313
3314         /* clear things first */
3315         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3316         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3317
3318         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3319               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3320               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3321               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3322               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3323               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3324               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3325               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3326
3327         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3328                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3329
3330         if (pf->flags & I40E_FLAG_PTP)
3331                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3332
3333         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3334
3335         /* SW_ITR_IDX = 0, but don't change INTENA */
3336         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3337                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3338
3339         /* OTHER_ITR_IDX = 0 */
3340         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3341 }
3342
3343 /**
3344  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3345  * @vsi: the VSI being configured
3346  **/
3347 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3348 {
3349         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3350         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3351         struct i40e_pf *pf = vsi->back;
3352         struct i40e_hw *hw = &pf->hw;
3353         u32 val;
3354
3355         /* set the ITR configuration */
3356         q_vector->itr_countdown = ITR_COUNTDOWN_START;
3357         q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting);
3358         q_vector->rx.latency_range = I40E_LOW_LATENCY;
3359         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
3360         q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting);
3361         q_vector->tx.latency_range = I40E_LOW_LATENCY;
3362         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
3363
3364         i40e_enable_misc_int_causes(pf);
3365
3366         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3367         wr32(hw, I40E_PFINT_LNKLST0, 0);
3368
3369         /* Associate the queue pair to the vector and enable the queue int */
3370         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                   |
3371               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3372               (nextqp      << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3373               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3374
3375         wr32(hw, I40E_QINT_RQCTL(0), val);
3376
3377         if (i40e_enabled_xdp_vsi(vsi)) {
3378                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                 |
3379                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3380                       (I40E_QUEUE_TYPE_TX
3381                        << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3382
3383                wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3384         }
3385
3386         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3387               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3388               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3389
3390         wr32(hw, I40E_QINT_TQCTL(0), val);
3391         i40e_flush(hw);
3392 }
3393
3394 /**
3395  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3396  * @pf: board private structure
3397  **/
3398 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3399 {
3400         struct i40e_hw *hw = &pf->hw;
3401
3402         wr32(hw, I40E_PFINT_DYN_CTL0,
3403              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3404         i40e_flush(hw);
3405 }
3406
3407 /**
3408  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3409  * @pf: board private structure
3410  * @clearpba: true when all pending interrupt events should be cleared
3411  **/
3412 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba)
3413 {
3414         struct i40e_hw *hw = &pf->hw;
3415         u32 val;
3416
3417         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3418               (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) |
3419               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3420
3421         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3422         i40e_flush(hw);
3423 }
3424
3425 /**
3426  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3427  * @irq: interrupt number
3428  * @data: pointer to a q_vector
3429  **/
3430 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3431 {
3432         struct i40e_q_vector *q_vector = data;
3433
3434         if (!q_vector->tx.ring && !q_vector->rx.ring)
3435                 return IRQ_HANDLED;
3436
3437         napi_schedule_irqoff(&q_vector->napi);
3438
3439         return IRQ_HANDLED;
3440 }
3441
3442 /**
3443  * i40e_irq_affinity_notify - Callback for affinity changes
3444  * @notify: context as to what irq was changed
3445  * @mask: the new affinity mask
3446  *
3447  * This is a callback function used by the irq_set_affinity_notifier function
3448  * so that we may register to receive changes to the irq affinity masks.
3449  **/
3450 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3451                                      const cpumask_t *mask)
3452 {
3453         struct i40e_q_vector *q_vector =
3454                 container_of(notify, struct i40e_q_vector, affinity_notify);
3455
3456         cpumask_copy(&q_vector->affinity_mask, mask);
3457 }
3458
3459 /**
3460  * i40e_irq_affinity_release - Callback for affinity notifier release
3461  * @ref: internal core kernel usage
3462  *
3463  * This is a callback function used by the irq_set_affinity_notifier function
3464  * to inform the current notification subscriber that they will no longer
3465  * receive notifications.
3466  **/
3467 static void i40e_irq_affinity_release(struct kref *ref) {}
3468
3469 /**
3470  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3471  * @vsi: the VSI being configured
3472  * @basename: name for the vector
3473  *
3474  * Allocates MSI-X vectors and requests interrupts from the kernel.
3475  **/
3476 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3477 {
3478         int q_vectors = vsi->num_q_vectors;
3479         struct i40e_pf *pf = vsi->back;
3480         int base = vsi->base_vector;
3481         int rx_int_idx = 0;
3482         int tx_int_idx = 0;
3483         int vector, err;
3484         int irq_num;
3485
3486         for (vector = 0; vector < q_vectors; vector++) {
3487                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3488
3489                 irq_num = pf->msix_entries[base + vector].vector;
3490
3491                 if (q_vector->tx.ring && q_vector->rx.ring) {
3492                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3493                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3494                         tx_int_idx++;
3495                 } else if (q_vector->rx.ring) {
3496                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3497                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3498                 } else if (q_vector->tx.ring) {
3499                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3500                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3501                 } else {
3502                         /* skip this unused q_vector */
3503                         continue;
3504                 }
3505                 err = request_irq(irq_num,
3506                                   vsi->irq_handler,
3507                                   0,
3508                                   q_vector->name,
3509                                   q_vector);
3510                 if (err) {
3511                         dev_info(&pf->pdev->dev,
3512                                  "MSIX request_irq failed, error: %d\n", err);
3513                         goto free_queue_irqs;
3514                 }
3515
3516                 /* register for affinity change notifications */
3517                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3518                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3519                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3520                 /* get_cpu_mask returns a static constant mask with
3521                  * a permanent lifetime so it's ok to use here.
3522                  */
3523                 irq_set_affinity_hint(irq_num, get_cpu_mask(q_vector->v_idx));
3524         }
3525
3526         vsi->irqs_ready = true;
3527         return 0;
3528
3529 free_queue_irqs:
3530         while (vector) {
3531                 vector--;
3532                 irq_num = pf->msix_entries[base + vector].vector;
3533                 irq_set_affinity_notifier(irq_num, NULL);
3534                 irq_set_affinity_hint(irq_num, NULL);
3535                 free_irq(irq_num, &vsi->q_vectors[vector]);
3536         }
3537         return err;
3538 }
3539
3540 /**
3541  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3542  * @vsi: the VSI being un-configured
3543  **/
3544 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3545 {
3546         struct i40e_pf *pf = vsi->back;
3547         struct i40e_hw *hw = &pf->hw;
3548         int base = vsi->base_vector;
3549         int i;
3550
3551         /* disable interrupt causation from each queue */
3552         for (i = 0; i < vsi->num_queue_pairs; i++) {
3553                 u32 val;
3554
3555                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
3556                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3557                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
3558
3559                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
3560                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3561                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
3562
3563                 if (!i40e_enabled_xdp_vsi(vsi))
3564                         continue;
3565                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3566         }
3567
3568         /* disable each interrupt */
3569         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3570                 for (i = vsi->base_vector;
3571                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3572                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3573
3574                 i40e_flush(hw);
3575                 for (i = 0; i < vsi->num_q_vectors; i++)
3576                         synchronize_irq(pf->msix_entries[i + base].vector);
3577         } else {
3578                 /* Legacy and MSI mode - this stops all interrupt handling */
3579                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3580                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3581                 i40e_flush(hw);
3582                 synchronize_irq(pf->pdev->irq);
3583         }
3584 }
3585
3586 /**
3587  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3588  * @vsi: the VSI being configured
3589  **/
3590 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3591 {
3592         struct i40e_pf *pf = vsi->back;
3593         int i;
3594
3595         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3596                 for (i = 0; i < vsi->num_q_vectors; i++)
3597                         i40e_irq_dynamic_enable(vsi, i);
3598         } else {
3599                 i40e_irq_dynamic_enable_icr0(pf, true);
3600         }
3601
3602         i40e_flush(&pf->hw);
3603         return 0;
3604 }
3605
3606 /**
3607  * i40e_free_misc_vector - Free the vector that handles non-queue events
3608  * @pf: board private structure
3609  **/
3610 static void i40e_free_misc_vector(struct i40e_pf *pf)
3611 {
3612         /* Disable ICR 0 */
3613         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3614         i40e_flush(&pf->hw);
3615
3616         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
3617                 synchronize_irq(pf->msix_entries[0].vector);
3618                 free_irq(pf->msix_entries[0].vector, pf);
3619                 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
3620         }
3621 }
3622
3623 /**
3624  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3625  * @irq: interrupt number
3626  * @data: pointer to a q_vector
3627  *
3628  * This is the handler used for all MSI/Legacy interrupts, and deals
3629  * with both queue and non-queue interrupts.  This is also used in
3630  * MSIX mode to handle the non-queue interrupts.
3631  **/
3632 static irqreturn_t i40e_intr(int irq, void *data)
3633 {
3634         struct i40e_pf *pf = (struct i40e_pf *)data;
3635         struct i40e_hw *hw = &pf->hw;
3636         irqreturn_t ret = IRQ_NONE;
3637         u32 icr0, icr0_remaining;
3638         u32 val, ena_mask;
3639
3640         icr0 = rd32(hw, I40E_PFINT_ICR0);
3641         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3642
3643         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3644         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3645                 goto enable_intr;
3646
3647         /* if interrupt but no bits showing, must be SWINT */
3648         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3649             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3650                 pf->sw_int_count++;
3651
3652         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3653             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3654                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3655                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3656                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
3657         }
3658
3659         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3660         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3661                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3662                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3663
3664                 /* We do not have a way to disarm Queue causes while leaving
3665                  * interrupt enabled for all other causes, ideally
3666                  * interrupt should be disabled while we are in NAPI but
3667                  * this is not a performance path and napi_schedule()
3668                  * can deal with rescheduling.
3669                  */
3670                 if (!test_bit(__I40E_DOWN, pf->state))
3671                         napi_schedule_irqoff(&q_vector->napi);
3672         }
3673
3674         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3675                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3676                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
3677                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3678         }
3679
3680         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3681                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3682                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
3683         }
3684
3685         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3686                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3687                 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
3688         }
3689
3690         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3691                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
3692                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
3693                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3694                 val = rd32(hw, I40E_GLGEN_RSTAT);
3695                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3696                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3697                 if (val == I40E_RESET_CORER) {
3698                         pf->corer_count++;
3699                 } else if (val == I40E_RESET_GLOBR) {
3700                         pf->globr_count++;
3701                 } else if (val == I40E_RESET_EMPR) {
3702                         pf->empr_count++;
3703                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
3704                 }
3705         }
3706
3707         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3708                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3709                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3710                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
3711                          rd32(hw, I40E_PFHMC_ERRORINFO),
3712                          rd32(hw, I40E_PFHMC_ERRORDATA));
3713         }
3714
3715         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3716                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3717
3718                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
3719                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3720                         i40e_ptp_tx_hwtstamp(pf);
3721                 }
3722         }
3723
3724         /* If a critical error is pending we have no choice but to reset the
3725          * device.
3726          * Report and mask out any remaining unexpected interrupts.
3727          */
3728         icr0_remaining = icr0 & ena_mask;
3729         if (icr0_remaining) {
3730                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
3731                          icr0_remaining);
3732                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
3733                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
3734                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
3735                         dev_info(&pf->pdev->dev, "device will be reset\n");
3736                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
3737                         i40e_service_event_schedule(pf);
3738                 }
3739                 ena_mask &= ~icr0_remaining;
3740         }
3741         ret = IRQ_HANDLED;
3742
3743 enable_intr:
3744         /* re-enable interrupt causes */
3745         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
3746         if (!test_bit(__I40E_DOWN, pf->state)) {
3747                 i40e_service_event_schedule(pf);
3748                 i40e_irq_dynamic_enable_icr0(pf, false);
3749         }
3750
3751         return ret;
3752 }
3753
3754 /**
3755  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
3756  * @tx_ring:  tx ring to clean
3757  * @budget:   how many cleans we're allowed
3758  *
3759  * Returns true if there's any budget left (e.g. the clean is finished)
3760  **/
3761 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
3762 {
3763         struct i40e_vsi *vsi = tx_ring->vsi;
3764         u16 i = tx_ring->next_to_clean;
3765         struct i40e_tx_buffer *tx_buf;
3766         struct i40e_tx_desc *tx_desc;
3767
3768         tx_buf = &tx_ring->tx_bi[i];
3769         tx_desc = I40E_TX_DESC(tx_ring, i);
3770         i -= tx_ring->count;
3771
3772         do {
3773                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
3774
3775                 /* if next_to_watch is not set then there is no work pending */
3776                 if (!eop_desc)
3777                         break;
3778
3779                 /* prevent any other reads prior to eop_desc */
3780                 read_barrier_depends();
3781
3782                 /* if the descriptor isn't done, no work yet to do */
3783                 if (!(eop_desc->cmd_type_offset_bsz &
3784                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
3785                         break;
3786
3787                 /* clear next_to_watch to prevent false hangs */
3788                 tx_buf->next_to_watch = NULL;
3789
3790                 tx_desc->buffer_addr = 0;
3791                 tx_desc->cmd_type_offset_bsz = 0;
3792                 /* move past filter desc */
3793                 tx_buf++;
3794                 tx_desc++;
3795                 i++;
3796                 if (unlikely(!i)) {
3797                         i -= tx_ring->count;
3798                         tx_buf = tx_ring->tx_bi;
3799                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3800                 }
3801                 /* unmap skb header data */
3802                 dma_unmap_single(tx_ring->dev,
3803                                  dma_unmap_addr(tx_buf, dma),
3804                                  dma_unmap_len(tx_buf, len),
3805                                  DMA_TO_DEVICE);
3806                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
3807                         kfree(tx_buf->raw_buf);
3808
3809                 tx_buf->raw_buf = NULL;
3810                 tx_buf->tx_flags = 0;
3811                 tx_buf->next_to_watch = NULL;
3812                 dma_unmap_len_set(tx_buf, len, 0);
3813                 tx_desc->buffer_addr = 0;
3814                 tx_desc->cmd_type_offset_bsz = 0;
3815
3816                 /* move us past the eop_desc for start of next FD desc */
3817                 tx_buf++;
3818                 tx_desc++;
3819                 i++;
3820                 if (unlikely(!i)) {
3821                         i -= tx_ring->count;
3822                         tx_buf = tx_ring->tx_bi;
3823                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3824                 }
3825
3826                 /* update budget accounting */
3827                 budget--;
3828         } while (likely(budget));
3829
3830         i += tx_ring->count;
3831         tx_ring->next_to_clean = i;
3832
3833         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
3834                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
3835
3836         return budget > 0;
3837 }
3838
3839 /**
3840  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
3841  * @irq: interrupt number
3842  * @data: pointer to a q_vector
3843  **/
3844 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
3845 {
3846         struct i40e_q_vector *q_vector = data;
3847         struct i40e_vsi *vsi;
3848
3849         if (!q_vector->tx.ring)
3850                 return IRQ_HANDLED;
3851
3852         vsi = q_vector->tx.ring->vsi;
3853         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
3854
3855         return IRQ_HANDLED;
3856 }
3857
3858 /**
3859  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
3860  * @vsi: the VSI being configured
3861  * @v_idx: vector index
3862  * @qp_idx: queue pair index
3863  **/
3864 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
3865 {
3866         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3867         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
3868         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
3869
3870         tx_ring->q_vector = q_vector;
3871         tx_ring->next = q_vector->tx.ring;
3872         q_vector->tx.ring = tx_ring;
3873         q_vector->tx.count++;
3874
3875         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
3876         if (i40e_enabled_xdp_vsi(vsi)) {
3877                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
3878
3879                 xdp_ring->q_vector = q_vector;
3880                 xdp_ring->next = q_vector->tx.ring;
3881                 q_vector->tx.ring = xdp_ring;
3882                 q_vector->tx.count++;
3883         }
3884
3885         rx_ring->q_vector = q_vector;
3886         rx_ring->next = q_vector->rx.ring;
3887         q_vector->rx.ring = rx_ring;
3888         q_vector->rx.count++;
3889 }
3890
3891 /**
3892  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
3893  * @vsi: the VSI being configured
3894  *
3895  * This function maps descriptor rings to the queue-specific vectors
3896  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
3897  * one vector per queue pair, but on a constrained vector budget, we
3898  * group the queue pairs as "efficiently" as possible.
3899  **/
3900 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
3901 {
3902         int qp_remaining = vsi->num_queue_pairs;
3903         int q_vectors = vsi->num_q_vectors;
3904         int num_ringpairs;
3905         int v_start = 0;
3906         int qp_idx = 0;
3907
3908         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
3909          * group them so there are multiple queues per vector.
3910          * It is also important to go through all the vectors available to be
3911          * sure that if we don't use all the vectors, that the remaining vectors
3912          * are cleared. This is especially important when decreasing the
3913          * number of queues in use.
3914          */
3915         for (; v_start < q_vectors; v_start++) {
3916                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
3917
3918                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
3919
3920                 q_vector->num_ringpairs = num_ringpairs;
3921
3922                 q_vector->rx.count = 0;
3923                 q_vector->tx.count = 0;
3924                 q_vector->rx.ring = NULL;
3925                 q_vector->tx.ring = NULL;
3926
3927                 while (num_ringpairs--) {
3928                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
3929                         qp_idx++;
3930                         qp_remaining--;
3931                 }
3932         }
3933 }
3934
3935 /**
3936  * i40e_vsi_request_irq - Request IRQ from the OS
3937  * @vsi: the VSI being configured
3938  * @basename: name for the vector
3939  **/
3940 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
3941 {
3942         struct i40e_pf *pf = vsi->back;
3943         int err;
3944
3945         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3946                 err = i40e_vsi_request_irq_msix(vsi, basename);
3947         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
3948                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
3949                                   pf->int_name, pf);
3950         else
3951                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
3952                                   pf->int_name, pf);
3953
3954         if (err)
3955                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
3956
3957         return err;
3958 }
3959
3960 #ifdef CONFIG_NET_POLL_CONTROLLER
3961 /**
3962  * i40e_netpoll - A Polling 'interrupt' handler
3963  * @netdev: network interface device structure
3964  *
3965  * This is used by netconsole to send skbs without having to re-enable
3966  * interrupts.  It's not called while the normal interrupt routine is executing.
3967  **/
3968 static void i40e_netpoll(struct net_device *netdev)
3969 {
3970         struct i40e_netdev_priv *np = netdev_priv(netdev);
3971         struct i40e_vsi *vsi = np->vsi;
3972         struct i40e_pf *pf = vsi->back;
3973         int i;
3974
3975         /* if interface is down do nothing */
3976         if (test_bit(__I40E_VSI_DOWN, vsi->state))
3977                 return;
3978
3979         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3980                 for (i = 0; i < vsi->num_q_vectors; i++)
3981                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
3982         } else {
3983                 i40e_intr(pf->pdev->irq, netdev);
3984         }
3985 }
3986 #endif
3987
3988 #define I40E_QTX_ENA_WAIT_COUNT 50
3989
3990 /**
3991  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
3992  * @pf: the PF being configured
3993  * @pf_q: the PF queue
3994  * @enable: enable or disable state of the queue
3995  *
3996  * This routine will wait for the given Tx queue of the PF to reach the
3997  * enabled or disabled state.
3998  * Returns -ETIMEDOUT in case of failing to reach the requested state after
3999  * multiple retries; else will return 0 in case of success.
4000  **/
4001 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4002 {
4003         int i;
4004         u32 tx_reg;
4005
4006         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4007                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4008                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4009                         break;
4010
4011                 usleep_range(10, 20);
4012         }
4013         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4014                 return -ETIMEDOUT;
4015
4016         return 0;
4017 }
4018
4019 /**
4020  * i40e_control_tx_q - Start or stop a particular Tx queue
4021  * @pf: the PF structure
4022  * @pf_q: the PF queue to configure
4023  * @enable: start or stop the queue
4024  *
4025  * This function enables or disables a single queue. Note that any delay
4026  * required after the operation is expected to be handled by the caller of
4027  * this function.
4028  **/
4029 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4030 {
4031         struct i40e_hw *hw = &pf->hw;
4032         u32 tx_reg;
4033         int i;
4034
4035         /* warn the TX unit of coming changes */
4036         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4037         if (!enable)
4038                 usleep_range(10, 20);
4039
4040         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4041                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4042                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4043                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4044                         break;
4045                 usleep_range(1000, 2000);
4046         }
4047
4048         /* Skip if the queue is already in the requested state */
4049         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4050                 return;
4051
4052         /* turn on/off the queue */
4053         if (enable) {
4054                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4055                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4056         } else {
4057                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4058         }
4059
4060         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4061 }
4062
4063 /**
4064  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4065  * @seid: VSI SEID
4066  * @pf: the PF structure
4067  * @pf_q: the PF queue to configure
4068  * @is_xdp: true if the queue is used for XDP
4069  * @enable: start or stop the queue
4070  **/
4071 static int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4072                                   bool is_xdp, bool enable)
4073 {
4074         int ret;
4075
4076         i40e_control_tx_q(pf, pf_q, enable);
4077
4078         /* wait for the change to finish */
4079         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4080         if (ret) {
4081                 dev_info(&pf->pdev->dev,
4082                          "VSI seid %d %sTx ring %d %sable timeout\n",
4083                          seid, (is_xdp ? "XDP " : ""), pf_q,
4084                          (enable ? "en" : "dis"));
4085         }
4086
4087         return ret;
4088 }
4089
4090 /**
4091  * i40e_vsi_control_tx - Start or stop a VSI's rings
4092  * @vsi: the VSI being configured
4093  * @enable: start or stop the rings
4094  **/
4095 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
4096 {
4097         struct i40e_pf *pf = vsi->back;
4098         int i, pf_q, ret = 0;
4099
4100         pf_q = vsi->base_queue;
4101         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4102                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4103                                              pf_q,
4104                                              false /*is xdp*/, enable);
4105                 if (ret)
4106                         break;
4107
4108                 if (!i40e_enabled_xdp_vsi(vsi))
4109                         continue;
4110
4111                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4112                                              pf_q + vsi->alloc_queue_pairs,
4113                                              true /*is xdp*/, enable);
4114                 if (ret)
4115                         break;
4116         }
4117
4118         return ret;
4119 }
4120
4121 /**
4122  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4123  * @pf: the PF being configured
4124  * @pf_q: the PF queue
4125  * @enable: enable or disable state of the queue
4126  *
4127  * This routine will wait for the given Rx queue of the PF to reach the
4128  * enabled or disabled state.
4129  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4130  * multiple retries; else will return 0 in case of success.
4131  **/
4132 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4133 {
4134         int i;
4135         u32 rx_reg;
4136
4137         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4138                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4139                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4140                         break;
4141
4142                 usleep_range(10, 20);
4143         }
4144         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4145                 return -ETIMEDOUT;
4146
4147         return 0;
4148 }
4149
4150 /**
4151  * i40e_control_rx_q - Start or stop a particular Rx queue
4152  * @pf: the PF structure
4153  * @pf_q: the PF queue to configure
4154  * @enable: start or stop the queue
4155  *
4156  * This function enables or disables a single queue. Note that any delay
4157  * required after the operation is expected to be handled by the caller of
4158  * this function.
4159  **/
4160 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4161 {
4162         struct i40e_hw *hw = &pf->hw;
4163         u32 rx_reg;
4164         int i;
4165
4166         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4167                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4168                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4169                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4170                         break;
4171                 usleep_range(1000, 2000);
4172         }
4173
4174         /* Skip if the queue is already in the requested state */
4175         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4176                 return;
4177
4178         /* turn on/off the queue */
4179         if (enable)
4180                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4181         else
4182                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4183
4184         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4185 }
4186
4187 /**
4188  * i40e_vsi_control_rx - Start or stop a VSI's rings
4189  * @vsi: the VSI being configured
4190  * @enable: start or stop the rings
4191  **/
4192 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
4193 {
4194         struct i40e_pf *pf = vsi->back;
4195         int i, pf_q, ret = 0;
4196
4197         pf_q = vsi->base_queue;
4198         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4199                 i40e_control_rx_q(pf, pf_q, enable);
4200
4201                 /* wait for the change to finish */
4202                 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4203                 if (ret) {
4204                         dev_info(&pf->pdev->dev,
4205                                  "VSI seid %d Rx ring %d %sable timeout\n",
4206                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
4207                         break;
4208                 }
4209         }
4210
4211         /* Due to HW errata, on Rx disable only, the register can indicate done
4212          * before it really is. Needs 50ms to be sure
4213          */
4214         if (!enable)
4215                 mdelay(50);
4216
4217         return ret;
4218 }
4219
4220 /**
4221  * i40e_vsi_start_rings - Start a VSI's rings
4222  * @vsi: the VSI being configured
4223  **/
4224 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4225 {
4226         int ret = 0;
4227
4228         /* do rx first for enable and last for disable */
4229         ret = i40e_vsi_control_rx(vsi, true);
4230         if (ret)
4231                 return ret;
4232         ret = i40e_vsi_control_tx(vsi, true);
4233
4234         return ret;
4235 }
4236
4237 /**
4238  * i40e_vsi_stop_rings - Stop a VSI's rings
4239  * @vsi: the VSI being configured
4240  **/
4241 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4242 {
4243         /* When port TX is suspended, don't wait */
4244         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4245                 return i40e_vsi_stop_rings_no_wait(vsi);
4246
4247         /* do rx first for enable and last for disable
4248          * Ignore return value, we need to shutdown whatever we can
4249          */
4250         i40e_vsi_control_tx(vsi, false);
4251         i40e_vsi_control_rx(vsi, false);
4252 }
4253
4254 /**
4255  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4256  * @vsi: the VSI being shutdown
4257  *
4258  * This function stops all the rings for a VSI but does not delay to verify
4259  * that rings have been disabled. It is expected that the caller is shutting
4260  * down multiple VSIs at once and will delay together for all the VSIs after
4261  * initiating the shutdown. This is particularly useful for shutting down lots
4262  * of VFs together. Otherwise, a large delay can be incurred while configuring
4263  * each VSI in serial.
4264  **/
4265 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4266 {
4267         struct i40e_pf *pf = vsi->back;
4268         int i, pf_q;
4269
4270         pf_q = vsi->base_queue;
4271         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4272                 i40e_control_tx_q(pf, pf_q, false);
4273                 i40e_control_rx_q(pf, pf_q, false);
4274         }
4275 }
4276
4277 /**
4278  * i40e_vsi_free_irq - Free the irq association with the OS
4279  * @vsi: the VSI being configured
4280  **/
4281 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4282 {
4283         struct i40e_pf *pf = vsi->back;
4284         struct i40e_hw *hw = &pf->hw;
4285         int base = vsi->base_vector;
4286         u32 val, qp;
4287         int i;
4288
4289         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4290                 if (!vsi->q_vectors)
4291                         return;
4292
4293                 if (!vsi->irqs_ready)
4294                         return;
4295
4296                 vsi->irqs_ready = false;
4297                 for (i = 0; i < vsi->num_q_vectors; i++) {
4298                         int irq_num;
4299                         u16 vector;
4300
4301                         vector = i + base;
4302                         irq_num = pf->msix_entries[vector].vector;
4303
4304                         /* free only the irqs that were actually requested */
4305                         if (!vsi->q_vectors[i] ||
4306                             !vsi->q_vectors[i]->num_ringpairs)
4307                                 continue;
4308
4309                         /* clear the affinity notifier in the IRQ descriptor */
4310                         irq_set_affinity_notifier(irq_num, NULL);
4311                         /* remove our suggested affinity mask for this IRQ */
4312                         irq_set_affinity_hint(irq_num, NULL);
4313                         synchronize_irq(irq_num);
4314                         free_irq(irq_num, vsi->q_vectors[i]);
4315
4316                         /* Tear down the interrupt queue link list
4317                          *
4318                          * We know that they come in pairs and always
4319                          * the Rx first, then the Tx.  To clear the
4320                          * link list, stick the EOL value into the
4321                          * next_q field of the registers.
4322                          */
4323                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4324                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4325                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4326                         val |= I40E_QUEUE_END_OF_LIST
4327                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4328                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4329
4330                         while (qp != I40E_QUEUE_END_OF_LIST) {
4331                                 u32 next;
4332
4333                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4334
4335                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4336                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4337                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4338                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4339
4340                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4341                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4342
4343                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4344
4345                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4346
4347                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4348                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4349
4350                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4351                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4352                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4353                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4354
4355                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4356                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4357
4358                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4359                                 qp = next;
4360                         }
4361                 }
4362         } else {
4363                 free_irq(pf->pdev->irq, pf);
4364
4365                 val = rd32(hw, I40E_PFINT_LNKLST0);
4366                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4367                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4368                 val |= I40E_QUEUE_END_OF_LIST
4369                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4370                 wr32(hw, I40E_PFINT_LNKLST0, val);
4371
4372                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4373                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4374                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4375                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4376                          I40E_QINT_RQCTL_INTEVENT_MASK);
4377
4378                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4379                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4380
4381                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4382
4383                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4384
4385                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4386                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4387                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4388                          I40E_QINT_TQCTL_INTEVENT_MASK);
4389
4390                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4391                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4392
4393                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4394         }
4395 }
4396
4397 /**
4398  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4399  * @vsi: the VSI being configured
4400  * @v_idx: Index of vector to be freed
4401  *
4402  * This function frees the memory allocated to the q_vector.  In addition if
4403  * NAPI is enabled it will delete any references to the NAPI struct prior
4404  * to freeing the q_vector.
4405  **/
4406 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4407 {
4408         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4409         struct i40e_ring *ring;
4410
4411         if (!q_vector)
4412                 return;
4413
4414         /* disassociate q_vector from rings */
4415         i40e_for_each_ring(ring, q_vector->tx)
4416                 ring->q_vector = NULL;
4417
4418         i40e_for_each_ring(ring, q_vector->rx)
4419                 ring->q_vector = NULL;
4420
4421         /* only VSI w/ an associated netdev is set up w/ NAPI */
4422         if (vsi->netdev)
4423                 netif_napi_del(&q_vector->napi);
4424
4425         vsi->q_vectors[v_idx] = NULL;
4426
4427         kfree_rcu(q_vector, rcu);
4428 }
4429
4430 /**
4431  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4432  * @vsi: the VSI being un-configured
4433  *
4434  * This frees the memory allocated to the q_vectors and
4435  * deletes references to the NAPI struct.
4436  **/
4437 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4438 {
4439         int v_idx;
4440
4441         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4442                 i40e_free_q_vector(vsi, v_idx);
4443 }
4444
4445 /**
4446  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4447  * @pf: board private structure
4448  **/
4449 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4450 {
4451         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4452         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4453                 pci_disable_msix(pf->pdev);
4454                 kfree(pf->msix_entries);
4455                 pf->msix_entries = NULL;
4456                 kfree(pf->irq_pile);
4457                 pf->irq_pile = NULL;
4458         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4459                 pci_disable_msi(pf->pdev);
4460         }
4461         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4462 }
4463
4464 /**
4465  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4466  * @pf: board private structure
4467  *
4468  * We go through and clear interrupt specific resources and reset the structure
4469  * to pre-load conditions
4470  **/
4471 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4472 {
4473         int i;
4474
4475         i40e_free_misc_vector(pf);
4476
4477         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4478                       I40E_IWARP_IRQ_PILE_ID);
4479
4480         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4481         for (i = 0; i < pf->num_alloc_vsi; i++)
4482                 if (pf->vsi[i])
4483                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4484         i40e_reset_interrupt_capability(pf);
4485 }
4486
4487 /**
4488  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4489  * @vsi: the VSI being configured
4490  **/
4491 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4492 {
4493         int q_idx;
4494
4495         if (!vsi->netdev)
4496                 return;
4497
4498         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4499                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4500
4501                 if (q_vector->rx.ring || q_vector->tx.ring)
4502                         napi_enable(&q_vector->napi);
4503         }
4504 }
4505
4506 /**
4507  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4508  * @vsi: the VSI being configured
4509  **/
4510 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4511 {
4512         int q_idx;
4513
4514         if (!vsi->netdev)
4515                 return;
4516
4517         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4518                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4519
4520                 if (q_vector->rx.ring || q_vector->tx.ring)
4521                         napi_disable(&q_vector->napi);
4522         }
4523 }
4524
4525 /**
4526  * i40e_vsi_close - Shut down a VSI
4527  * @vsi: the vsi to be quelled
4528  **/
4529 static void i40e_vsi_close(struct i40e_vsi *vsi)
4530 {
4531         struct i40e_pf *pf = vsi->back;
4532         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4533                 i40e_down(vsi);
4534         i40e_vsi_free_irq(vsi);
4535         i40e_vsi_free_tx_resources(vsi);
4536         i40e_vsi_free_rx_resources(vsi);
4537         vsi->current_netdev_flags = 0;
4538         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
4539         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4540                 pf->flags |=  I40E_FLAG_CLIENT_RESET;
4541 }
4542
4543 /**
4544  * i40e_quiesce_vsi - Pause a given VSI
4545  * @vsi: the VSI being paused
4546  **/
4547 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4548 {
4549         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4550                 return;
4551
4552         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4553         if (vsi->netdev && netif_running(vsi->netdev))
4554                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4555         else
4556                 i40e_vsi_close(vsi);
4557 }
4558
4559 /**
4560  * i40e_unquiesce_vsi - Resume a given VSI
4561  * @vsi: the VSI being resumed
4562  **/
4563 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4564 {
4565         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
4566                 return;
4567
4568         if (vsi->netdev && netif_running(vsi->netdev))
4569                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4570         else
4571                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4572 }
4573
4574 /**
4575  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4576  * @pf: the PF
4577  **/
4578 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4579 {
4580         int v;
4581
4582         for (v = 0; v < pf->num_alloc_vsi; v++) {
4583                 if (pf->vsi[v])
4584                         i40e_quiesce_vsi(pf->vsi[v]);
4585         }
4586 }
4587
4588 /**
4589  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4590  * @pf: the PF
4591  **/
4592 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4593 {
4594         int v;
4595
4596         for (v = 0; v < pf->num_alloc_vsi; v++) {
4597                 if (pf->vsi[v])
4598                         i40e_unquiesce_vsi(pf->vsi[v]);
4599         }
4600 }
4601
4602 /**
4603  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4604  * @vsi: the VSI being configured
4605  *
4606  * Wait until all queues on a given VSI have been disabled.
4607  **/
4608 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4609 {
4610         struct i40e_pf *pf = vsi->back;
4611         int i, pf_q, ret;
4612
4613         pf_q = vsi->base_queue;
4614         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4615                 /* Check and wait for the Tx queue */
4616                 ret = i40e_pf_txq_wait(pf, pf_q, false);
4617                 if (ret) {
4618                         dev_info(&pf->pdev->dev,
4619                                  "VSI seid %d Tx ring %d disable timeout\n",
4620                                  vsi->seid, pf_q);
4621                         return ret;
4622                 }
4623
4624                 if (!i40e_enabled_xdp_vsi(vsi))
4625                         goto wait_rx;
4626
4627                 /* Check and wait for the XDP Tx queue */
4628                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
4629                                        false);
4630                 if (ret) {
4631                         dev_info(&pf->pdev->dev,
4632                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
4633                                  vsi->seid, pf_q);
4634                         return ret;
4635                 }
4636 wait_rx:
4637                 /* Check and wait for the Rx queue */
4638                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4639                 if (ret) {
4640                         dev_info(&pf->pdev->dev,
4641                                  "VSI seid %d Rx ring %d disable timeout\n",
4642                                  vsi->seid, pf_q);
4643                         return ret;
4644                 }
4645         }
4646
4647         return 0;
4648 }
4649
4650 #ifdef CONFIG_I40E_DCB
4651 /**
4652  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
4653  * @pf: the PF
4654  *
4655  * This function waits for the queues to be in disabled state for all the
4656  * VSIs that are managed by this PF.
4657  **/
4658 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4659 {
4660         int v, ret = 0;
4661
4662         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4663                 if (pf->vsi[v]) {
4664                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
4665                         if (ret)
4666                                 break;
4667                 }
4668         }
4669
4670         return ret;
4671 }
4672
4673 #endif
4674
4675 /**
4676  * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue
4677  * @q_idx: TX queue number
4678  * @vsi: Pointer to VSI struct
4679  *
4680  * This function checks specified queue for given VSI. Detects hung condition.
4681  * We proactively detect hung TX queues by checking if interrupts are disabled
4682  * but there are pending descriptors.  If it appears hung, attempt to recover
4683  * by triggering a SW interrupt.
4684  **/
4685 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi)
4686 {
4687         struct i40e_ring *tx_ring = NULL;
4688         struct i40e_pf  *pf;
4689         u32 val, tx_pending;
4690         int i;
4691
4692         pf = vsi->back;
4693
4694         /* now that we have an index, find the tx_ring struct */
4695         for (i = 0; i < vsi->num_queue_pairs; i++) {
4696                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
4697                         if (q_idx == vsi->tx_rings[i]->queue_index) {
4698                                 tx_ring = vsi->tx_rings[i];
4699                                 break;
4700                         }
4701                 }
4702         }
4703
4704         if (!tx_ring)
4705                 return;
4706
4707         /* Read interrupt register */
4708         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4709                 val = rd32(&pf->hw,
4710                            I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
4711                                                tx_ring->vsi->base_vector - 1));
4712         else
4713                 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
4714
4715         tx_pending = i40e_get_tx_pending(tx_ring);
4716
4717         /* Interrupts are disabled and TX pending is non-zero,
4718          * trigger the SW interrupt (don't wait). Worst case
4719          * there will be one extra interrupt which may result
4720          * into not cleaning any queues because queues are cleaned.
4721          */
4722         if (tx_pending && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK)))
4723                 i40e_force_wb(vsi, tx_ring->q_vector);
4724 }
4725
4726 /**
4727  * i40e_detect_recover_hung - Function to detect and recover hung_queues
4728  * @pf:  pointer to PF struct
4729  *
4730  * LAN VSI has netdev and netdev has TX queues. This function is to check
4731  * each of those TX queues if they are hung, trigger recovery by issuing
4732  * SW interrupt.
4733  **/
4734 static void i40e_detect_recover_hung(struct i40e_pf *pf)
4735 {
4736         struct net_device *netdev;
4737         struct i40e_vsi *vsi;
4738         unsigned int i;
4739
4740         /* Only for LAN VSI */
4741         vsi = pf->vsi[pf->lan_vsi];
4742
4743         if (!vsi)
4744                 return;
4745
4746         /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */
4747         if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
4748             test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
4749                 return;
4750
4751         /* Make sure type is MAIN VSI */
4752         if (vsi->type != I40E_VSI_MAIN)
4753                 return;
4754
4755         netdev = vsi->netdev;
4756         if (!netdev)
4757                 return;
4758
4759         /* Bail out if netif_carrier is not OK */
4760         if (!netif_carrier_ok(netdev))
4761                 return;
4762
4763         /* Go thru' TX queues for netdev */
4764         for (i = 0; i < netdev->num_tx_queues; i++) {
4765                 struct netdev_queue *q;
4766
4767                 q = netdev_get_tx_queue(netdev, i);
4768                 if (q)
4769                         i40e_detect_recover_hung_queue(i, vsi);
4770         }
4771 }
4772
4773 /**
4774  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
4775  * @pf: pointer to PF
4776  *
4777  * Get TC map for ISCSI PF type that will include iSCSI TC
4778  * and LAN TC.
4779  **/
4780 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
4781 {
4782         struct i40e_dcb_app_priority_table app;
4783         struct i40e_hw *hw = &pf->hw;
4784         u8 enabled_tc = 1; /* TC0 is always enabled */
4785         u8 tc, i;
4786         /* Get the iSCSI APP TLV */
4787         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4788
4789         for (i = 0; i < dcbcfg->numapps; i++) {
4790                 app = dcbcfg->app[i];
4791                 if (app.selector == I40E_APP_SEL_TCPIP &&
4792                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
4793                         tc = dcbcfg->etscfg.prioritytable[app.priority];
4794                         enabled_tc |= BIT(tc);
4795                         break;
4796                 }
4797         }
4798
4799         return enabled_tc;
4800 }
4801
4802 /**
4803  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
4804  * @dcbcfg: the corresponding DCBx configuration structure
4805  *
4806  * Return the number of TCs from given DCBx configuration
4807  **/
4808 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
4809 {
4810         int i, tc_unused = 0;
4811         u8 num_tc = 0;
4812         u8 ret = 0;
4813
4814         /* Scan the ETS Config Priority Table to find
4815          * traffic class enabled for a given priority
4816          * and create a bitmask of enabled TCs
4817          */
4818         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
4819                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
4820
4821         /* Now scan the bitmask to check for
4822          * contiguous TCs starting with TC0
4823          */
4824         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4825                 if (num_tc & BIT(i)) {
4826                         if (!tc_unused) {
4827                                 ret++;
4828                         } else {
4829                                 pr_err("Non-contiguous TC - Disabling DCB\n");
4830                                 return 1;
4831                         }
4832                 } else {
4833                         tc_unused = 1;
4834                 }
4835         }
4836
4837         /* There is always at least TC0 */
4838         if (!ret)
4839                 ret = 1;
4840
4841         return ret;
4842 }
4843
4844 /**
4845  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
4846  * @dcbcfg: the corresponding DCBx configuration structure
4847  *
4848  * Query the current DCB configuration and return the number of
4849  * traffic classes enabled from the given DCBX config
4850  **/
4851 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
4852 {
4853         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
4854         u8 enabled_tc = 1;
4855         u8 i;
4856
4857         for (i = 0; i < num_tc; i++)
4858                 enabled_tc |= BIT(i);
4859
4860         return enabled_tc;
4861 }
4862
4863 /**
4864  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
4865  * @pf: PF being queried
4866  *
4867  * Return number of traffic classes enabled for the given PF
4868  **/
4869 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
4870 {
4871         struct i40e_hw *hw = &pf->hw;
4872         u8 i, enabled_tc = 1;
4873         u8 num_tc = 0;
4874         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4875
4876         /* If DCB is not enabled then always in single TC */
4877         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4878                 return 1;
4879
4880         /* SFP mode will be enabled for all TCs on port */
4881         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4882                 return i40e_dcb_get_num_tc(dcbcfg);
4883
4884         /* MFP mode return count of enabled TCs for this PF */
4885         if (pf->hw.func_caps.iscsi)
4886                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
4887         else
4888                 return 1; /* Only TC0 */
4889
4890         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4891                 if (enabled_tc & BIT(i))
4892                         num_tc++;
4893         }
4894         return num_tc;
4895 }
4896
4897 /**
4898  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
4899  * @pf: PF being queried
4900  *
4901  * Return a bitmap for enabled traffic classes for this PF.
4902  **/
4903 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
4904 {
4905         /* If DCB is not enabled for this PF then just return default TC */
4906         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4907                 return I40E_DEFAULT_TRAFFIC_CLASS;
4908
4909         /* SFP mode we want PF to be enabled for all TCs */
4910         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4911                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
4912
4913         /* MFP enabled and iSCSI PF type */
4914         if (pf->hw.func_caps.iscsi)
4915                 return i40e_get_iscsi_tc_map(pf);
4916         else
4917                 return I40E_DEFAULT_TRAFFIC_CLASS;
4918 }
4919
4920 /**
4921  * i40e_vsi_get_bw_info - Query VSI BW Information
4922  * @vsi: the VSI being queried
4923  *
4924  * Returns 0 on success, negative value on failure
4925  **/
4926 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
4927 {
4928         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
4929         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
4930         struct i40e_pf *pf = vsi->back;
4931         struct i40e_hw *hw = &pf->hw;
4932         i40e_status ret;
4933         u32 tc_bw_max;
4934         int i;
4935
4936         /* Get the VSI level BW configuration */
4937         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4938         if (ret) {
4939                 dev_info(&pf->pdev->dev,
4940                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
4941                          i40e_stat_str(&pf->hw, ret),
4942                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4943                 return -EINVAL;
4944         }
4945
4946         /* Get the VSI level BW configuration per TC */
4947         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
4948                                                NULL);
4949         if (ret) {
4950                 dev_info(&pf->pdev->dev,
4951                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
4952                          i40e_stat_str(&pf->hw, ret),
4953                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4954                 return -EINVAL;
4955         }
4956
4957         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
4958                 dev_info(&pf->pdev->dev,
4959                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
4960                          bw_config.tc_valid_bits,
4961                          bw_ets_config.tc_valid_bits);
4962                 /* Still continuing */
4963         }
4964
4965         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
4966         vsi->bw_max_quanta = bw_config.max_bw;
4967         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
4968                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
4969         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4970                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
4971                 vsi->bw_ets_limit_credits[i] =
4972                                         le16_to_cpu(bw_ets_config.credits[i]);
4973                 /* 3 bits out of 4 for each TC */
4974                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
4975         }
4976
4977         return 0;
4978 }
4979
4980 /**
4981  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
4982  * @vsi: the VSI being configured
4983  * @enabled_tc: TC bitmap
4984  * @bw_credits: BW shared credits per TC
4985  *
4986  * Returns 0 on success, negative value on failure
4987  **/
4988 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
4989                                        u8 *bw_share)
4990 {
4991         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
4992         i40e_status ret;
4993         int i;
4994
4995         bw_data.tc_valid_bits = enabled_tc;
4996         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4997                 bw_data.tc_bw_credits[i] = bw_share[i];
4998
4999         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
5000                                        NULL);
5001         if (ret) {
5002                 dev_info(&vsi->back->pdev->dev,
5003                          "AQ command Config VSI BW allocation per TC failed = %d\n",
5004                          vsi->back->hw.aq.asq_last_status);
5005                 return -EINVAL;
5006         }
5007
5008         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5009                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5010
5011         return 0;
5012 }
5013
5014 /**
5015  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5016  * @vsi: the VSI being configured
5017  * @enabled_tc: TC map to be enabled
5018  *
5019  **/
5020 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5021 {
5022         struct net_device *netdev = vsi->netdev;
5023         struct i40e_pf *pf = vsi->back;
5024         struct i40e_hw *hw = &pf->hw;
5025         u8 netdev_tc = 0;
5026         int i;
5027         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5028
5029         if (!netdev)
5030                 return;
5031
5032         if (!enabled_tc) {
5033                 netdev_reset_tc(netdev);
5034                 return;
5035         }
5036
5037         /* Set up actual enabled TCs on the VSI */
5038         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5039                 return;
5040
5041         /* set per TC queues for the VSI */
5042         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5043                 /* Only set TC queues for enabled tcs
5044                  *
5045                  * e.g. For a VSI that has TC0 and TC3 enabled the
5046                  * enabled_tc bitmap would be 0x00001001; the driver
5047                  * will set the numtc for netdev as 2 that will be
5048                  * referenced by the netdev layer as TC 0 and 1.
5049                  */
5050                 if (vsi->tc_config.enabled_tc & BIT(i))
5051                         netdev_set_tc_queue(netdev,
5052                                         vsi->tc_config.tc_info[i].netdev_tc,
5053                                         vsi->tc_config.tc_info[i].qcount,
5054                                         vsi->tc_config.tc_info[i].qoffset);
5055         }
5056
5057         /* Assign UP2TC map for the VSI */
5058         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5059                 /* Get the actual TC# for the UP */
5060                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5061                 /* Get the mapped netdev TC# for the UP */
5062                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5063                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5064         }
5065 }
5066
5067 /**
5068  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5069  * @vsi: the VSI being configured
5070  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5071  **/
5072 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5073                                       struct i40e_vsi_context *ctxt)
5074 {
5075         /* copy just the sections touched not the entire info
5076          * since not all sections are valid as returned by
5077          * update vsi params
5078          */
5079         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5080         memcpy(&vsi->info.queue_mapping,
5081                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5082         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5083                sizeof(vsi->info.tc_mapping));
5084 }
5085
5086 /**
5087  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5088  * @vsi: VSI to be configured
5089  * @enabled_tc: TC bitmap
5090  *
5091  * This configures a particular VSI for TCs that are mapped to the
5092  * given TC bitmap. It uses default bandwidth share for TCs across
5093  * VSIs to configure TC for a particular VSI.
5094  *
5095  * NOTE:
5096  * It is expected that the VSI queues have been quisced before calling
5097  * this function.
5098  **/
5099 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5100 {
5101         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5102         struct i40e_vsi_context ctxt;
5103         int ret = 0;
5104         int i;
5105
5106         /* Check if enabled_tc is same as existing or new TCs */
5107         if (vsi->tc_config.enabled_tc == enabled_tc)
5108                 return ret;
5109
5110         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5111         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5112                 if (enabled_tc & BIT(i))
5113                         bw_share[i] = 1;
5114         }
5115
5116         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5117         if (ret) {
5118                 dev_info(&vsi->back->pdev->dev,
5119                          "Failed configuring TC map %d for VSI %d\n",
5120                          enabled_tc, vsi->seid);
5121                 goto out;
5122         }
5123
5124         /* Update Queue Pairs Mapping for currently enabled UPs */
5125         ctxt.seid = vsi->seid;
5126         ctxt.pf_num = vsi->back->hw.pf_id;
5127         ctxt.vf_num = 0;
5128         ctxt.uplink_seid = vsi->uplink_seid;
5129         ctxt.info = vsi->info;
5130         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5131
5132         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5133                 ctxt.info.valid_sections |=
5134                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5135                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5136         }
5137
5138         /* Update the VSI after updating the VSI queue-mapping information */
5139         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
5140         if (ret) {
5141                 dev_info(&vsi->back->pdev->dev,
5142                          "Update vsi tc config failed, err %s aq_err %s\n",
5143                          i40e_stat_str(&vsi->back->hw, ret),
5144                          i40e_aq_str(&vsi->back->hw,
5145                                      vsi->back->hw.aq.asq_last_status));
5146                 goto out;
5147         }
5148         /* update the local VSI info with updated queue map */
5149         i40e_vsi_update_queue_map(vsi, &ctxt);
5150         vsi->info.valid_sections = 0;
5151
5152         /* Update current VSI BW information */
5153         ret = i40e_vsi_get_bw_info(vsi);
5154         if (ret) {
5155                 dev_info(&vsi->back->pdev->dev,
5156                          "Failed updating vsi bw info, err %s aq_err %s\n",
5157                          i40e_stat_str(&vsi->back->hw, ret),
5158                          i40e_aq_str(&vsi->back->hw,
5159                                      vsi->back->hw.aq.asq_last_status));
5160                 goto out;
5161         }
5162
5163         /* Update the netdev TC setup */
5164         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5165 out:
5166         return ret;
5167 }
5168
5169 /**
5170  * i40e_veb_config_tc - Configure TCs for given VEB
5171  * @veb: given VEB
5172  * @enabled_tc: TC bitmap
5173  *
5174  * Configures given TC bitmap for VEB (switching) element
5175  **/
5176 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
5177 {
5178         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
5179         struct i40e_pf *pf = veb->pf;
5180         int ret = 0;
5181         int i;
5182
5183         /* No TCs or already enabled TCs just return */
5184         if (!enabled_tc || veb->enabled_tc == enabled_tc)
5185                 return ret;
5186
5187         bw_data.tc_valid_bits = enabled_tc;
5188         /* bw_data.absolute_credits is not set (relative) */
5189
5190         /* Enable ETS TCs with equal BW Share for now */
5191         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5192                 if (enabled_tc & BIT(i))
5193                         bw_data.tc_bw_share_credits[i] = 1;
5194         }
5195
5196         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
5197                                                    &bw_data, NULL);
5198         if (ret) {
5199                 dev_info(&pf->pdev->dev,
5200                          "VEB bw config failed, err %s aq_err %s\n",
5201                          i40e_stat_str(&pf->hw, ret),
5202                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5203                 goto out;
5204         }
5205
5206         /* Update the BW information */
5207         ret = i40e_veb_get_bw_info(veb);
5208         if (ret) {
5209                 dev_info(&pf->pdev->dev,
5210                          "Failed getting veb bw config, err %s aq_err %s\n",
5211                          i40e_stat_str(&pf->hw, ret),
5212                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5213         }
5214
5215 out:
5216         return ret;
5217 }
5218
5219 #ifdef CONFIG_I40E_DCB
5220 /**
5221  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
5222  * @pf: PF struct
5223  *
5224  * Reconfigure VEB/VSIs on a given PF; it is assumed that
5225  * the caller would've quiesce all the VSIs before calling
5226  * this function
5227  **/
5228 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
5229 {
5230         u8 tc_map = 0;
5231         int ret;
5232         u8 v;
5233
5234         /* Enable the TCs available on PF to all VEBs */
5235         tc_map = i40e_pf_get_tc_map(pf);
5236         for (v = 0; v < I40E_MAX_VEB; v++) {
5237                 if (!pf->veb[v])
5238                         continue;
5239                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
5240                 if (ret) {
5241                         dev_info(&pf->pdev->dev,
5242                                  "Failed configuring TC for VEB seid=%d\n",
5243                                  pf->veb[v]->seid);
5244                         /* Will try to configure as many components */
5245                 }
5246         }
5247
5248         /* Update each VSI */
5249         for (v = 0; v < pf->num_alloc_vsi; v++) {
5250                 if (!pf->vsi[v])
5251                         continue;
5252
5253                 /* - Enable all TCs for the LAN VSI
5254                  * - For all others keep them at TC0 for now
5255                  */
5256                 if (v == pf->lan_vsi)
5257                         tc_map = i40e_pf_get_tc_map(pf);
5258                 else
5259                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
5260
5261                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
5262                 if (ret) {
5263                         dev_info(&pf->pdev->dev,
5264                                  "Failed configuring TC for VSI seid=%d\n",
5265                                  pf->vsi[v]->seid);
5266                         /* Will try to configure as many components */
5267                 } else {
5268                         /* Re-configure VSI vectors based on updated TC map */
5269                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
5270                         if (pf->vsi[v]->netdev)
5271                                 i40e_dcbnl_set_all(pf->vsi[v]);
5272                 }
5273         }
5274 }
5275
5276 /**
5277  * i40e_resume_port_tx - Resume port Tx
5278  * @pf: PF struct
5279  *
5280  * Resume a port's Tx and issue a PF reset in case of failure to
5281  * resume.
5282  **/
5283 static int i40e_resume_port_tx(struct i40e_pf *pf)
5284 {
5285         struct i40e_hw *hw = &pf->hw;
5286         int ret;
5287
5288         ret = i40e_aq_resume_port_tx(hw, NULL);
5289         if (ret) {
5290                 dev_info(&pf->pdev->dev,
5291                          "Resume Port Tx failed, err %s aq_err %s\n",
5292                           i40e_stat_str(&pf->hw, ret),
5293                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5294                 /* Schedule PF reset to recover */
5295                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
5296                 i40e_service_event_schedule(pf);
5297         }
5298
5299         return ret;
5300 }
5301
5302 /**
5303  * i40e_init_pf_dcb - Initialize DCB configuration
5304  * @pf: PF being configured
5305  *
5306  * Query the current DCB configuration and cache it
5307  * in the hardware structure
5308  **/
5309 static int i40e_init_pf_dcb(struct i40e_pf *pf)
5310 {
5311         struct i40e_hw *hw = &pf->hw;
5312         int err = 0;
5313
5314         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */
5315         if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT)
5316                 goto out;
5317
5318         /* Get the initial DCB configuration */
5319         err = i40e_init_dcb(hw);
5320         if (!err) {
5321                 /* Device/Function is not DCBX capable */
5322                 if ((!hw->func_caps.dcb) ||
5323                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
5324                         dev_info(&pf->pdev->dev,
5325                                  "DCBX offload is not supported or is disabled for this PF.\n");
5326                 } else {
5327                         /* When status is not DISABLED then DCBX in FW */
5328                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
5329                                        DCB_CAP_DCBX_VER_IEEE;
5330
5331                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
5332                         /* Enable DCB tagging only when more than one TC
5333                          * or explicitly disable if only one TC
5334                          */
5335                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
5336                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
5337                         else
5338                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
5339                         dev_dbg(&pf->pdev->dev,
5340                                 "DCBX offload is supported for this PF.\n");
5341                 }
5342         } else {
5343                 dev_info(&pf->pdev->dev,
5344                          "Query for DCB configuration failed, err %s aq_err %s\n",
5345                          i40e_stat_str(&pf->hw, err),
5346                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5347         }
5348
5349 out:
5350         return err;
5351 }
5352 #endif /* CONFIG_I40E_DCB */
5353 #define SPEED_SIZE 14
5354 #define FC_SIZE 8
5355 /**
5356  * i40e_print_link_message - print link up or down
5357  * @vsi: the VSI for which link needs a message
5358  */
5359 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
5360 {
5361         enum i40e_aq_link_speed new_speed;
5362         struct i40e_pf *pf = vsi->back;
5363         char *speed = "Unknown";
5364         char *fc = "Unknown";
5365         char *fec = "";
5366         char *req_fec = "";
5367         char *an = "";
5368
5369         new_speed = pf->hw.phy.link_info.link_speed;
5370
5371         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
5372                 return;
5373         vsi->current_isup = isup;
5374         vsi->current_speed = new_speed;
5375         if (!isup) {
5376                 netdev_info(vsi->netdev, "NIC Link is Down\n");
5377                 return;
5378         }
5379
5380         /* Warn user if link speed on NPAR enabled partition is not at
5381          * least 10GB
5382          */
5383         if (pf->hw.func_caps.npar_enable &&
5384             (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
5385              pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
5386                 netdev_warn(vsi->netdev,
5387                             "The partition detected link speed that is less than 10Gbps\n");
5388
5389         switch (pf->hw.phy.link_info.link_speed) {
5390         case I40E_LINK_SPEED_40GB:
5391                 speed = "40 G";
5392                 break;
5393         case I40E_LINK_SPEED_20GB:
5394                 speed = "20 G";
5395                 break;
5396         case I40E_LINK_SPEED_25GB:
5397                 speed = "25 G";
5398                 break;
5399         case I40E_LINK_SPEED_10GB:
5400                 speed = "10 G";
5401                 break;
5402         case I40E_LINK_SPEED_1GB:
5403                 speed = "1000 M";
5404                 break;
5405         case I40E_LINK_SPEED_100MB:
5406                 speed = "100 M";
5407                 break;
5408         default:
5409                 break;
5410         }
5411
5412         switch (pf->hw.fc.current_mode) {
5413         case I40E_FC_FULL:
5414                 fc = "RX/TX";
5415                 break;
5416         case I40E_FC_TX_PAUSE:
5417                 fc = "TX";
5418                 break;
5419         case I40E_FC_RX_PAUSE:
5420                 fc = "RX";
5421                 break;
5422         default:
5423                 fc = "None";
5424                 break;
5425         }
5426
5427         if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
5428                 req_fec = ", Requested FEC: None";
5429                 fec = ", FEC: None";
5430                 an = ", Autoneg: False";
5431
5432                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
5433                         an = ", Autoneg: True";
5434
5435                 if (pf->hw.phy.link_info.fec_info &
5436                     I40E_AQ_CONFIG_FEC_KR_ENA)
5437                         fec = ", FEC: CL74 FC-FEC/BASE-R";
5438                 else if (pf->hw.phy.link_info.fec_info &
5439                          I40E_AQ_CONFIG_FEC_RS_ENA)
5440                         fec = ", FEC: CL108 RS-FEC";
5441
5442                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
5443                  * both RS and FC are requested
5444                  */
5445                 if (vsi->back->hw.phy.link_info.req_fec_info &
5446                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
5447                         if (vsi->back->hw.phy.link_info.req_fec_info &
5448                             I40E_AQ_REQUEST_FEC_RS)
5449                                 req_fec = ", Requested FEC: CL108 RS-FEC";
5450                         else
5451                                 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R";
5452                 }
5453         }
5454
5455         netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n",
5456                     speed, req_fec, fec, an, fc);
5457 }
5458
5459 /**
5460  * i40e_up_complete - Finish the last steps of bringing up a connection
5461  * @vsi: the VSI being configured
5462  **/
5463 static int i40e_up_complete(struct i40e_vsi *vsi)
5464 {
5465         struct i40e_pf *pf = vsi->back;
5466         int err;
5467
5468         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5469                 i40e_vsi_configure_msix(vsi);
5470         else
5471                 i40e_configure_msi_and_legacy(vsi);
5472
5473         /* start rings */
5474         err = i40e_vsi_start_rings(vsi);
5475         if (err)
5476                 return err;
5477
5478         clear_bit(__I40E_VSI_DOWN, vsi->state);
5479         i40e_napi_enable_all(vsi);
5480         i40e_vsi_enable_irq(vsi);
5481
5482         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
5483             (vsi->netdev)) {
5484                 i40e_print_link_message(vsi, true);
5485                 netif_tx_start_all_queues(vsi->netdev);
5486                 netif_carrier_on(vsi->netdev);
5487         }
5488
5489         /* replay FDIR SB filters */
5490         if (vsi->type == I40E_VSI_FDIR) {
5491                 /* reset fd counters */
5492                 pf->fd_add_err = 0;
5493                 pf->fd_atr_cnt = 0;
5494                 i40e_fdir_filter_restore(vsi);
5495         }
5496
5497         /* On the next run of the service_task, notify any clients of the new
5498          * opened netdev
5499          */
5500         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
5501         i40e_service_event_schedule(pf);
5502
5503         return 0;
5504 }
5505
5506 /**
5507  * i40e_vsi_reinit_locked - Reset the VSI
5508  * @vsi: the VSI being configured
5509  *
5510  * Rebuild the ring structs after some configuration
5511  * has changed, e.g. MTU size.
5512  **/
5513 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
5514 {
5515         struct i40e_pf *pf = vsi->back;
5516
5517         WARN_ON(in_interrupt());
5518         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
5519                 usleep_range(1000, 2000);
5520         i40e_down(vsi);
5521
5522         i40e_up(vsi);
5523         clear_bit(__I40E_CONFIG_BUSY, pf->state);
5524 }
5525
5526 /**
5527  * i40e_up - Bring the connection back up after being down
5528  * @vsi: the VSI being configured
5529  **/
5530 int i40e_up(struct i40e_vsi *vsi)
5531 {
5532         int err;
5533
5534         err = i40e_vsi_configure(vsi);
5535         if (!err)
5536                 err = i40e_up_complete(vsi);
5537
5538         return err;
5539 }
5540
5541 /**
5542  * i40e_down - Shutdown the connection processing
5543  * @vsi: the VSI being stopped
5544  **/
5545 void i40e_down(struct i40e_vsi *vsi)
5546 {
5547         int i;
5548
5549         /* It is assumed that the caller of this function
5550          * sets the vsi->state __I40E_VSI_DOWN bit.
5551          */
5552         if (vsi->netdev) {
5553                 netif_carrier_off(vsi->netdev);
5554                 netif_tx_disable(vsi->netdev);
5555         }
5556         i40e_vsi_disable_irq(vsi);
5557         i40e_vsi_stop_rings(vsi);
5558         i40e_napi_disable_all(vsi);
5559
5560         for (i = 0; i < vsi->num_queue_pairs; i++) {
5561                 i40e_clean_tx_ring(vsi->tx_rings[i]);
5562                 if (i40e_enabled_xdp_vsi(vsi))
5563                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
5564                 i40e_clean_rx_ring(vsi->rx_rings[i]);
5565         }
5566
5567 }
5568
5569 /**
5570  * i40e_setup_tc - configure multiple traffic classes
5571  * @netdev: net device to configure
5572  * @tc: number of traffic classes to enable
5573  **/
5574 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
5575 {
5576         struct i40e_netdev_priv *np = netdev_priv(netdev);
5577         struct i40e_vsi *vsi = np->vsi;
5578         struct i40e_pf *pf = vsi->back;
5579         u8 enabled_tc = 0;
5580         int ret = -EINVAL;
5581         int i;
5582
5583         /* Check if DCB enabled to continue */
5584         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5585                 netdev_info(netdev, "DCB is not enabled for adapter\n");
5586                 goto exit;
5587         }
5588
5589         /* Check if MFP enabled */
5590         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
5591                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
5592                 goto exit;
5593         }
5594
5595         /* Check whether tc count is within enabled limit */
5596         if (tc > i40e_pf_get_num_tc(pf)) {
5597                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
5598                 goto exit;
5599         }
5600
5601         /* Generate TC map for number of tc requested */
5602         for (i = 0; i < tc; i++)
5603                 enabled_tc |= BIT(i);
5604
5605         /* Requesting same TC configuration as already enabled */
5606         if (enabled_tc == vsi->tc_config.enabled_tc)
5607                 return 0;
5608
5609         /* Quiesce VSI queues */
5610         i40e_quiesce_vsi(vsi);
5611
5612         /* Configure VSI for enabled TCs */
5613         ret = i40e_vsi_config_tc(vsi, enabled_tc);
5614         if (ret) {
5615                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
5616                             vsi->seid);
5617                 goto exit;
5618         }
5619
5620         /* Unquiesce VSI */
5621         i40e_unquiesce_vsi(vsi);
5622
5623 exit:
5624         return ret;
5625 }
5626
5627 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
5628                            void *type_data)
5629 {
5630         struct tc_mqprio_qopt *mqprio = type_data;
5631
5632         if (type != TC_SETUP_MQPRIO)
5633                 return -EOPNOTSUPP;
5634
5635         mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
5636
5637         return i40e_setup_tc(netdev, mqprio->num_tc);
5638 }
5639
5640 /**
5641  * i40e_open - Called when a network interface is made active
5642  * @netdev: network interface device structure
5643  *
5644  * The open entry point is called when a network interface is made
5645  * active by the system (IFF_UP).  At this point all resources needed
5646  * for transmit and receive operations are allocated, the interrupt
5647  * handler is registered with the OS, the netdev watchdog subtask is
5648  * enabled, and the stack is notified that the interface is ready.
5649  *
5650  * Returns 0 on success, negative value on failure
5651  **/
5652 int i40e_open(struct net_device *netdev)
5653 {
5654         struct i40e_netdev_priv *np = netdev_priv(netdev);
5655         struct i40e_vsi *vsi = np->vsi;
5656         struct i40e_pf *pf = vsi->back;
5657         int err;
5658
5659         /* disallow open during test or if eeprom is broken */
5660         if (test_bit(__I40E_TESTING, pf->state) ||
5661             test_bit(__I40E_BAD_EEPROM, pf->state))
5662                 return -EBUSY;
5663
5664         netif_carrier_off(netdev);
5665
5666         err = i40e_vsi_open(vsi);
5667         if (err)
5668                 return err;
5669
5670         /* configure global TSO hardware offload settings */
5671         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
5672                                                        TCP_FLAG_FIN) >> 16);
5673         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
5674                                                        TCP_FLAG_FIN |
5675                                                        TCP_FLAG_CWR) >> 16);
5676         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
5677
5678         udp_tunnel_get_rx_info(netdev);
5679
5680         return 0;
5681 }
5682
5683 /**
5684  * i40e_vsi_open -
5685  * @vsi: the VSI to open
5686  *
5687  * Finish initialization of the VSI.
5688  *
5689  * Returns 0 on success, negative value on failure
5690  *
5691  * Note: expects to be called while under rtnl_lock()
5692  **/
5693 int i40e_vsi_open(struct i40e_vsi *vsi)
5694 {
5695         struct i40e_pf *pf = vsi->back;
5696         char int_name[I40E_INT_NAME_STR_LEN];
5697         int err;
5698
5699         /* allocate descriptors */
5700         err = i40e_vsi_setup_tx_resources(vsi);
5701         if (err)
5702                 goto err_setup_tx;
5703         err = i40e_vsi_setup_rx_resources(vsi);
5704         if (err)
5705                 goto err_setup_rx;
5706
5707         err = i40e_vsi_configure(vsi);
5708         if (err)
5709                 goto err_setup_rx;
5710
5711         if (vsi->netdev) {
5712                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
5713                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
5714                 err = i40e_vsi_request_irq(vsi, int_name);
5715                 if (err)
5716                         goto err_setup_rx;
5717
5718                 /* Notify the stack of the actual queue counts. */
5719                 err = netif_set_real_num_tx_queues(vsi->netdev,
5720                                                    vsi->num_queue_pairs);
5721                 if (err)
5722                         goto err_set_queues;
5723
5724                 err = netif_set_real_num_rx_queues(vsi->netdev,
5725                                                    vsi->num_queue_pairs);
5726                 if (err)
5727                         goto err_set_queues;
5728
5729         } else if (vsi->type == I40E_VSI_FDIR) {
5730                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
5731                          dev_driver_string(&pf->pdev->dev),
5732                          dev_name(&pf->pdev->dev));
5733                 err = i40e_vsi_request_irq(vsi, int_name);
5734
5735         } else {
5736                 err = -EINVAL;
5737                 goto err_setup_rx;
5738         }
5739
5740         err = i40e_up_complete(vsi);
5741         if (err)
5742                 goto err_up_complete;
5743
5744         return 0;
5745
5746 err_up_complete:
5747         i40e_down(vsi);
5748 err_set_queues:
5749         i40e_vsi_free_irq(vsi);
5750 err_setup_rx:
5751         i40e_vsi_free_rx_resources(vsi);
5752 err_setup_tx:
5753         i40e_vsi_free_tx_resources(vsi);
5754         if (vsi == pf->vsi[pf->lan_vsi])
5755                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true);
5756
5757         return err;
5758 }
5759
5760 /**
5761  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
5762  * @pf: Pointer to PF
5763  *
5764  * This function destroys the hlist where all the Flow Director
5765  * filters were saved.
5766  **/
5767 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
5768 {
5769         struct i40e_fdir_filter *filter;
5770         struct i40e_flex_pit *pit_entry, *tmp;
5771         struct hlist_node *node2;
5772
5773         hlist_for_each_entry_safe(filter, node2,
5774                                   &pf->fdir_filter_list, fdir_node) {
5775                 hlist_del(&filter->fdir_node);
5776                 kfree(filter);
5777         }
5778
5779         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
5780                 list_del(&pit_entry->list);
5781                 kfree(pit_entry);
5782         }
5783         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
5784
5785         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
5786                 list_del(&pit_entry->list);
5787                 kfree(pit_entry);
5788         }
5789         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
5790
5791         pf->fdir_pf_active_filters = 0;
5792         pf->fd_tcp4_filter_cnt = 0;
5793         pf->fd_udp4_filter_cnt = 0;
5794         pf->fd_sctp4_filter_cnt = 0;
5795         pf->fd_ip4_filter_cnt = 0;
5796
5797         /* Reprogram the default input set for TCP/IPv4 */
5798         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
5799                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5800                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5801
5802         /* Reprogram the default input set for UDP/IPv4 */
5803         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
5804                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5805                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5806
5807         /* Reprogram the default input set for SCTP/IPv4 */
5808         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
5809                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5810                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5811
5812         /* Reprogram the default input set for Other/IPv4 */
5813         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
5814                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
5815 }
5816
5817 /**
5818  * i40e_close - Disables a network interface
5819  * @netdev: network interface device structure
5820  *
5821  * The close entry point is called when an interface is de-activated
5822  * by the OS.  The hardware is still under the driver's control, but
5823  * this netdev interface is disabled.
5824  *
5825  * Returns 0, this is not allowed to fail
5826  **/
5827 int i40e_close(struct net_device *netdev)
5828 {
5829         struct i40e_netdev_priv *np = netdev_priv(netdev);
5830         struct i40e_vsi *vsi = np->vsi;
5831
5832         i40e_vsi_close(vsi);
5833
5834         return 0;
5835 }
5836
5837 /**
5838  * i40e_do_reset - Start a PF or Core Reset sequence
5839  * @pf: board private structure
5840  * @reset_flags: which reset is requested
5841  * @lock_acquired: indicates whether or not the lock has been acquired
5842  * before this function was called.
5843  *
5844  * The essential difference in resets is that the PF Reset
5845  * doesn't clear the packet buffers, doesn't reset the PE
5846  * firmware, and doesn't bother the other PFs on the chip.
5847  **/
5848 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
5849 {
5850         u32 val;
5851
5852         WARN_ON(in_interrupt());
5853
5854
5855         /* do the biggest reset indicated */
5856         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
5857
5858                 /* Request a Global Reset
5859                  *
5860                  * This will start the chip's countdown to the actual full
5861                  * chip reset event, and a warning interrupt to be sent
5862                  * to all PFs, including the requestor.  Our handler
5863                  * for the warning interrupt will deal with the shutdown
5864                  * and recovery of the switch setup.
5865                  */
5866                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
5867                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5868                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
5869                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5870
5871         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
5872
5873                 /* Request a Core Reset
5874                  *
5875                  * Same as Global Reset, except does *not* include the MAC/PHY
5876                  */
5877                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
5878                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5879                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
5880                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5881                 i40e_flush(&pf->hw);
5882
5883         } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) {
5884
5885                 /* Request a PF Reset
5886                  *
5887                  * Resets only the PF-specific registers
5888                  *
5889                  * This goes directly to the tear-down and rebuild of
5890                  * the switch, since we need to do all the recovery as
5891                  * for the Core Reset.
5892                  */
5893                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
5894                 i40e_handle_reset_warning(pf, lock_acquired);
5895
5896         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
5897                 int v;
5898
5899                 /* Find the VSI(s) that requested a re-init */
5900                 dev_info(&pf->pdev->dev,
5901                          "VSI reinit requested\n");
5902                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5903                         struct i40e_vsi *vsi = pf->vsi[v];
5904
5905                         if (vsi != NULL &&
5906                             test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
5907                                                vsi->state))
5908                                 i40e_vsi_reinit_locked(pf->vsi[v]);
5909                 }
5910         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
5911                 int v;
5912
5913                 /* Find the VSI(s) that needs to be brought down */
5914                 dev_info(&pf->pdev->dev, "VSI down requested\n");
5915                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5916                         struct i40e_vsi *vsi = pf->vsi[v];
5917
5918                         if (vsi != NULL &&
5919                             test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
5920                                                vsi->state)) {
5921                                 set_bit(__I40E_VSI_DOWN, vsi->state);
5922                                 i40e_down(vsi);
5923                         }
5924                 }
5925         } else {
5926                 dev_info(&pf->pdev->dev,
5927                          "bad reset request 0x%08x\n", reset_flags);
5928         }
5929 }
5930
5931 #ifdef CONFIG_I40E_DCB
5932 /**
5933  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
5934  * @pf: board private structure
5935  * @old_cfg: current DCB config
5936  * @new_cfg: new DCB config
5937  **/
5938 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
5939                             struct i40e_dcbx_config *old_cfg,
5940                             struct i40e_dcbx_config *new_cfg)
5941 {
5942         bool need_reconfig = false;
5943
5944         /* Check if ETS configuration has changed */
5945         if (memcmp(&new_cfg->etscfg,
5946                    &old_cfg->etscfg,
5947                    sizeof(new_cfg->etscfg))) {
5948                 /* If Priority Table has changed reconfig is needed */
5949                 if (memcmp(&new_cfg->etscfg.prioritytable,
5950                            &old_cfg->etscfg.prioritytable,
5951                            sizeof(new_cfg->etscfg.prioritytable))) {
5952                         need_reconfig = true;
5953                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
5954                 }
5955
5956                 if (memcmp(&new_cfg->etscfg.tcbwtable,
5957                            &old_cfg->etscfg.tcbwtable,
5958                            sizeof(new_cfg->etscfg.tcbwtable)))
5959                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
5960
5961                 if (memcmp(&new_cfg->etscfg.tsatable,
5962                            &old_cfg->etscfg.tsatable,
5963                            sizeof(new_cfg->etscfg.tsatable)))
5964                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
5965         }
5966
5967         /* Check if PFC configuration has changed */
5968         if (memcmp(&new_cfg->pfc,
5969                    &old_cfg->pfc,
5970                    sizeof(new_cfg->pfc))) {
5971                 need_reconfig = true;
5972                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
5973         }
5974
5975         /* Check if APP Table has changed */
5976         if (memcmp(&new_cfg->app,
5977                    &old_cfg->app,
5978                    sizeof(new_cfg->app))) {
5979                 need_reconfig = true;
5980                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
5981         }
5982
5983         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
5984         return need_reconfig;
5985 }
5986
5987 /**
5988  * i40e_handle_lldp_event - Handle LLDP Change MIB event
5989  * @pf: board private structure
5990  * @e: event info posted on ARQ
5991  **/
5992 static int i40e_handle_lldp_event(struct i40e_pf *pf,
5993                                   struct i40e_arq_event_info *e)
5994 {
5995         struct i40e_aqc_lldp_get_mib *mib =
5996                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
5997         struct i40e_hw *hw = &pf->hw;
5998         struct i40e_dcbx_config tmp_dcbx_cfg;
5999         bool need_reconfig = false;
6000         int ret = 0;
6001         u8 type;
6002
6003         /* Not DCB capable or capability disabled */
6004         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
6005                 return ret;
6006
6007         /* Ignore if event is not for Nearest Bridge */
6008         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
6009                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
6010         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
6011         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
6012                 return ret;
6013
6014         /* Check MIB Type and return if event for Remote MIB update */
6015         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
6016         dev_dbg(&pf->pdev->dev,
6017                 "LLDP event mib type %s\n", type ? "remote" : "local");
6018         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
6019                 /* Update the remote cached instance and return */
6020                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
6021                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
6022                                 &hw->remote_dcbx_config);
6023                 goto exit;
6024         }
6025
6026         /* Store the old configuration */
6027         tmp_dcbx_cfg = hw->local_dcbx_config;
6028
6029         /* Reset the old DCBx configuration data */
6030         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
6031         /* Get updated DCBX data from firmware */
6032         ret = i40e_get_dcb_config(&pf->hw);
6033         if (ret) {
6034                 dev_info(&pf->pdev->dev,
6035                          "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
6036                          i40e_stat_str(&pf->hw, ret),
6037                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6038                 goto exit;
6039         }
6040
6041         /* No change detected in DCBX configs */
6042         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
6043                     sizeof(tmp_dcbx_cfg))) {
6044                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
6045                 goto exit;
6046         }
6047
6048         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
6049                                                &hw->local_dcbx_config);
6050
6051         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
6052
6053         if (!need_reconfig)
6054                 goto exit;
6055
6056         /* Enable DCB tagging only when more than one TC */
6057         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6058                 pf->flags |= I40E_FLAG_DCB_ENABLED;
6059         else
6060                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6061
6062         set_bit(__I40E_PORT_SUSPENDED, pf->state);
6063         /* Reconfiguration needed quiesce all VSIs */
6064         i40e_pf_quiesce_all_vsi(pf);
6065
6066         /* Changes in configuration update VEB/VSI */
6067         i40e_dcb_reconfigure(pf);
6068
6069         ret = i40e_resume_port_tx(pf);
6070
6071         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
6072         /* In case of error no point in resuming VSIs */
6073         if (ret)
6074                 goto exit;
6075
6076         /* Wait for the PF's queues to be disabled */
6077         ret = i40e_pf_wait_queues_disabled(pf);
6078         if (ret) {
6079                 /* Schedule PF reset to recover */
6080                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6081                 i40e_service_event_schedule(pf);
6082         } else {
6083                 i40e_pf_unquiesce_all_vsi(pf);
6084         pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
6085                       I40E_FLAG_CLIENT_L2_CHANGE);
6086         }
6087
6088 exit:
6089         return ret;
6090 }
6091 #endif /* CONFIG_I40E_DCB */
6092
6093 /**
6094  * i40e_do_reset_safe - Protected reset path for userland calls.
6095  * @pf: board private structure
6096  * @reset_flags: which reset is requested
6097  *
6098  **/
6099 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
6100 {
6101         rtnl_lock();
6102         i40e_do_reset(pf, reset_flags, true);
6103         rtnl_unlock();
6104 }
6105
6106 /**
6107  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
6108  * @pf: board private structure
6109  * @e: event info posted on ARQ
6110  *
6111  * Handler for LAN Queue Overflow Event generated by the firmware for PF
6112  * and VF queues
6113  **/
6114 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
6115                                            struct i40e_arq_event_info *e)
6116 {
6117         struct i40e_aqc_lan_overflow *data =
6118                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
6119         u32 queue = le32_to_cpu(data->prtdcb_rupto);
6120         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
6121         struct i40e_hw *hw = &pf->hw;
6122         struct i40e_vf *vf;
6123         u16 vf_id;
6124
6125         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
6126                 queue, qtx_ctl);
6127
6128         /* Queue belongs to VF, find the VF and issue VF reset */
6129         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
6130             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
6131                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
6132                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
6133                 vf_id -= hw->func_caps.vf_base_id;
6134                 vf = &pf->vf[vf_id];
6135                 i40e_vc_notify_vf_reset(vf);
6136                 /* Allow VF to process pending reset notification */
6137                 msleep(20);
6138                 i40e_reset_vf(vf, false);
6139         }
6140 }
6141
6142 /**
6143  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
6144  * @pf: board private structure
6145  **/
6146 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
6147 {
6148         u32 val, fcnt_prog;
6149
6150         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
6151         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
6152         return fcnt_prog;
6153 }
6154
6155 /**
6156  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
6157  * @pf: board private structure
6158  **/
6159 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
6160 {
6161         u32 val, fcnt_prog;
6162
6163         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
6164         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
6165                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
6166                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
6167         return fcnt_prog;
6168 }
6169
6170 /**
6171  * i40e_get_global_fd_count - Get total FD filters programmed on device
6172  * @pf: board private structure
6173  **/
6174 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
6175 {
6176         u32 val, fcnt_prog;
6177
6178         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
6179         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
6180                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
6181                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
6182         return fcnt_prog;
6183 }
6184
6185 /**
6186  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
6187  * @pf: board private structure
6188  **/
6189 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
6190 {
6191         struct i40e_fdir_filter *filter;
6192         u32 fcnt_prog, fcnt_avail;
6193         struct hlist_node *node;
6194
6195         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
6196                 return;
6197
6198         /* Check if we have enough room to re-enable FDir SB capability. */
6199         fcnt_prog = i40e_get_global_fd_count(pf);
6200         fcnt_avail = pf->fdir_pf_filter_count;
6201         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
6202             (pf->fd_add_err == 0) ||
6203             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) {
6204                 if (pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) {
6205                         pf->flags &= ~I40E_FLAG_FD_SB_AUTO_DISABLED;
6206                         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
6207                             (I40E_DEBUG_FD & pf->hw.debug_mask))
6208                                 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
6209                 }
6210         }
6211
6212         /* We should wait for even more space before re-enabling ATR.
6213          * Additionally, we cannot enable ATR as long as we still have TCP SB
6214          * rules active.
6215          */
6216         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
6217             (pf->fd_tcp4_filter_cnt == 0)) {
6218                 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) {
6219                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
6220                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
6221                             (I40E_DEBUG_FD & pf->hw.debug_mask))
6222                                 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
6223                 }
6224         }
6225
6226         /* if hw had a problem adding a filter, delete it */
6227         if (pf->fd_inv > 0) {
6228                 hlist_for_each_entry_safe(filter, node,
6229                                           &pf->fdir_filter_list, fdir_node) {
6230                         if (filter->fd_id == pf->fd_inv) {
6231                                 hlist_del(&filter->fdir_node);
6232                                 kfree(filter);
6233                                 pf->fdir_pf_active_filters--;
6234                         }
6235                 }
6236         }
6237 }
6238
6239 #define I40E_MIN_FD_FLUSH_INTERVAL 10
6240 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
6241 /**
6242  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
6243  * @pf: board private structure
6244  **/
6245 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
6246 {
6247         unsigned long min_flush_time;
6248         int flush_wait_retry = 50;
6249         bool disable_atr = false;
6250         int fd_room;
6251         int reg;
6252
6253         if (!time_after(jiffies, pf->fd_flush_timestamp +
6254                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
6255                 return;
6256
6257         /* If the flush is happening too quick and we have mostly SB rules we
6258          * should not re-enable ATR for some time.
6259          */
6260         min_flush_time = pf->fd_flush_timestamp +
6261                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
6262         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
6263
6264         if (!(time_after(jiffies, min_flush_time)) &&
6265             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
6266                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6267                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
6268                 disable_atr = true;
6269         }
6270
6271         pf->fd_flush_timestamp = jiffies;
6272         pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED;
6273         /* flush all filters */
6274         wr32(&pf->hw, I40E_PFQF_CTL_1,
6275              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
6276         i40e_flush(&pf->hw);
6277         pf->fd_flush_cnt++;
6278         pf->fd_add_err = 0;
6279         do {
6280                 /* Check FD flush status every 5-6msec */
6281                 usleep_range(5000, 6000);
6282                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
6283                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
6284                         break;
6285         } while (flush_wait_retry--);
6286         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
6287                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
6288         } else {
6289                 /* replay sideband filters */
6290                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
6291                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
6292                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
6293                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
6294                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6295                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
6296         }
6297 }
6298
6299 /**
6300  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
6301  * @pf: board private structure
6302  **/
6303 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
6304 {
6305         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
6306 }
6307
6308 /* We can see up to 256 filter programming desc in transit if the filters are
6309  * being applied really fast; before we see the first
6310  * filter miss error on Rx queue 0. Accumulating enough error messages before
6311  * reacting will make sure we don't cause flush too often.
6312  */
6313 #define I40E_MAX_FD_PROGRAM_ERROR 256
6314
6315 /**
6316  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
6317  * @pf: board private structure
6318  **/
6319 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
6320 {
6321
6322         /* if interface is down do nothing */
6323         if (test_bit(__I40E_DOWN, pf->state))
6324                 return;
6325
6326         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
6327                 i40e_fdir_flush_and_replay(pf);
6328
6329         i40e_fdir_check_and_reenable(pf);
6330
6331 }
6332
6333 /**
6334  * i40e_vsi_link_event - notify VSI of a link event
6335  * @vsi: vsi to be notified
6336  * @link_up: link up or down
6337  **/
6338 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
6339 {
6340         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
6341                 return;
6342
6343         switch (vsi->type) {
6344         case I40E_VSI_MAIN:
6345                 if (!vsi->netdev || !vsi->netdev_registered)
6346                         break;
6347
6348                 if (link_up) {
6349                         netif_carrier_on(vsi->netdev);
6350                         netif_tx_wake_all_queues(vsi->netdev);
6351                 } else {
6352                         netif_carrier_off(vsi->netdev);
6353                         netif_tx_stop_all_queues(vsi->netdev);
6354                 }
6355                 break;
6356
6357         case I40E_VSI_SRIOV:
6358         case I40E_VSI_VMDQ2:
6359         case I40E_VSI_CTRL:
6360         case I40E_VSI_IWARP:
6361         case I40E_VSI_MIRROR:
6362         default:
6363                 /* there is no notification for other VSIs */
6364                 break;
6365         }
6366 }
6367
6368 /**
6369  * i40e_veb_link_event - notify elements on the veb of a link event
6370  * @veb: veb to be notified
6371  * @link_up: link up or down
6372  **/
6373 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
6374 {
6375         struct i40e_pf *pf;
6376         int i;
6377
6378         if (!veb || !veb->pf)
6379                 return;
6380         pf = veb->pf;
6381
6382         /* depth first... */
6383         for (i = 0; i < I40E_MAX_VEB; i++)
6384                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
6385                         i40e_veb_link_event(pf->veb[i], link_up);
6386
6387         /* ... now the local VSIs */
6388         for (i = 0; i < pf->num_alloc_vsi; i++)
6389                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
6390                         i40e_vsi_link_event(pf->vsi[i], link_up);
6391 }
6392
6393 /**
6394  * i40e_link_event - Update netif_carrier status
6395  * @pf: board private structure
6396  **/
6397 static void i40e_link_event(struct i40e_pf *pf)
6398 {
6399         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6400         u8 new_link_speed, old_link_speed;
6401         i40e_status status;
6402         bool new_link, old_link;
6403
6404         /* save off old link status information */
6405         pf->hw.phy.link_info_old = pf->hw.phy.link_info;
6406
6407         /* set this to force the get_link_status call to refresh state */
6408         pf->hw.phy.get_link_info = true;
6409
6410         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
6411
6412         status = i40e_get_link_status(&pf->hw, &new_link);
6413
6414         /* On success, disable temp link polling */
6415         if (status == I40E_SUCCESS) {
6416                 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)
6417                         pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING;
6418         } else {
6419                 /* Enable link polling temporarily until i40e_get_link_status
6420                  * returns I40E_SUCCESS
6421                  */
6422                 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING;
6423                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
6424                         status);
6425                 return;
6426         }
6427
6428         old_link_speed = pf->hw.phy.link_info_old.link_speed;
6429         new_link_speed = pf->hw.phy.link_info.link_speed;
6430
6431         if (new_link == old_link &&
6432             new_link_speed == old_link_speed &&
6433             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
6434              new_link == netif_carrier_ok(vsi->netdev)))
6435                 return;
6436
6437         i40e_print_link_message(vsi, new_link);
6438
6439         /* Notify the base of the switch tree connected to
6440          * the link.  Floating VEBs are not notified.
6441          */
6442         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
6443                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
6444         else
6445                 i40e_vsi_link_event(vsi, new_link);
6446
6447         if (pf->vf)
6448                 i40e_vc_notify_link_state(pf);
6449
6450         if (pf->flags & I40E_FLAG_PTP)
6451                 i40e_ptp_set_increment(pf);
6452 }
6453
6454 /**
6455  * i40e_watchdog_subtask - periodic checks not using event driven response
6456  * @pf: board private structure
6457  **/
6458 static void i40e_watchdog_subtask(struct i40e_pf *pf)
6459 {
6460         int i;
6461
6462         /* if interface is down do nothing */
6463         if (test_bit(__I40E_DOWN, pf->state) ||
6464             test_bit(__I40E_CONFIG_BUSY, pf->state))
6465                 return;
6466
6467         /* make sure we don't do these things too often */
6468         if (time_before(jiffies, (pf->service_timer_previous +
6469                                   pf->service_timer_period)))
6470                 return;
6471         pf->service_timer_previous = jiffies;
6472
6473         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
6474             (pf->flags & I40E_FLAG_TEMP_LINK_POLLING))
6475                 i40e_link_event(pf);
6476
6477         /* Update the stats for active netdevs so the network stack
6478          * can look at updated numbers whenever it cares to
6479          */
6480         for (i = 0; i < pf->num_alloc_vsi; i++)
6481                 if (pf->vsi[i] && pf->vsi[i]->netdev)
6482                         i40e_update_stats(pf->vsi[i]);
6483
6484         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
6485                 /* Update the stats for the active switching components */
6486                 for (i = 0; i < I40E_MAX_VEB; i++)
6487                         if (pf->veb[i])
6488                                 i40e_update_veb_stats(pf->veb[i]);
6489         }
6490
6491         i40e_ptp_rx_hang(pf);
6492         i40e_ptp_tx_hang(pf);
6493 }
6494
6495 /**
6496  * i40e_reset_subtask - Set up for resetting the device and driver
6497  * @pf: board private structure
6498  **/
6499 static void i40e_reset_subtask(struct i40e_pf *pf)
6500 {
6501         u32 reset_flags = 0;
6502
6503         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
6504                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
6505                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
6506         }
6507         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
6508                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
6509                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6510         }
6511         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
6512                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
6513                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
6514         }
6515         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
6516                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
6517                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
6518         }
6519         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
6520                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
6521                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
6522         }
6523
6524         /* If there's a recovery already waiting, it takes
6525          * precedence before starting a new reset sequence.
6526          */
6527         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
6528                 i40e_prep_for_reset(pf, false);
6529                 i40e_reset(pf);
6530                 i40e_rebuild(pf, false, false);
6531         }
6532
6533         /* If we're already down or resetting, just bail */
6534         if (reset_flags &&
6535             !test_bit(__I40E_DOWN, pf->state) &&
6536             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
6537                 i40e_do_reset(pf, reset_flags, false);
6538         }
6539 }
6540
6541 /**
6542  * i40e_handle_link_event - Handle link event
6543  * @pf: board private structure
6544  * @e: event info posted on ARQ
6545  **/
6546 static void i40e_handle_link_event(struct i40e_pf *pf,
6547                                    struct i40e_arq_event_info *e)
6548 {
6549         struct i40e_aqc_get_link_status *status =
6550                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
6551
6552         /* Do a new status request to re-enable LSE reporting
6553          * and load new status information into the hw struct
6554          * This completely ignores any state information
6555          * in the ARQ event info, instead choosing to always
6556          * issue the AQ update link status command.
6557          */
6558         i40e_link_event(pf);
6559
6560         /* check for unqualified module, if link is down */
6561         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
6562             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
6563             (!(status->link_info & I40E_AQ_LINK_UP)))
6564                 dev_err(&pf->pdev->dev,
6565                         "The driver failed to link because an unqualified module was detected.\n");
6566 }
6567
6568 /**
6569  * i40e_clean_adminq_subtask - Clean the AdminQ rings
6570  * @pf: board private structure
6571  **/
6572 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
6573 {
6574         struct i40e_arq_event_info event;
6575         struct i40e_hw *hw = &pf->hw;
6576         u16 pending, i = 0;
6577         i40e_status ret;
6578         u16 opcode;
6579         u32 oldval;
6580         u32 val;
6581
6582         /* Do not run clean AQ when PF reset fails */
6583         if (test_bit(__I40E_RESET_FAILED, pf->state))
6584                 return;
6585
6586         /* check for error indications */
6587         val = rd32(&pf->hw, pf->hw.aq.arq.len);
6588         oldval = val;
6589         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
6590                 if (hw->debug_mask & I40E_DEBUG_AQ)
6591                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
6592                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
6593         }
6594         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
6595                 if (hw->debug_mask & I40E_DEBUG_AQ)
6596                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
6597                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
6598                 pf->arq_overflows++;
6599         }
6600         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
6601                 if (hw->debug_mask & I40E_DEBUG_AQ)
6602                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
6603                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
6604         }
6605         if (oldval != val)
6606                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
6607
6608         val = rd32(&pf->hw, pf->hw.aq.asq.len);
6609         oldval = val;
6610         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
6611                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6612                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
6613                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
6614         }
6615         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
6616                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6617                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
6618                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
6619         }
6620         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
6621                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6622                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
6623                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
6624         }
6625         if (oldval != val)
6626                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
6627
6628         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
6629         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
6630         if (!event.msg_buf)
6631                 return;
6632
6633         do {
6634                 ret = i40e_clean_arq_element(hw, &event, &pending);
6635                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
6636                         break;
6637                 else if (ret) {
6638                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
6639                         break;
6640                 }
6641
6642                 opcode = le16_to_cpu(event.desc.opcode);
6643                 switch (opcode) {
6644
6645                 case i40e_aqc_opc_get_link_status:
6646                         i40e_handle_link_event(pf, &event);
6647                         break;
6648                 case i40e_aqc_opc_send_msg_to_pf:
6649                         ret = i40e_vc_process_vf_msg(pf,
6650                                         le16_to_cpu(event.desc.retval),
6651                                         le32_to_cpu(event.desc.cookie_high),
6652                                         le32_to_cpu(event.desc.cookie_low),
6653                                         event.msg_buf,
6654                                         event.msg_len);
6655                         break;
6656                 case i40e_aqc_opc_lldp_update_mib:
6657                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
6658 #ifdef CONFIG_I40E_DCB
6659                         rtnl_lock();
6660                         ret = i40e_handle_lldp_event(pf, &event);
6661                         rtnl_unlock();
6662 #endif /* CONFIG_I40E_DCB */
6663                         break;
6664                 case i40e_aqc_opc_event_lan_overflow:
6665                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
6666                         i40e_handle_lan_overflow_event(pf, &event);
6667                         break;
6668                 case i40e_aqc_opc_send_msg_to_peer:
6669                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
6670                         break;
6671                 case i40e_aqc_opc_nvm_erase:
6672                 case i40e_aqc_opc_nvm_update:
6673                 case i40e_aqc_opc_oem_post_update:
6674                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
6675                                    "ARQ NVM operation 0x%04x completed\n",
6676                                    opcode);
6677                         break;
6678                 default:
6679                         dev_info(&pf->pdev->dev,
6680                                  "ARQ: Unknown event 0x%04x ignored\n",
6681                                  opcode);
6682                         break;
6683                 }
6684         } while (i++ < pf->adminq_work_limit);
6685
6686         if (i < pf->adminq_work_limit)
6687                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
6688
6689         /* re-enable Admin queue interrupt cause */
6690         val = rd32(hw, I40E_PFINT_ICR0_ENA);
6691         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
6692         wr32(hw, I40E_PFINT_ICR0_ENA, val);
6693         i40e_flush(hw);
6694
6695         kfree(event.msg_buf);
6696 }
6697
6698 /**
6699  * i40e_verify_eeprom - make sure eeprom is good to use
6700  * @pf: board private structure
6701  **/
6702 static void i40e_verify_eeprom(struct i40e_pf *pf)
6703 {
6704         int err;
6705
6706         err = i40e_diag_eeprom_test(&pf->hw);
6707         if (err) {
6708                 /* retry in case of garbage read */
6709                 err = i40e_diag_eeprom_test(&pf->hw);
6710                 if (err) {
6711                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
6712                                  err);
6713                         set_bit(__I40E_BAD_EEPROM, pf->state);
6714                 }
6715         }
6716
6717         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
6718                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
6719                 clear_bit(__I40E_BAD_EEPROM, pf->state);
6720         }
6721 }
6722
6723 /**
6724  * i40e_enable_pf_switch_lb
6725  * @pf: pointer to the PF structure
6726  *
6727  * enable switch loop back or die - no point in a return value
6728  **/
6729 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
6730 {
6731         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6732         struct i40e_vsi_context ctxt;
6733         int ret;
6734
6735         ctxt.seid = pf->main_vsi_seid;
6736         ctxt.pf_num = pf->hw.pf_id;
6737         ctxt.vf_num = 0;
6738         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6739         if (ret) {
6740                 dev_info(&pf->pdev->dev,
6741                          "couldn't get PF vsi config, err %s aq_err %s\n",
6742                          i40e_stat_str(&pf->hw, ret),
6743                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6744                 return;
6745         }
6746         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6747         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6748         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6749
6750         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6751         if (ret) {
6752                 dev_info(&pf->pdev->dev,
6753                          "update vsi switch failed, err %s aq_err %s\n",
6754                          i40e_stat_str(&pf->hw, ret),
6755                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6756         }
6757 }
6758
6759 /**
6760  * i40e_disable_pf_switch_lb
6761  * @pf: pointer to the PF structure
6762  *
6763  * disable switch loop back or die - no point in a return value
6764  **/
6765 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
6766 {
6767         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6768         struct i40e_vsi_context ctxt;
6769         int ret;
6770
6771         ctxt.seid = pf->main_vsi_seid;
6772         ctxt.pf_num = pf->hw.pf_id;
6773         ctxt.vf_num = 0;
6774         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6775         if (ret) {
6776                 dev_info(&pf->pdev->dev,
6777                          "couldn't get PF vsi config, err %s aq_err %s\n",
6778                          i40e_stat_str(&pf->hw, ret),
6779                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6780                 return;
6781         }
6782         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6783         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6784         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6785
6786         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6787         if (ret) {
6788                 dev_info(&pf->pdev->dev,
6789                          "update vsi switch failed, err %s aq_err %s\n",
6790                          i40e_stat_str(&pf->hw, ret),
6791                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6792         }
6793 }
6794
6795 /**
6796  * i40e_config_bridge_mode - Configure the HW bridge mode
6797  * @veb: pointer to the bridge instance
6798  *
6799  * Configure the loop back mode for the LAN VSI that is downlink to the
6800  * specified HW bridge instance. It is expected this function is called
6801  * when a new HW bridge is instantiated.
6802  **/
6803 static void i40e_config_bridge_mode(struct i40e_veb *veb)
6804 {
6805         struct i40e_pf *pf = veb->pf;
6806
6807         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
6808                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
6809                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
6810         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
6811                 i40e_disable_pf_switch_lb(pf);
6812         else
6813                 i40e_enable_pf_switch_lb(pf);
6814 }
6815
6816 /**
6817  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
6818  * @veb: pointer to the VEB instance
6819  *
6820  * This is a recursive function that first builds the attached VSIs then
6821  * recurses in to build the next layer of VEB.  We track the connections
6822  * through our own index numbers because the seid's from the HW could
6823  * change across the reset.
6824  **/
6825 static int i40e_reconstitute_veb(struct i40e_veb *veb)
6826 {
6827         struct i40e_vsi *ctl_vsi = NULL;
6828         struct i40e_pf *pf = veb->pf;
6829         int v, veb_idx;
6830         int ret;
6831
6832         /* build VSI that owns this VEB, temporarily attached to base VEB */
6833         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
6834                 if (pf->vsi[v] &&
6835                     pf->vsi[v]->veb_idx == veb->idx &&
6836                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
6837                         ctl_vsi = pf->vsi[v];
6838                         break;
6839                 }
6840         }
6841         if (!ctl_vsi) {
6842                 dev_info(&pf->pdev->dev,
6843                          "missing owner VSI for veb_idx %d\n", veb->idx);
6844                 ret = -ENOENT;
6845                 goto end_reconstitute;
6846         }
6847         if (ctl_vsi != pf->vsi[pf->lan_vsi])
6848                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6849         ret = i40e_add_vsi(ctl_vsi);
6850         if (ret) {
6851                 dev_info(&pf->pdev->dev,
6852                          "rebuild of veb_idx %d owner VSI failed: %d\n",
6853                          veb->idx, ret);
6854                 goto end_reconstitute;
6855         }
6856         i40e_vsi_reset_stats(ctl_vsi);
6857
6858         /* create the VEB in the switch and move the VSI onto the VEB */
6859         ret = i40e_add_veb(veb, ctl_vsi);
6860         if (ret)
6861                 goto end_reconstitute;
6862
6863         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
6864                 veb->bridge_mode = BRIDGE_MODE_VEB;
6865         else
6866                 veb->bridge_mode = BRIDGE_MODE_VEPA;
6867         i40e_config_bridge_mode(veb);
6868
6869         /* create the remaining VSIs attached to this VEB */
6870         for (v = 0; v < pf->num_alloc_vsi; v++) {
6871                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
6872                         continue;
6873
6874                 if (pf->vsi[v]->veb_idx == veb->idx) {
6875                         struct i40e_vsi *vsi = pf->vsi[v];
6876
6877                         vsi->uplink_seid = veb->seid;
6878                         ret = i40e_add_vsi(vsi);
6879                         if (ret) {
6880                                 dev_info(&pf->pdev->dev,
6881                                          "rebuild of vsi_idx %d failed: %d\n",
6882                                          v, ret);
6883                                 goto end_reconstitute;
6884                         }
6885                         i40e_vsi_reset_stats(vsi);
6886                 }
6887         }
6888
6889         /* create any VEBs attached to this VEB - RECURSION */
6890         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
6891                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
6892                         pf->veb[veb_idx]->uplink_seid = veb->seid;
6893                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
6894                         if (ret)
6895                                 break;
6896                 }
6897         }
6898
6899 end_reconstitute:
6900         return ret;
6901 }
6902
6903 /**
6904  * i40e_get_capabilities - get info about the HW
6905  * @pf: the PF struct
6906  **/
6907 static int i40e_get_capabilities(struct i40e_pf *pf)
6908 {
6909         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
6910         u16 data_size;
6911         int buf_len;
6912         int err;
6913
6914         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
6915         do {
6916                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
6917                 if (!cap_buf)
6918                         return -ENOMEM;
6919
6920                 /* this loads the data into the hw struct for us */
6921                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
6922                                             &data_size,
6923                                             i40e_aqc_opc_list_func_capabilities,
6924                                             NULL);
6925                 /* data loaded, buffer no longer needed */
6926                 kfree(cap_buf);
6927
6928                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
6929                         /* retry with a larger buffer */
6930                         buf_len = data_size;
6931                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
6932                         dev_info(&pf->pdev->dev,
6933                                  "capability discovery failed, err %s aq_err %s\n",
6934                                  i40e_stat_str(&pf->hw, err),
6935                                  i40e_aq_str(&pf->hw,
6936                                              pf->hw.aq.asq_last_status));
6937                         return -ENODEV;
6938                 }
6939         } while (err);
6940
6941         if (pf->hw.debug_mask & I40E_DEBUG_USER)
6942                 dev_info(&pf->pdev->dev,
6943                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
6944                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
6945                          pf->hw.func_caps.num_msix_vectors,
6946                          pf->hw.func_caps.num_msix_vectors_vf,
6947                          pf->hw.func_caps.fd_filters_guaranteed,
6948                          pf->hw.func_caps.fd_filters_best_effort,
6949                          pf->hw.func_caps.num_tx_qp,
6950                          pf->hw.func_caps.num_vsis);
6951
6952 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
6953                        + pf->hw.func_caps.num_vfs)
6954         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
6955                 dev_info(&pf->pdev->dev,
6956                          "got num_vsis %d, setting num_vsis to %d\n",
6957                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
6958                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
6959         }
6960
6961         return 0;
6962 }
6963
6964 static int i40e_vsi_clear(struct i40e_vsi *vsi);
6965
6966 /**
6967  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
6968  * @pf: board private structure
6969  **/
6970 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
6971 {
6972         struct i40e_vsi *vsi;
6973
6974         /* quick workaround for an NVM issue that leaves a critical register
6975          * uninitialized
6976          */
6977         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
6978                 static const u32 hkey[] = {
6979                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
6980                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
6981                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
6982                         0x95b3a76d};
6983                 int i;
6984
6985                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
6986                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
6987         }
6988
6989         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
6990                 return;
6991
6992         /* find existing VSI and see if it needs configuring */
6993         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
6994
6995         /* create a new VSI if none exists */
6996         if (!vsi) {
6997                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
6998                                      pf->vsi[pf->lan_vsi]->seid, 0);
6999                 if (!vsi) {
7000                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
7001                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
7002                         return;
7003                 }
7004         }
7005
7006         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
7007 }
7008
7009 /**
7010  * i40e_fdir_teardown - release the Flow Director resources
7011  * @pf: board private structure
7012  **/
7013 static void i40e_fdir_teardown(struct i40e_pf *pf)
7014 {
7015         struct i40e_vsi *vsi;
7016
7017         i40e_fdir_filter_exit(pf);
7018         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
7019         if (vsi)
7020                 i40e_vsi_release(vsi);
7021 }
7022
7023 /**
7024  * i40e_prep_for_reset - prep for the core to reset
7025  * @pf: board private structure
7026  * @lock_acquired: indicates whether or not the lock has been acquired
7027  * before this function was called.
7028  *
7029  * Close up the VFs and other things in prep for PF Reset.
7030   **/
7031 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
7032 {
7033         struct i40e_hw *hw = &pf->hw;
7034         i40e_status ret = 0;
7035         u32 v;
7036
7037         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
7038         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
7039                 return;
7040         if (i40e_check_asq_alive(&pf->hw))
7041                 i40e_vc_notify_reset(pf);
7042
7043         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
7044
7045         /* quiesce the VSIs and their queues that are not already DOWN */
7046         /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */
7047         if (!lock_acquired)
7048                 rtnl_lock();
7049         i40e_pf_quiesce_all_vsi(pf);
7050         if (!lock_acquired)
7051                 rtnl_unlock();
7052
7053         for (v = 0; v < pf->num_alloc_vsi; v++) {
7054                 if (pf->vsi[v])
7055                         pf->vsi[v]->seid = 0;
7056         }
7057
7058         i40e_shutdown_adminq(&pf->hw);
7059
7060         /* call shutdown HMC */
7061         if (hw->hmc.hmc_obj) {
7062                 ret = i40e_shutdown_lan_hmc(hw);
7063                 if (ret)
7064                         dev_warn(&pf->pdev->dev,
7065                                  "shutdown_lan_hmc failed: %d\n", ret);
7066         }
7067 }
7068
7069 /**
7070  * i40e_send_version - update firmware with driver version
7071  * @pf: PF struct
7072  */
7073 static void i40e_send_version(struct i40e_pf *pf)
7074 {
7075         struct i40e_driver_version dv;
7076
7077         dv.major_version = DRV_VERSION_MAJOR;
7078         dv.minor_version = DRV_VERSION_MINOR;
7079         dv.build_version = DRV_VERSION_BUILD;
7080         dv.subbuild_version = 0;
7081         strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
7082         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
7083 }
7084
7085 /**
7086  * i40e_get_oem_version - get OEM specific version information
7087  * @hw: pointer to the hardware structure
7088  **/
7089 static void i40e_get_oem_version(struct i40e_hw *hw)
7090 {
7091         u16 block_offset = 0xffff;
7092         u16 block_length = 0;
7093         u16 capabilities = 0;
7094         u16 gen_snap = 0;
7095         u16 release = 0;
7096
7097 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
7098 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
7099 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
7100 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
7101 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
7102 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
7103 #define I40E_NVM_OEM_LENGTH                     3
7104
7105         /* Check if pointer to OEM version block is valid. */
7106         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
7107         if (block_offset == 0xffff)
7108                 return;
7109
7110         /* Check if OEM version block has correct length. */
7111         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
7112                            &block_length);
7113         if (block_length < I40E_NVM_OEM_LENGTH)
7114                 return;
7115
7116         /* Check if OEM version format is as expected. */
7117         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
7118                            &capabilities);
7119         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
7120                 return;
7121
7122         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
7123                            &gen_snap);
7124         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
7125                            &release);
7126         hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
7127         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
7128 }
7129
7130 /**
7131  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
7132  * @pf: board private structure
7133  **/
7134 static int i40e_reset(struct i40e_pf *pf)
7135 {
7136         struct i40e_hw *hw = &pf->hw;
7137         i40e_status ret;
7138
7139         ret = i40e_pf_reset(hw);
7140         if (ret) {
7141                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
7142                 set_bit(__I40E_RESET_FAILED, pf->state);
7143                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
7144         } else {
7145                 pf->pfr_count++;
7146         }
7147         return ret;
7148 }
7149
7150 /**
7151  * i40e_rebuild - rebuild using a saved config
7152  * @pf: board private structure
7153  * @reinit: if the Main VSI needs to re-initialized.
7154  * @lock_acquired: indicates whether or not the lock has been acquired
7155  * before this function was called.
7156  **/
7157 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
7158 {
7159         struct i40e_hw *hw = &pf->hw;
7160         u8 set_fc_aq_fail = 0;
7161         i40e_status ret;
7162         u32 val;
7163         int v;
7164
7165         if (test_bit(__I40E_DOWN, pf->state))
7166                 goto clear_recovery;
7167         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
7168
7169         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
7170         ret = i40e_init_adminq(&pf->hw);
7171         if (ret) {
7172                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
7173                          i40e_stat_str(&pf->hw, ret),
7174                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7175                 goto clear_recovery;
7176         }
7177         i40e_get_oem_version(&pf->hw);
7178
7179         /* re-verify the eeprom if we just had an EMP reset */
7180         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
7181                 i40e_verify_eeprom(pf);
7182
7183         i40e_clear_pxe_mode(hw);
7184         ret = i40e_get_capabilities(pf);
7185         if (ret)
7186                 goto end_core_reset;
7187
7188         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
7189                                 hw->func_caps.num_rx_qp, 0, 0);
7190         if (ret) {
7191                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
7192                 goto end_core_reset;
7193         }
7194         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
7195         if (ret) {
7196                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
7197                 goto end_core_reset;
7198         }
7199
7200 #ifdef CONFIG_I40E_DCB
7201         ret = i40e_init_pf_dcb(pf);
7202         if (ret) {
7203                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
7204                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
7205                 /* Continue without DCB enabled */
7206         }
7207 #endif /* CONFIG_I40E_DCB */
7208         /* do basic switch setup */
7209         if (!lock_acquired)
7210                 rtnl_lock();
7211         ret = i40e_setup_pf_switch(pf, reinit);
7212         if (ret)
7213                 goto end_unlock;
7214
7215         /* The driver only wants link up/down and module qualification
7216          * reports from firmware.  Note the negative logic.
7217          */
7218         ret = i40e_aq_set_phy_int_mask(&pf->hw,
7219                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
7220                                          I40E_AQ_EVENT_MEDIA_NA |
7221                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
7222         if (ret)
7223                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
7224                          i40e_stat_str(&pf->hw, ret),
7225                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7226
7227         /* make sure our flow control settings are restored */
7228         ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
7229         if (ret)
7230                 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
7231                         i40e_stat_str(&pf->hw, ret),
7232                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7233
7234         /* Rebuild the VSIs and VEBs that existed before reset.
7235          * They are still in our local switch element arrays, so only
7236          * need to rebuild the switch model in the HW.
7237          *
7238          * If there were VEBs but the reconstitution failed, we'll try
7239          * try to recover minimal use by getting the basic PF VSI working.
7240          */
7241         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
7242                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
7243                 /* find the one VEB connected to the MAC, and find orphans */
7244                 for (v = 0; v < I40E_MAX_VEB; v++) {
7245                         if (!pf->veb[v])
7246                                 continue;
7247
7248                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
7249                             pf->veb[v]->uplink_seid == 0) {
7250                                 ret = i40e_reconstitute_veb(pf->veb[v]);
7251
7252                                 if (!ret)
7253                                         continue;
7254
7255                                 /* If Main VEB failed, we're in deep doodoo,
7256                                  * so give up rebuilding the switch and set up
7257                                  * for minimal rebuild of PF VSI.
7258                                  * If orphan failed, we'll report the error
7259                                  * but try to keep going.
7260                                  */
7261                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
7262                                         dev_info(&pf->pdev->dev,
7263                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
7264                                                  ret);
7265                                         pf->vsi[pf->lan_vsi]->uplink_seid
7266                                                                 = pf->mac_seid;
7267                                         break;
7268                                 } else if (pf->veb[v]->uplink_seid == 0) {
7269                                         dev_info(&pf->pdev->dev,
7270                                                  "rebuild of orphan VEB failed: %d\n",
7271                                                  ret);
7272                                 }
7273                         }
7274                 }
7275         }
7276
7277         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
7278                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
7279                 /* no VEB, so rebuild only the Main VSI */
7280                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
7281                 if (ret) {
7282                         dev_info(&pf->pdev->dev,
7283                                  "rebuild of Main VSI failed: %d\n", ret);
7284                         goto end_unlock;
7285                 }
7286         }
7287
7288         /* Reconfigure hardware for allowing smaller MSS in the case
7289          * of TSO, so that we avoid the MDD being fired and causing
7290          * a reset in the case of small MSS+TSO.
7291          */
7292 #define I40E_REG_MSS          0x000E64DC
7293 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
7294 #define I40E_64BYTE_MSS       0x400000
7295         val = rd32(hw, I40E_REG_MSS);
7296         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
7297                 val &= ~I40E_REG_MSS_MIN_MASK;
7298                 val |= I40E_64BYTE_MSS;
7299                 wr32(hw, I40E_REG_MSS, val);
7300         }
7301
7302         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
7303                 msleep(75);
7304                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
7305                 if (ret)
7306                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
7307                                  i40e_stat_str(&pf->hw, ret),
7308                                  i40e_aq_str(&pf->hw,
7309                                              pf->hw.aq.asq_last_status));
7310         }
7311         /* reinit the misc interrupt */
7312         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7313                 ret = i40e_setup_misc_vector(pf);
7314
7315         /* Add a filter to drop all Flow control frames from any VSI from being
7316          * transmitted. By doing so we stop a malicious VF from sending out
7317          * PAUSE or PFC frames and potentially controlling traffic for other
7318          * PF/VF VSIs.
7319          * The FW can still send Flow control frames if enabled.
7320          */
7321         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
7322                                                        pf->main_vsi_seid);
7323
7324         /* restart the VSIs that were rebuilt and running before the reset */
7325         i40e_pf_unquiesce_all_vsi(pf);
7326
7327         /* Release the RTNL lock before we start resetting VFs */
7328         if (!lock_acquired)
7329                 rtnl_unlock();
7330
7331         i40e_reset_all_vfs(pf, true);
7332
7333         /* tell the firmware that we're starting */
7334         i40e_send_version(pf);
7335
7336         /* We've already released the lock, so don't do it again */
7337         goto end_core_reset;
7338
7339 end_unlock:
7340         if (!lock_acquired)
7341                 rtnl_unlock();
7342 end_core_reset:
7343         clear_bit(__I40E_RESET_FAILED, pf->state);
7344 clear_recovery:
7345         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
7346 }
7347
7348 /**
7349  * i40e_reset_and_rebuild - reset and rebuild using a saved config
7350  * @pf: board private structure
7351  * @reinit: if the Main VSI needs to re-initialized.
7352  * @lock_acquired: indicates whether or not the lock has been acquired
7353  * before this function was called.
7354  **/
7355 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
7356                                    bool lock_acquired)
7357 {
7358         int ret;
7359         /* Now we wait for GRST to settle out.
7360          * We don't have to delete the VEBs or VSIs from the hw switch
7361          * because the reset will make them disappear.
7362          */
7363         ret = i40e_reset(pf);
7364         if (!ret)
7365                 i40e_rebuild(pf, reinit, lock_acquired);
7366 }
7367
7368 /**
7369  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
7370  * @pf: board private structure
7371  *
7372  * Close up the VFs and other things in prep for a Core Reset,
7373  * then get ready to rebuild the world.
7374  * @lock_acquired: indicates whether or not the lock has been acquired
7375  * before this function was called.
7376  **/
7377 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
7378 {
7379         i40e_prep_for_reset(pf, lock_acquired);
7380         i40e_reset_and_rebuild(pf, false, lock_acquired);
7381 }
7382
7383 /**
7384  * i40e_handle_mdd_event
7385  * @pf: pointer to the PF structure
7386  *
7387  * Called from the MDD irq handler to identify possibly malicious vfs
7388  **/
7389 static void i40e_handle_mdd_event(struct i40e_pf *pf)
7390 {
7391         struct i40e_hw *hw = &pf->hw;
7392         bool mdd_detected = false;
7393         bool pf_mdd_detected = false;
7394         struct i40e_vf *vf;
7395         u32 reg;
7396         int i;
7397
7398         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
7399                 return;
7400
7401         /* find what triggered the MDD event */
7402         reg = rd32(hw, I40E_GL_MDET_TX);
7403         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
7404                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
7405                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
7406                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
7407                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
7408                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
7409                                 I40E_GL_MDET_TX_EVENT_SHIFT;
7410                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
7411                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
7412                                 pf->hw.func_caps.base_queue;
7413                 if (netif_msg_tx_err(pf))
7414                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
7415                                  event, queue, pf_num, vf_num);
7416                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
7417                 mdd_detected = true;
7418         }
7419         reg = rd32(hw, I40E_GL_MDET_RX);
7420         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
7421                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
7422                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
7423                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
7424                                 I40E_GL_MDET_RX_EVENT_SHIFT;
7425                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
7426                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
7427                                 pf->hw.func_caps.base_queue;
7428                 if (netif_msg_rx_err(pf))
7429                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
7430                                  event, queue, func);
7431                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
7432                 mdd_detected = true;
7433         }
7434
7435         if (mdd_detected) {
7436                 reg = rd32(hw, I40E_PF_MDET_TX);
7437                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
7438                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
7439                         dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
7440                         pf_mdd_detected = true;
7441                 }
7442                 reg = rd32(hw, I40E_PF_MDET_RX);
7443                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
7444                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
7445                         dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
7446                         pf_mdd_detected = true;
7447                 }
7448                 /* Queue belongs to the PF, initiate a reset */
7449                 if (pf_mdd_detected) {
7450                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
7451                         i40e_service_event_schedule(pf);
7452                 }
7453         }
7454
7455         /* see if one of the VFs needs its hand slapped */
7456         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
7457                 vf = &(pf->vf[i]);
7458                 reg = rd32(hw, I40E_VP_MDET_TX(i));
7459                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
7460                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
7461                         vf->num_mdd_events++;
7462                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
7463                                  i);
7464                 }
7465
7466                 reg = rd32(hw, I40E_VP_MDET_RX(i));
7467                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
7468                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
7469                         vf->num_mdd_events++;
7470                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
7471                                  i);
7472                 }
7473
7474                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
7475                         dev_info(&pf->pdev->dev,
7476                                  "Too many MDD events on VF %d, disabled\n", i);
7477                         dev_info(&pf->pdev->dev,
7478                                  "Use PF Control I/F to re-enable the VF\n");
7479                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
7480                 }
7481         }
7482
7483         /* re-enable mdd interrupt cause */
7484         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
7485         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
7486         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
7487         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
7488         i40e_flush(hw);
7489 }
7490
7491 static const char *i40e_tunnel_name(struct i40e_udp_port_config *port)
7492 {
7493         switch (port->type) {
7494         case UDP_TUNNEL_TYPE_VXLAN:
7495                 return "vxlan";
7496         case UDP_TUNNEL_TYPE_GENEVE:
7497                 return "geneve";
7498         default:
7499                 return "unknown";
7500         }
7501 }
7502
7503 /**
7504  * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters
7505  * @pf: board private structure
7506  **/
7507 static void i40e_sync_udp_filters(struct i40e_pf *pf)
7508 {
7509         int i;
7510
7511         /* loop through and set pending bit for all active UDP filters */
7512         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7513                 if (pf->udp_ports[i].port)
7514                         pf->pending_udp_bitmap |= BIT_ULL(i);
7515         }
7516
7517         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
7518 }
7519
7520 /**
7521  * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
7522  * @pf: board private structure
7523  **/
7524 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
7525 {
7526         struct i40e_hw *hw = &pf->hw;
7527         i40e_status ret;
7528         u16 port;
7529         int i;
7530
7531         if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC))
7532                 return;
7533
7534         pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC;
7535
7536         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7537                 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
7538                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
7539                         port = pf->udp_ports[i].port;
7540                         if (port)
7541                                 ret = i40e_aq_add_udp_tunnel(hw, port,
7542                                                         pf->udp_ports[i].type,
7543                                                         NULL, NULL);
7544                         else
7545                                 ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
7546
7547                         if (ret) {
7548                                 dev_info(&pf->pdev->dev,
7549                                          "%s %s port %d, index %d failed, err %s aq_err %s\n",
7550                                          i40e_tunnel_name(&pf->udp_ports[i]),
7551                                          port ? "add" : "delete",
7552                                          port, i,
7553                                          i40e_stat_str(&pf->hw, ret),
7554                                          i40e_aq_str(&pf->hw,
7555                                                      pf->hw.aq.asq_last_status));
7556                                 pf->udp_ports[i].port = 0;
7557                         }
7558                 }
7559         }
7560 }
7561
7562 /**
7563  * i40e_service_task - Run the driver's async subtasks
7564  * @work: pointer to work_struct containing our data
7565  **/
7566 static void i40e_service_task(struct work_struct *work)
7567 {
7568         struct i40e_pf *pf = container_of(work,
7569                                           struct i40e_pf,
7570                                           service_task);
7571         unsigned long start_time = jiffies;
7572
7573         /* don't bother with service tasks if a reset is in progress */
7574         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
7575                 return;
7576
7577         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
7578                 return;
7579
7580         i40e_detect_recover_hung(pf);
7581         i40e_sync_filters_subtask(pf);
7582         i40e_reset_subtask(pf);
7583         i40e_handle_mdd_event(pf);
7584         i40e_vc_process_vflr_event(pf);
7585         i40e_watchdog_subtask(pf);
7586         i40e_fdir_reinit_subtask(pf);
7587         if (pf->flags & I40E_FLAG_CLIENT_RESET) {
7588                 /* Client subtask will reopen next time through. */
7589                 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true);
7590                 pf->flags &= ~I40E_FLAG_CLIENT_RESET;
7591         } else {
7592                 i40e_client_subtask(pf);
7593                 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) {
7594                         i40e_notify_client_of_l2_param_changes(
7595                                                         pf->vsi[pf->lan_vsi]);
7596                         pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE;
7597                 }
7598         }
7599         i40e_sync_filters_subtask(pf);
7600         i40e_sync_udp_filters_subtask(pf);
7601         i40e_clean_adminq_subtask(pf);
7602
7603         /* flush memory to make sure state is correct before next watchdog */
7604         smp_mb__before_atomic();
7605         clear_bit(__I40E_SERVICE_SCHED, pf->state);
7606
7607         /* If the tasks have taken longer than one timer cycle or there
7608          * is more work to be done, reschedule the service task now
7609          * rather than wait for the timer to tick again.
7610          */
7611         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
7612             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
7613             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
7614             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
7615                 i40e_service_event_schedule(pf);
7616 }
7617
7618 /**
7619  * i40e_service_timer - timer callback
7620  * @data: pointer to PF struct
7621  **/
7622 static void i40e_service_timer(unsigned long data)
7623 {
7624         struct i40e_pf *pf = (struct i40e_pf *)data;
7625
7626         mod_timer(&pf->service_timer,
7627                   round_jiffies(jiffies + pf->service_timer_period));
7628         i40e_service_event_schedule(pf);
7629 }
7630
7631 /**
7632  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
7633  * @vsi: the VSI being configured
7634  **/
7635 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
7636 {
7637         struct i40e_pf *pf = vsi->back;
7638
7639         switch (vsi->type) {
7640         case I40E_VSI_MAIN:
7641                 vsi->alloc_queue_pairs = pf->num_lan_qps;
7642                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7643                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7644                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7645                         vsi->num_q_vectors = pf->num_lan_msix;
7646                 else
7647                         vsi->num_q_vectors = 1;
7648
7649                 break;
7650
7651         case I40E_VSI_FDIR:
7652                 vsi->alloc_queue_pairs = 1;
7653                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
7654                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7655                 vsi->num_q_vectors = pf->num_fdsb_msix;
7656                 break;
7657
7658         case I40E_VSI_VMDQ2:
7659                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
7660                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7661                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7662                 vsi->num_q_vectors = pf->num_vmdq_msix;
7663                 break;
7664
7665         case I40E_VSI_SRIOV:
7666                 vsi->alloc_queue_pairs = pf->num_vf_qps;
7667                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7668                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7669                 break;
7670
7671         default:
7672                 WARN_ON(1);
7673                 return -ENODATA;
7674         }
7675
7676         return 0;
7677 }
7678
7679 /**
7680  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
7681  * @type: VSI pointer
7682  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
7683  *
7684  * On error: returns error code (negative)
7685  * On success: returns 0
7686  **/
7687 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
7688 {
7689         struct i40e_ring **next_rings;
7690         int size;
7691         int ret = 0;
7692
7693         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
7694         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
7695                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
7696         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
7697         if (!vsi->tx_rings)
7698                 return -ENOMEM;
7699         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
7700         if (i40e_enabled_xdp_vsi(vsi)) {
7701                 vsi->xdp_rings = next_rings;
7702                 next_rings += vsi->alloc_queue_pairs;
7703         }
7704         vsi->rx_rings = next_rings;
7705
7706         if (alloc_qvectors) {
7707                 /* allocate memory for q_vector pointers */
7708                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
7709                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
7710                 if (!vsi->q_vectors) {
7711                         ret = -ENOMEM;
7712                         goto err_vectors;
7713                 }
7714         }
7715         return ret;
7716
7717 err_vectors:
7718         kfree(vsi->tx_rings);
7719         return ret;
7720 }
7721
7722 /**
7723  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
7724  * @pf: board private structure
7725  * @type: type of VSI
7726  *
7727  * On error: returns error code (negative)
7728  * On success: returns vsi index in PF (positive)
7729  **/
7730 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
7731 {
7732         int ret = -ENODEV;
7733         struct i40e_vsi *vsi;
7734         int vsi_idx;
7735         int i;
7736
7737         /* Need to protect the allocation of the VSIs at the PF level */
7738         mutex_lock(&pf->switch_mutex);
7739
7740         /* VSI list may be fragmented if VSI creation/destruction has
7741          * been happening.  We can afford to do a quick scan to look
7742          * for any free VSIs in the list.
7743          *
7744          * find next empty vsi slot, looping back around if necessary
7745          */
7746         i = pf->next_vsi;
7747         while (i < pf->num_alloc_vsi && pf->vsi[i])
7748                 i++;
7749         if (i >= pf->num_alloc_vsi) {
7750                 i = 0;
7751                 while (i < pf->next_vsi && pf->vsi[i])
7752                         i++;
7753         }
7754
7755         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
7756                 vsi_idx = i;             /* Found one! */
7757         } else {
7758                 ret = -ENODEV;
7759                 goto unlock_pf;  /* out of VSI slots! */
7760         }
7761         pf->next_vsi = ++i;
7762
7763         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
7764         if (!vsi) {
7765                 ret = -ENOMEM;
7766                 goto unlock_pf;
7767         }
7768         vsi->type = type;
7769         vsi->back = pf;
7770         set_bit(__I40E_VSI_DOWN, vsi->state);
7771         vsi->flags = 0;
7772         vsi->idx = vsi_idx;
7773         vsi->int_rate_limit = 0;
7774         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
7775                                 pf->rss_table_size : 64;
7776         vsi->netdev_registered = false;
7777         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
7778         hash_init(vsi->mac_filter_hash);
7779         vsi->irqs_ready = false;
7780
7781         ret = i40e_set_num_rings_in_vsi(vsi);
7782         if (ret)
7783                 goto err_rings;
7784
7785         ret = i40e_vsi_alloc_arrays(vsi, true);
7786         if (ret)
7787                 goto err_rings;
7788
7789         /* Setup default MSIX irq handler for VSI */
7790         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
7791
7792         /* Initialize VSI lock */
7793         spin_lock_init(&vsi->mac_filter_hash_lock);
7794         pf->vsi[vsi_idx] = vsi;
7795         ret = vsi_idx;
7796         goto unlock_pf;
7797
7798 err_rings:
7799         pf->next_vsi = i - 1;
7800         kfree(vsi);
7801 unlock_pf:
7802         mutex_unlock(&pf->switch_mutex);
7803         return ret;
7804 }
7805
7806 /**
7807  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
7808  * @type: VSI pointer
7809  * @free_qvectors: a bool to specify if q_vectors need to be freed.
7810  *
7811  * On error: returns error code (negative)
7812  * On success: returns 0
7813  **/
7814 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
7815 {
7816         /* free the ring and vector containers */
7817         if (free_qvectors) {
7818                 kfree(vsi->q_vectors);
7819                 vsi->q_vectors = NULL;
7820         }
7821         kfree(vsi->tx_rings);
7822         vsi->tx_rings = NULL;
7823         vsi->rx_rings = NULL;
7824         vsi->xdp_rings = NULL;
7825 }
7826
7827 /**
7828  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
7829  * and lookup table
7830  * @vsi: Pointer to VSI structure
7831  */
7832 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
7833 {
7834         if (!vsi)
7835                 return;
7836
7837         kfree(vsi->rss_hkey_user);
7838         vsi->rss_hkey_user = NULL;
7839
7840         kfree(vsi->rss_lut_user);
7841         vsi->rss_lut_user = NULL;
7842 }
7843
7844 /**
7845  * i40e_vsi_clear - Deallocate the VSI provided
7846  * @vsi: the VSI being un-configured
7847  **/
7848 static int i40e_vsi_clear(struct i40e_vsi *vsi)
7849 {
7850         struct i40e_pf *pf;
7851
7852         if (!vsi)
7853                 return 0;
7854
7855         if (!vsi->back)
7856                 goto free_vsi;
7857         pf = vsi->back;
7858
7859         mutex_lock(&pf->switch_mutex);
7860         if (!pf->vsi[vsi->idx]) {
7861                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
7862                         vsi->idx, vsi->idx, vsi, vsi->type);
7863                 goto unlock_vsi;
7864         }
7865
7866         if (pf->vsi[vsi->idx] != vsi) {
7867                 dev_err(&pf->pdev->dev,
7868                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
7869                         pf->vsi[vsi->idx]->idx,
7870                         pf->vsi[vsi->idx],
7871                         pf->vsi[vsi->idx]->type,
7872                         vsi->idx, vsi, vsi->type);
7873                 goto unlock_vsi;
7874         }
7875
7876         /* updates the PF for this cleared vsi */
7877         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
7878         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
7879
7880         i40e_vsi_free_arrays(vsi, true);
7881         i40e_clear_rss_config_user(vsi);
7882
7883         pf->vsi[vsi->idx] = NULL;
7884         if (vsi->idx < pf->next_vsi)
7885                 pf->next_vsi = vsi->idx;
7886
7887 unlock_vsi:
7888         mutex_unlock(&pf->switch_mutex);
7889 free_vsi:
7890         kfree(vsi);
7891
7892         return 0;
7893 }
7894
7895 /**
7896  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
7897  * @vsi: the VSI being cleaned
7898  **/
7899 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
7900 {
7901         int i;
7902
7903         if (vsi->tx_rings && vsi->tx_rings[0]) {
7904                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7905                         kfree_rcu(vsi->tx_rings[i], rcu);
7906                         vsi->tx_rings[i] = NULL;
7907                         vsi->rx_rings[i] = NULL;
7908                         if (vsi->xdp_rings)
7909                                 vsi->xdp_rings[i] = NULL;
7910                 }
7911         }
7912 }
7913
7914 /**
7915  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
7916  * @vsi: the VSI being configured
7917  **/
7918 static int i40e_alloc_rings(struct i40e_vsi *vsi)
7919 {
7920         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
7921         struct i40e_pf *pf = vsi->back;
7922         struct i40e_ring *ring;
7923
7924         /* Set basic values in the rings to be used later during open() */
7925         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7926                 /* allocate space for both Tx and Rx in one shot */
7927                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
7928                 if (!ring)
7929                         goto err_out;
7930
7931                 ring->queue_index = i;
7932                 ring->reg_idx = vsi->base_queue + i;
7933                 ring->ring_active = false;
7934                 ring->vsi = vsi;
7935                 ring->netdev = vsi->netdev;
7936                 ring->dev = &pf->pdev->dev;
7937                 ring->count = vsi->num_desc;
7938                 ring->size = 0;
7939                 ring->dcb_tc = 0;
7940                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
7941                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7942                 ring->tx_itr_setting = pf->tx_itr_default;
7943                 vsi->tx_rings[i] = ring++;
7944
7945                 if (!i40e_enabled_xdp_vsi(vsi))
7946                         goto setup_rx;
7947
7948                 ring->queue_index = vsi->alloc_queue_pairs + i;
7949                 ring->reg_idx = vsi->base_queue + ring->queue_index;
7950                 ring->ring_active = false;
7951                 ring->vsi = vsi;
7952                 ring->netdev = NULL;
7953                 ring->dev = &pf->pdev->dev;
7954                 ring->count = vsi->num_desc;
7955                 ring->size = 0;
7956                 ring->dcb_tc = 0;
7957                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
7958                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7959                 set_ring_xdp(ring);
7960                 ring->tx_itr_setting = pf->tx_itr_default;
7961                 vsi->xdp_rings[i] = ring++;
7962
7963 setup_rx:
7964                 ring->queue_index = i;
7965                 ring->reg_idx = vsi->base_queue + i;
7966                 ring->ring_active = false;
7967                 ring->vsi = vsi;
7968                 ring->netdev = vsi->netdev;
7969                 ring->dev = &pf->pdev->dev;
7970                 ring->count = vsi->num_desc;
7971                 ring->size = 0;
7972                 ring->dcb_tc = 0;
7973                 ring->rx_itr_setting = pf->rx_itr_default;
7974                 vsi->rx_rings[i] = ring;
7975         }
7976
7977         return 0;
7978
7979 err_out:
7980         i40e_vsi_clear_rings(vsi);
7981         return -ENOMEM;
7982 }
7983
7984 /**
7985  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
7986  * @pf: board private structure
7987  * @vectors: the number of MSI-X vectors to request
7988  *
7989  * Returns the number of vectors reserved, or error
7990  **/
7991 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
7992 {
7993         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
7994                                         I40E_MIN_MSIX, vectors);
7995         if (vectors < 0) {
7996                 dev_info(&pf->pdev->dev,
7997                          "MSI-X vector reservation failed: %d\n", vectors);
7998                 vectors = 0;
7999         }
8000
8001         return vectors;
8002 }
8003
8004 /**
8005  * i40e_init_msix - Setup the MSIX capability
8006  * @pf: board private structure
8007  *
8008  * Work with the OS to set up the MSIX vectors needed.
8009  *
8010  * Returns the number of vectors reserved or negative on failure
8011  **/
8012 static int i40e_init_msix(struct i40e_pf *pf)
8013 {
8014         struct i40e_hw *hw = &pf->hw;
8015         int cpus, extra_vectors;
8016         int vectors_left;
8017         int v_budget, i;
8018         int v_actual;
8019         int iwarp_requested = 0;
8020
8021         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
8022                 return -ENODEV;
8023
8024         /* The number of vectors we'll request will be comprised of:
8025          *   - Add 1 for "other" cause for Admin Queue events, etc.
8026          *   - The number of LAN queue pairs
8027          *      - Queues being used for RSS.
8028          *              We don't need as many as max_rss_size vectors.
8029          *              use rss_size instead in the calculation since that
8030          *              is governed by number of cpus in the system.
8031          *      - assumes symmetric Tx/Rx pairing
8032          *   - The number of VMDq pairs
8033          *   - The CPU count within the NUMA node if iWARP is enabled
8034          * Once we count this up, try the request.
8035          *
8036          * If we can't get what we want, we'll simplify to nearly nothing
8037          * and try again.  If that still fails, we punt.
8038          */
8039         vectors_left = hw->func_caps.num_msix_vectors;
8040         v_budget = 0;
8041
8042         /* reserve one vector for miscellaneous handler */
8043         if (vectors_left) {
8044                 v_budget++;
8045                 vectors_left--;
8046         }
8047
8048         /* reserve some vectors for the main PF traffic queues. Initially we
8049          * only reserve at most 50% of the available vectors, in the case that
8050          * the number of online CPUs is large. This ensures that we can enable
8051          * extra features as well. Once we've enabled the other features, we
8052          * will use any remaining vectors to reach as close as we can to the
8053          * number of online CPUs.
8054          */
8055         cpus = num_online_cpus();
8056         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
8057         vectors_left -= pf->num_lan_msix;
8058
8059         /* reserve one vector for sideband flow director */
8060         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8061                 if (vectors_left) {
8062                         pf->num_fdsb_msix = 1;
8063                         v_budget++;
8064                         vectors_left--;
8065                 } else {
8066                         pf->num_fdsb_msix = 0;
8067                 }
8068         }
8069
8070         /* can we reserve enough for iWARP? */
8071         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8072                 iwarp_requested = pf->num_iwarp_msix;
8073
8074                 if (!vectors_left)
8075                         pf->num_iwarp_msix = 0;
8076                 else if (vectors_left < pf->num_iwarp_msix)
8077                         pf->num_iwarp_msix = 1;
8078                 v_budget += pf->num_iwarp_msix;
8079                 vectors_left -= pf->num_iwarp_msix;
8080         }
8081
8082         /* any vectors left over go for VMDq support */
8083         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
8084                 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps;
8085                 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted);
8086
8087                 if (!vectors_left) {
8088                         pf->num_vmdq_msix = 0;
8089                         pf->num_vmdq_qps = 0;
8090                 } else {
8091                         /* if we're short on vectors for what's desired, we limit
8092                          * the queues per vmdq.  If this is still more than are
8093                          * available, the user will need to change the number of
8094                          * queues/vectors used by the PF later with the ethtool
8095                          * channels command
8096                          */
8097                         if (vmdq_vecs < vmdq_vecs_wanted)
8098                                 pf->num_vmdq_qps = 1;
8099                         pf->num_vmdq_msix = pf->num_vmdq_qps;
8100
8101                         v_budget += vmdq_vecs;
8102                         vectors_left -= vmdq_vecs;
8103                 }
8104         }
8105
8106         /* On systems with a large number of SMP cores, we previously limited
8107          * the number of vectors for num_lan_msix to be at most 50% of the
8108          * available vectors, to allow for other features. Now, we add back
8109          * the remaining vectors. However, we ensure that the total
8110          * num_lan_msix will not exceed num_online_cpus(). To do this, we
8111          * calculate the number of vectors we can add without going over the
8112          * cap of CPUs. For systems with a small number of CPUs this will be
8113          * zero.
8114          */
8115         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
8116         pf->num_lan_msix += extra_vectors;
8117         vectors_left -= extra_vectors;
8118
8119         WARN(vectors_left < 0,
8120              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
8121
8122         v_budget += pf->num_lan_msix;
8123         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
8124                                    GFP_KERNEL);
8125         if (!pf->msix_entries)
8126                 return -ENOMEM;
8127
8128         for (i = 0; i < v_budget; i++)
8129                 pf->msix_entries[i].entry = i;
8130         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
8131
8132         if (v_actual < I40E_MIN_MSIX) {
8133                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
8134                 kfree(pf->msix_entries);
8135                 pf->msix_entries = NULL;
8136                 pci_disable_msix(pf->pdev);
8137                 return -ENODEV;
8138
8139         } else if (v_actual == I40E_MIN_MSIX) {
8140                 /* Adjust for minimal MSIX use */
8141                 pf->num_vmdq_vsis = 0;
8142                 pf->num_vmdq_qps = 0;
8143                 pf->num_lan_qps = 1;
8144                 pf->num_lan_msix = 1;
8145
8146         } else if (!vectors_left) {
8147                 /* If we have limited resources, we will start with no vectors
8148                  * for the special features and then allocate vectors to some
8149                  * of these features based on the policy and at the end disable
8150                  * the features that did not get any vectors.
8151                  */
8152                 int vec;
8153
8154                 dev_info(&pf->pdev->dev,
8155                          "MSI-X vector limit reached, attempting to redistribute vectors\n");
8156                 /* reserve the misc vector */
8157                 vec = v_actual - 1;
8158
8159                 /* Scale vector usage down */
8160                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
8161                 pf->num_vmdq_vsis = 1;
8162                 pf->num_vmdq_qps = 1;
8163
8164                 /* partition out the remaining vectors */
8165                 switch (vec) {
8166                 case 2:
8167                         pf->num_lan_msix = 1;
8168                         break;
8169                 case 3:
8170                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8171                                 pf->num_lan_msix = 1;
8172                                 pf->num_iwarp_msix = 1;
8173                         } else {
8174                                 pf->num_lan_msix = 2;
8175                         }
8176                         break;
8177                 default:
8178                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8179                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
8180                                                  iwarp_requested);
8181                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
8182                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
8183                         } else {
8184                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
8185                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
8186                         }
8187                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8188                                 pf->num_fdsb_msix = 1;
8189                                 vec--;
8190                         }
8191                         pf->num_lan_msix = min_t(int,
8192                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
8193                                                               pf->num_lan_msix);
8194                         pf->num_lan_qps = pf->num_lan_msix;
8195                         break;
8196                 }
8197         }
8198
8199         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
8200             (pf->num_fdsb_msix == 0)) {
8201                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
8202                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8203         }
8204         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
8205             (pf->num_vmdq_msix == 0)) {
8206                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
8207                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
8208         }
8209
8210         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
8211             (pf->num_iwarp_msix == 0)) {
8212                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
8213                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
8214         }
8215         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
8216                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
8217                    pf->num_lan_msix,
8218                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
8219                    pf->num_fdsb_msix,
8220                    pf->num_iwarp_msix);
8221
8222         return v_actual;
8223 }
8224
8225 /**
8226  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
8227  * @vsi: the VSI being configured
8228  * @v_idx: index of the vector in the vsi struct
8229  * @cpu: cpu to be used on affinity_mask
8230  *
8231  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
8232  **/
8233 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
8234 {
8235         struct i40e_q_vector *q_vector;
8236
8237         /* allocate q_vector */
8238         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
8239         if (!q_vector)
8240                 return -ENOMEM;
8241
8242         q_vector->vsi = vsi;
8243         q_vector->v_idx = v_idx;
8244         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
8245
8246         if (vsi->netdev)
8247                 netif_napi_add(vsi->netdev, &q_vector->napi,
8248                                i40e_napi_poll, NAPI_POLL_WEIGHT);
8249
8250         q_vector->rx.latency_range = I40E_LOW_LATENCY;
8251         q_vector->tx.latency_range = I40E_LOW_LATENCY;
8252
8253         /* tie q_vector and vsi together */
8254         vsi->q_vectors[v_idx] = q_vector;
8255
8256         return 0;
8257 }
8258
8259 /**
8260  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
8261  * @vsi: the VSI being configured
8262  *
8263  * We allocate one q_vector per queue interrupt.  If allocation fails we
8264  * return -ENOMEM.
8265  **/
8266 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
8267 {
8268         struct i40e_pf *pf = vsi->back;
8269         int err, v_idx, num_q_vectors, current_cpu;
8270
8271         /* if not MSIX, give the one vector only to the LAN VSI */
8272         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
8273                 num_q_vectors = vsi->num_q_vectors;
8274         else if (vsi == pf->vsi[pf->lan_vsi])
8275                 num_q_vectors = 1;
8276         else
8277                 return -EINVAL;
8278
8279         current_cpu = cpumask_first(cpu_online_mask);
8280
8281         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
8282                 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
8283                 if (err)
8284                         goto err_out;
8285                 current_cpu = cpumask_next(current_cpu, cpu_online_mask);
8286                 if (unlikely(current_cpu >= nr_cpu_ids))
8287                         current_cpu = cpumask_first(cpu_online_mask);
8288         }
8289
8290         return 0;
8291
8292 err_out:
8293         while (v_idx--)
8294                 i40e_free_q_vector(vsi, v_idx);
8295
8296         return err;
8297 }
8298
8299 /**
8300  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
8301  * @pf: board private structure to initialize
8302  **/
8303 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
8304 {
8305         int vectors = 0;
8306         ssize_t size;
8307
8308         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
8309                 vectors = i40e_init_msix(pf);
8310                 if (vectors < 0) {
8311                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
8312                                        I40E_FLAG_IWARP_ENABLED  |
8313                                        I40E_FLAG_RSS_ENABLED    |
8314                                        I40E_FLAG_DCB_CAPABLE    |
8315                                        I40E_FLAG_DCB_ENABLED    |
8316                                        I40E_FLAG_SRIOV_ENABLED  |
8317                                        I40E_FLAG_FD_SB_ENABLED  |
8318                                        I40E_FLAG_FD_ATR_ENABLED |
8319                                        I40E_FLAG_VMDQ_ENABLED);
8320
8321                         /* rework the queue expectations without MSIX */
8322                         i40e_determine_queue_usage(pf);
8323                 }
8324         }
8325
8326         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
8327             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
8328                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
8329                 vectors = pci_enable_msi(pf->pdev);
8330                 if (vectors < 0) {
8331                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
8332                                  vectors);
8333                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
8334                 }
8335                 vectors = 1;  /* one MSI or Legacy vector */
8336         }
8337
8338         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
8339                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
8340
8341         /* set up vector assignment tracking */
8342         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
8343         pf->irq_pile = kzalloc(size, GFP_KERNEL);
8344         if (!pf->irq_pile) {
8345                 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n");
8346                 return -ENOMEM;
8347         }
8348         pf->irq_pile->num_entries = vectors;
8349         pf->irq_pile->search_hint = 0;
8350
8351         /* track first vector for misc interrupts, ignore return */
8352         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
8353
8354         return 0;
8355 }
8356
8357 /**
8358  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
8359  * @pf: board private structure
8360  *
8361  * This sets up the handler for MSIX 0, which is used to manage the
8362  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
8363  * when in MSI or Legacy interrupt mode.
8364  **/
8365 static int i40e_setup_misc_vector(struct i40e_pf *pf)
8366 {
8367         struct i40e_hw *hw = &pf->hw;
8368         int err = 0;
8369
8370         /* Only request the IRQ once, the first time through. */
8371         if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
8372                 err = request_irq(pf->msix_entries[0].vector,
8373                                   i40e_intr, 0, pf->int_name, pf);
8374                 if (err) {
8375                         clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
8376                         dev_info(&pf->pdev->dev,
8377                                  "request_irq for %s failed: %d\n",
8378                                  pf->int_name, err);
8379                         return -EFAULT;
8380                 }
8381         }
8382
8383         i40e_enable_misc_int_causes(pf);
8384
8385         /* associate no queues to the misc vector */
8386         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
8387         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
8388
8389         i40e_flush(hw);
8390
8391         i40e_irq_dynamic_enable_icr0(pf, true);
8392
8393         return err;
8394 }
8395
8396 /**
8397  * i40e_config_rss_aq - Prepare for RSS using AQ commands
8398  * @vsi: vsi structure
8399  * @seed: RSS hash seed
8400  **/
8401 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8402                               u8 *lut, u16 lut_size)
8403 {
8404         struct i40e_pf *pf = vsi->back;
8405         struct i40e_hw *hw = &pf->hw;
8406         int ret = 0;
8407
8408         if (seed) {
8409                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
8410                         (struct i40e_aqc_get_set_rss_key_data *)seed;
8411                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
8412                 if (ret) {
8413                         dev_info(&pf->pdev->dev,
8414                                  "Cannot set RSS key, err %s aq_err %s\n",
8415                                  i40e_stat_str(hw, ret),
8416                                  i40e_aq_str(hw, hw->aq.asq_last_status));
8417                         return ret;
8418                 }
8419         }
8420         if (lut) {
8421                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8422
8423                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8424                 if (ret) {
8425                         dev_info(&pf->pdev->dev,
8426                                  "Cannot set RSS lut, err %s aq_err %s\n",
8427                                  i40e_stat_str(hw, ret),
8428                                  i40e_aq_str(hw, hw->aq.asq_last_status));
8429                         return ret;
8430                 }
8431         }
8432         return ret;
8433 }
8434
8435 /**
8436  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
8437  * @vsi: Pointer to vsi structure
8438  * @seed: Buffter to store the hash keys
8439  * @lut: Buffer to store the lookup table entries
8440  * @lut_size: Size of buffer to store the lookup table entries
8441  *
8442  * Return 0 on success, negative on failure
8443  */
8444 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8445                            u8 *lut, u16 lut_size)
8446 {
8447         struct i40e_pf *pf = vsi->back;
8448         struct i40e_hw *hw = &pf->hw;
8449         int ret = 0;
8450
8451         if (seed) {
8452                 ret = i40e_aq_get_rss_key(hw, vsi->id,
8453                         (struct i40e_aqc_get_set_rss_key_data *)seed);
8454                 if (ret) {
8455                         dev_info(&pf->pdev->dev,
8456                                  "Cannot get RSS key, err %s aq_err %s\n",
8457                                  i40e_stat_str(&pf->hw, ret),
8458                                  i40e_aq_str(&pf->hw,
8459                                              pf->hw.aq.asq_last_status));
8460                         return ret;
8461                 }
8462         }
8463
8464         if (lut) {
8465                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8466
8467                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8468                 if (ret) {
8469                         dev_info(&pf->pdev->dev,
8470                                  "Cannot get RSS lut, err %s aq_err %s\n",
8471                                  i40e_stat_str(&pf->hw, ret),
8472                                  i40e_aq_str(&pf->hw,
8473                                              pf->hw.aq.asq_last_status));
8474                         return ret;
8475                 }
8476         }
8477
8478         return ret;
8479 }
8480
8481 /**
8482  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
8483  * @vsi: VSI structure
8484  **/
8485 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
8486 {
8487         u8 seed[I40E_HKEY_ARRAY_SIZE];
8488         struct i40e_pf *pf = vsi->back;
8489         u8 *lut;
8490         int ret;
8491
8492         if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
8493                 return 0;
8494
8495         if (!vsi->rss_size)
8496                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
8497                                       vsi->num_queue_pairs);
8498         if (!vsi->rss_size)
8499                 return -EINVAL;
8500
8501         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8502         if (!lut)
8503                 return -ENOMEM;
8504         /* Use the user configured hash keys and lookup table if there is one,
8505          * otherwise use default
8506          */
8507         if (vsi->rss_lut_user)
8508                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8509         else
8510                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8511         if (vsi->rss_hkey_user)
8512                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8513         else
8514                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8515         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
8516         kfree(lut);
8517
8518         return ret;
8519 }
8520
8521 /**
8522  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
8523  * @vsi: Pointer to vsi structure
8524  * @seed: RSS hash seed
8525  * @lut: Lookup table
8526  * @lut_size: Lookup table size
8527  *
8528  * Returns 0 on success, negative on failure
8529  **/
8530 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
8531                                const u8 *lut, u16 lut_size)
8532 {
8533         struct i40e_pf *pf = vsi->back;
8534         struct i40e_hw *hw = &pf->hw;
8535         u16 vf_id = vsi->vf_id;
8536         u8 i;
8537
8538         /* Fill out hash function seed */
8539         if (seed) {
8540                 u32 *seed_dw = (u32 *)seed;
8541
8542                 if (vsi->type == I40E_VSI_MAIN) {
8543                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8544                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
8545                 } else if (vsi->type == I40E_VSI_SRIOV) {
8546                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
8547                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
8548                 } else {
8549                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
8550                 }
8551         }
8552
8553         if (lut) {
8554                 u32 *lut_dw = (u32 *)lut;
8555
8556                 if (vsi->type == I40E_VSI_MAIN) {
8557                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
8558                                 return -EINVAL;
8559                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8560                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
8561                 } else if (vsi->type == I40E_VSI_SRIOV) {
8562                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
8563                                 return -EINVAL;
8564                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
8565                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
8566                 } else {
8567                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
8568                 }
8569         }
8570         i40e_flush(hw);
8571
8572         return 0;
8573 }
8574
8575 /**
8576  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
8577  * @vsi: Pointer to VSI structure
8578  * @seed: Buffer to store the keys
8579  * @lut: Buffer to store the lookup table entries
8580  * @lut_size: Size of buffer to store the lookup table entries
8581  *
8582  * Returns 0 on success, negative on failure
8583  */
8584 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
8585                             u8 *lut, u16 lut_size)
8586 {
8587         struct i40e_pf *pf = vsi->back;
8588         struct i40e_hw *hw = &pf->hw;
8589         u16 i;
8590
8591         if (seed) {
8592                 u32 *seed_dw = (u32 *)seed;
8593
8594                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8595                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
8596         }
8597         if (lut) {
8598                 u32 *lut_dw = (u32 *)lut;
8599
8600                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
8601                         return -EINVAL;
8602                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8603                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
8604         }
8605
8606         return 0;
8607 }
8608
8609 /**
8610  * i40e_config_rss - Configure RSS keys and lut
8611  * @vsi: Pointer to VSI structure
8612  * @seed: RSS hash seed
8613  * @lut: Lookup table
8614  * @lut_size: Lookup table size
8615  *
8616  * Returns 0 on success, negative on failure
8617  */
8618 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8619 {
8620         struct i40e_pf *pf = vsi->back;
8621
8622         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
8623                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
8624         else
8625                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
8626 }
8627
8628 /**
8629  * i40e_get_rss - Get RSS keys and lut
8630  * @vsi: Pointer to VSI structure
8631  * @seed: Buffer to store the keys
8632  * @lut: Buffer to store the lookup table entries
8633  * lut_size: Size of buffer to store the lookup table entries
8634  *
8635  * Returns 0 on success, negative on failure
8636  */
8637 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8638 {
8639         struct i40e_pf *pf = vsi->back;
8640
8641         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
8642                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
8643         else
8644                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
8645 }
8646
8647 /**
8648  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
8649  * @pf: Pointer to board private structure
8650  * @lut: Lookup table
8651  * @rss_table_size: Lookup table size
8652  * @rss_size: Range of queue number for hashing
8653  */
8654 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
8655                        u16 rss_table_size, u16 rss_size)
8656 {
8657         u16 i;
8658
8659         for (i = 0; i < rss_table_size; i++)
8660                 lut[i] = i % rss_size;
8661 }
8662
8663 /**
8664  * i40e_pf_config_rss - Prepare for RSS if used
8665  * @pf: board private structure
8666  **/
8667 static int i40e_pf_config_rss(struct i40e_pf *pf)
8668 {
8669         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8670         u8 seed[I40E_HKEY_ARRAY_SIZE];
8671         u8 *lut;
8672         struct i40e_hw *hw = &pf->hw;
8673         u32 reg_val;
8674         u64 hena;
8675         int ret;
8676
8677         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
8678         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
8679                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
8680         hena |= i40e_pf_get_default_rss_hena(pf);
8681
8682         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
8683         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
8684
8685         /* Determine the RSS table size based on the hardware capabilities */
8686         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
8687         reg_val = (pf->rss_table_size == 512) ?
8688                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
8689                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
8690         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
8691
8692         /* Determine the RSS size of the VSI */
8693         if (!vsi->rss_size) {
8694                 u16 qcount;
8695
8696                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
8697                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
8698         }
8699         if (!vsi->rss_size)
8700                 return -EINVAL;
8701
8702         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8703         if (!lut)
8704                 return -ENOMEM;
8705
8706         /* Use user configured lut if there is one, otherwise use default */
8707         if (vsi->rss_lut_user)
8708                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8709         else
8710                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8711
8712         /* Use user configured hash key if there is one, otherwise
8713          * use default.
8714          */
8715         if (vsi->rss_hkey_user)
8716                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8717         else
8718                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8719         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
8720         kfree(lut);
8721
8722         return ret;
8723 }
8724
8725 /**
8726  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
8727  * @pf: board private structure
8728  * @queue_count: the requested queue count for rss.
8729  *
8730  * returns 0 if rss is not enabled, if enabled returns the final rss queue
8731  * count which may be different from the requested queue count.
8732  * Note: expects to be called while under rtnl_lock()
8733  **/
8734 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
8735 {
8736         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8737         int new_rss_size;
8738
8739         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
8740                 return 0;
8741
8742         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
8743
8744         if (queue_count != vsi->num_queue_pairs) {
8745                 u16 qcount;
8746
8747                 vsi->req_queue_pairs = queue_count;
8748                 i40e_prep_for_reset(pf, true);
8749
8750                 pf->alloc_rss_size = new_rss_size;
8751
8752                 i40e_reset_and_rebuild(pf, true, true);
8753
8754                 /* Discard the user configured hash keys and lut, if less
8755                  * queues are enabled.
8756                  */
8757                 if (queue_count < vsi->rss_size) {
8758                         i40e_clear_rss_config_user(vsi);
8759                         dev_dbg(&pf->pdev->dev,
8760                                 "discard user configured hash keys and lut\n");
8761                 }
8762
8763                 /* Reset vsi->rss_size, as number of enabled queues changed */
8764                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
8765                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
8766
8767                 i40e_pf_config_rss(pf);
8768         }
8769         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
8770                  vsi->req_queue_pairs, pf->rss_size_max);
8771         return pf->alloc_rss_size;
8772 }
8773
8774 /**
8775  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
8776  * @pf: board private structure
8777  **/
8778 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
8779 {
8780         i40e_status status;
8781         bool min_valid, max_valid;
8782         u32 max_bw, min_bw;
8783
8784         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
8785                                            &min_valid, &max_valid);
8786
8787         if (!status) {
8788                 if (min_valid)
8789                         pf->min_bw = min_bw;
8790                 if (max_valid)
8791                         pf->max_bw = max_bw;
8792         }
8793
8794         return status;
8795 }
8796
8797 /**
8798  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
8799  * @pf: board private structure
8800  **/
8801 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
8802 {
8803         struct i40e_aqc_configure_partition_bw_data bw_data;
8804         i40e_status status;
8805
8806         /* Set the valid bit for this PF */
8807         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
8808         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
8809         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
8810
8811         /* Set the new bandwidths */
8812         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
8813
8814         return status;
8815 }
8816
8817 /**
8818  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
8819  * @pf: board private structure
8820  **/
8821 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
8822 {
8823         /* Commit temporary BW setting to permanent NVM image */
8824         enum i40e_admin_queue_err last_aq_status;
8825         i40e_status ret;
8826         u16 nvm_word;
8827
8828         if (pf->hw.partition_id != 1) {
8829                 dev_info(&pf->pdev->dev,
8830                          "Commit BW only works on partition 1! This is partition %d",
8831                          pf->hw.partition_id);
8832                 ret = I40E_NOT_SUPPORTED;
8833                 goto bw_commit_out;
8834         }
8835
8836         /* Acquire NVM for read access */
8837         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
8838         last_aq_status = pf->hw.aq.asq_last_status;
8839         if (ret) {
8840                 dev_info(&pf->pdev->dev,
8841                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
8842                          i40e_stat_str(&pf->hw, ret),
8843                          i40e_aq_str(&pf->hw, last_aq_status));
8844                 goto bw_commit_out;
8845         }
8846
8847         /* Read word 0x10 of NVM - SW compatibility word 1 */
8848         ret = i40e_aq_read_nvm(&pf->hw,
8849                                I40E_SR_NVM_CONTROL_WORD,
8850                                0x10, sizeof(nvm_word), &nvm_word,
8851                                false, NULL);
8852         /* Save off last admin queue command status before releasing
8853          * the NVM
8854          */
8855         last_aq_status = pf->hw.aq.asq_last_status;
8856         i40e_release_nvm(&pf->hw);
8857         if (ret) {
8858                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
8859                          i40e_stat_str(&pf->hw, ret),
8860                          i40e_aq_str(&pf->hw, last_aq_status));
8861                 goto bw_commit_out;
8862         }
8863
8864         /* Wait a bit for NVM release to complete */
8865         msleep(50);
8866
8867         /* Acquire NVM for write access */
8868         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
8869         last_aq_status = pf->hw.aq.asq_last_status;
8870         if (ret) {
8871                 dev_info(&pf->pdev->dev,
8872                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
8873                          i40e_stat_str(&pf->hw, ret),
8874                          i40e_aq_str(&pf->hw, last_aq_status));
8875                 goto bw_commit_out;
8876         }
8877         /* Write it back out unchanged to initiate update NVM,
8878          * which will force a write of the shadow (alt) RAM to
8879          * the NVM - thus storing the bandwidth values permanently.
8880          */
8881         ret = i40e_aq_update_nvm(&pf->hw,
8882                                  I40E_SR_NVM_CONTROL_WORD,
8883                                  0x10, sizeof(nvm_word),
8884                                  &nvm_word, true, NULL);
8885         /* Save off last admin queue command status before releasing
8886          * the NVM
8887          */
8888         last_aq_status = pf->hw.aq.asq_last_status;
8889         i40e_release_nvm(&pf->hw);
8890         if (ret)
8891                 dev_info(&pf->pdev->dev,
8892                          "BW settings NOT SAVED, err %s aq_err %s\n",
8893                          i40e_stat_str(&pf->hw, ret),
8894                          i40e_aq_str(&pf->hw, last_aq_status));
8895 bw_commit_out:
8896
8897         return ret;
8898 }
8899
8900 /**
8901  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
8902  * @pf: board private structure to initialize
8903  *
8904  * i40e_sw_init initializes the Adapter private data structure.
8905  * Fields are initialized based on PCI device information and
8906  * OS network device settings (MTU size).
8907  **/
8908 static int i40e_sw_init(struct i40e_pf *pf)
8909 {
8910         int err = 0;
8911         int size;
8912
8913         /* Set default capability flags */
8914         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
8915                     I40E_FLAG_MSI_ENABLED     |
8916                     I40E_FLAG_MSIX_ENABLED;
8917
8918         /* Set default ITR */
8919         pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF;
8920         pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF;
8921
8922         /* Depending on PF configurations, it is possible that the RSS
8923          * maximum might end up larger than the available queues
8924          */
8925         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
8926         pf->alloc_rss_size = 1;
8927         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
8928         pf->rss_size_max = min_t(int, pf->rss_size_max,
8929                                  pf->hw.func_caps.num_tx_qp);
8930         if (pf->hw.func_caps.rss) {
8931                 pf->flags |= I40E_FLAG_RSS_ENABLED;
8932                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
8933                                            num_online_cpus());
8934         }
8935
8936         /* MFP mode enabled */
8937         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
8938                 pf->flags |= I40E_FLAG_MFP_ENABLED;
8939                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
8940                 if (i40e_get_partition_bw_setting(pf)) {
8941                         dev_warn(&pf->pdev->dev,
8942                                  "Could not get partition bw settings\n");
8943                 } else {
8944                         dev_info(&pf->pdev->dev,
8945                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
8946                                  pf->min_bw, pf->max_bw);
8947
8948                         /* nudge the Tx scheduler */
8949                         i40e_set_partition_bw_setting(pf);
8950                 }
8951         }
8952
8953         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
8954             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
8955                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
8956                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
8957                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
8958                     pf->hw.num_partitions > 1)
8959                         dev_info(&pf->pdev->dev,
8960                                  "Flow Director Sideband mode Disabled in MFP mode\n");
8961                 else
8962                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8963                 pf->fdir_pf_filter_count =
8964                                  pf->hw.func_caps.fd_filters_guaranteed;
8965                 pf->hw.fdir_shared_filter_count =
8966                                  pf->hw.func_caps.fd_filters_best_effort;
8967         }
8968
8969         if (pf->hw.mac.type == I40E_MAC_X722) {
8970                 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
8971                                     I40E_HW_128_QP_RSS_CAPABLE |
8972                                     I40E_HW_ATR_EVICT_CAPABLE |
8973                                     I40E_HW_WB_ON_ITR_CAPABLE |
8974                                     I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
8975                                     I40E_HW_NO_PCI_LINK_CHECK |
8976                                     I40E_HW_USE_SET_LLDP_MIB |
8977                                     I40E_HW_GENEVE_OFFLOAD_CAPABLE |
8978                                     I40E_HW_PTP_L4_CAPABLE |
8979                                     I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
8980                                     I40E_HW_OUTER_UDP_CSUM_CAPABLE);
8981
8982 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
8983                 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
8984                     I40E_FDEVICT_PCTYPE_DEFAULT) {
8985                         dev_warn(&pf->pdev->dev,
8986                                  "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
8987                         pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
8988                 }
8989         } else if ((pf->hw.aq.api_maj_ver > 1) ||
8990                    ((pf->hw.aq.api_maj_ver == 1) &&
8991                     (pf->hw.aq.api_min_ver > 4))) {
8992                 /* Supported in FW API version higher than 1.4 */
8993                 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
8994         }
8995
8996         /* Enable HW ATR eviction if possible */
8997         if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
8998                 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
8999
9000         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
9001             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
9002             (pf->hw.aq.fw_maj_ver < 4))) {
9003                 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
9004                 /* No DCB support  for FW < v4.33 */
9005                 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
9006         }
9007
9008         /* Disable FW LLDP if FW < v4.3 */
9009         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
9010             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
9011             (pf->hw.aq.fw_maj_ver < 4)))
9012                 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
9013
9014         /* Use the FW Set LLDP MIB API if FW > v4.40 */
9015         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
9016             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
9017             (pf->hw.aq.fw_maj_ver >= 5)))
9018                 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
9019
9020         if (pf->hw.func_caps.vmdq) {
9021                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
9022                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
9023                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
9024         }
9025
9026         if (pf->hw.func_caps.iwarp) {
9027                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
9028                 /* IWARP needs one extra vector for CQP just like MISC.*/
9029                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
9030         }
9031
9032 #ifdef CONFIG_PCI_IOV
9033         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
9034                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
9035                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
9036                 pf->num_req_vfs = min_t(int,
9037                                         pf->hw.func_caps.num_vfs,
9038                                         I40E_MAX_VF_COUNT);
9039         }
9040 #endif /* CONFIG_PCI_IOV */
9041         pf->eeprom_version = 0xDEAD;
9042         pf->lan_veb = I40E_NO_VEB;
9043         pf->lan_vsi = I40E_NO_VSI;
9044
9045         /* By default FW has this off for performance reasons */
9046         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
9047
9048         /* set up queue assignment tracking */
9049         size = sizeof(struct i40e_lump_tracking)
9050                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
9051         pf->qp_pile = kzalloc(size, GFP_KERNEL);
9052         if (!pf->qp_pile) {
9053                 err = -ENOMEM;
9054                 goto sw_init_done;
9055         }
9056         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
9057         pf->qp_pile->search_hint = 0;
9058
9059         pf->tx_timeout_recovery_level = 1;
9060
9061         mutex_init(&pf->switch_mutex);
9062
9063 sw_init_done:
9064         return err;
9065 }
9066
9067 /**
9068  * i40e_set_ntuple - set the ntuple feature flag and take action
9069  * @pf: board private structure to initialize
9070  * @features: the feature set that the stack is suggesting
9071  *
9072  * returns a bool to indicate if reset needs to happen
9073  **/
9074 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
9075 {
9076         bool need_reset = false;
9077
9078         /* Check if Flow Director n-tuple support was enabled or disabled.  If
9079          * the state changed, we need to reset.
9080          */
9081         if (features & NETIF_F_NTUPLE) {
9082                 /* Enable filters and mark for reset */
9083                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
9084                         need_reset = true;
9085                 /* enable FD_SB only if there is MSI-X vector */
9086                 if (pf->num_fdsb_msix > 0)
9087                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
9088         } else {
9089                 /* turn off filters, mark for reset and clear SW filter list */
9090                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
9091                         need_reset = true;
9092                         i40e_fdir_filter_exit(pf);
9093                 }
9094                 pf->flags &= ~(I40E_FLAG_FD_SB_ENABLED |
9095                                I40E_FLAG_FD_SB_AUTO_DISABLED);
9096                 /* reset fd counters */
9097                 pf->fd_add_err = 0;
9098                 pf->fd_atr_cnt = 0;
9099                 /* if ATR was auto disabled it can be re-enabled. */
9100                 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) {
9101                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
9102                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
9103                             (I40E_DEBUG_FD & pf->hw.debug_mask))
9104                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
9105                 }
9106         }
9107         return need_reset;
9108 }
9109
9110 /**
9111  * i40e_clear_rss_lut - clear the rx hash lookup table
9112  * @vsi: the VSI being configured
9113  **/
9114 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
9115 {
9116         struct i40e_pf *pf = vsi->back;
9117         struct i40e_hw *hw = &pf->hw;
9118         u16 vf_id = vsi->vf_id;
9119         u8 i;
9120
9121         if (vsi->type == I40E_VSI_MAIN) {
9122                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
9123                         wr32(hw, I40E_PFQF_HLUT(i), 0);
9124         } else if (vsi->type == I40E_VSI_SRIOV) {
9125                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
9126                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
9127         } else {
9128                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
9129         }
9130 }
9131
9132 /**
9133  * i40e_set_features - set the netdev feature flags
9134  * @netdev: ptr to the netdev being adjusted
9135  * @features: the feature set that the stack is suggesting
9136  * Note: expects to be called while under rtnl_lock()
9137  **/
9138 static int i40e_set_features(struct net_device *netdev,
9139                              netdev_features_t features)
9140 {
9141         struct i40e_netdev_priv *np = netdev_priv(netdev);
9142         struct i40e_vsi *vsi = np->vsi;
9143         struct i40e_pf *pf = vsi->back;
9144         bool need_reset;
9145
9146         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
9147                 i40e_pf_config_rss(pf);
9148         else if (!(features & NETIF_F_RXHASH) &&
9149                  netdev->features & NETIF_F_RXHASH)
9150                 i40e_clear_rss_lut(vsi);
9151
9152         if (features & NETIF_F_HW_VLAN_CTAG_RX)
9153                 i40e_vlan_stripping_enable(vsi);
9154         else
9155                 i40e_vlan_stripping_disable(vsi);
9156
9157         need_reset = i40e_set_ntuple(pf, features);
9158
9159         if (need_reset)
9160                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true);
9161
9162         return 0;
9163 }
9164
9165 /**
9166  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
9167  * @pf: board private structure
9168  * @port: The UDP port to look up
9169  *
9170  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
9171  **/
9172 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
9173 {
9174         u8 i;
9175
9176         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
9177                 if (pf->udp_ports[i].port == port)
9178                         return i;
9179         }
9180
9181         return i;
9182 }
9183
9184 /**
9185  * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
9186  * @netdev: This physical port's netdev
9187  * @ti: Tunnel endpoint information
9188  **/
9189 static void i40e_udp_tunnel_add(struct net_device *netdev,
9190                                 struct udp_tunnel_info *ti)
9191 {
9192         struct i40e_netdev_priv *np = netdev_priv(netdev);
9193         struct i40e_vsi *vsi = np->vsi;
9194         struct i40e_pf *pf = vsi->back;
9195         u16 port = ntohs(ti->port);
9196         u8 next_idx;
9197         u8 idx;
9198
9199         idx = i40e_get_udp_port_idx(pf, port);
9200
9201         /* Check if port already exists */
9202         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
9203                 netdev_info(netdev, "port %d already offloaded\n", port);
9204                 return;
9205         }
9206
9207         /* Now check if there is space to add the new port */
9208         next_idx = i40e_get_udp_port_idx(pf, 0);
9209
9210         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
9211                 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
9212                             port);
9213                 return;
9214         }
9215
9216         switch (ti->type) {
9217         case UDP_TUNNEL_TYPE_VXLAN:
9218                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
9219                 break;
9220         case UDP_TUNNEL_TYPE_GENEVE:
9221                 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE))
9222                         return;
9223                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
9224                 break;
9225         default:
9226                 return;
9227         }
9228
9229         /* New port: add it and mark its index in the bitmap */
9230         pf->udp_ports[next_idx].port = port;
9231         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
9232         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
9233 }
9234
9235 /**
9236  * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
9237  * @netdev: This physical port's netdev
9238  * @ti: Tunnel endpoint information
9239  **/
9240 static void i40e_udp_tunnel_del(struct net_device *netdev,
9241                                 struct udp_tunnel_info *ti)
9242 {
9243         struct i40e_netdev_priv *np = netdev_priv(netdev);
9244         struct i40e_vsi *vsi = np->vsi;
9245         struct i40e_pf *pf = vsi->back;
9246         u16 port = ntohs(ti->port);
9247         u8 idx;
9248
9249         idx = i40e_get_udp_port_idx(pf, port);
9250
9251         /* Check if port already exists */
9252         if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
9253                 goto not_found;
9254
9255         switch (ti->type) {
9256         case UDP_TUNNEL_TYPE_VXLAN:
9257                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
9258                         goto not_found;
9259                 break;
9260         case UDP_TUNNEL_TYPE_GENEVE:
9261                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
9262                         goto not_found;
9263                 break;
9264         default:
9265                 goto not_found;
9266         }
9267
9268         /* if port exists, set it to 0 (mark for deletion)
9269          * and make it pending
9270          */
9271         pf->udp_ports[idx].port = 0;
9272         pf->pending_udp_bitmap |= BIT_ULL(idx);
9273         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
9274
9275         return;
9276 not_found:
9277         netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
9278                     port);
9279 }
9280
9281 static int i40e_get_phys_port_id(struct net_device *netdev,
9282                                  struct netdev_phys_item_id *ppid)
9283 {
9284         struct i40e_netdev_priv *np = netdev_priv(netdev);
9285         struct i40e_pf *pf = np->vsi->back;
9286         struct i40e_hw *hw = &pf->hw;
9287
9288         if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
9289                 return -EOPNOTSUPP;
9290
9291         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
9292         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
9293
9294         return 0;
9295 }
9296
9297 /**
9298  * i40e_ndo_fdb_add - add an entry to the hardware database
9299  * @ndm: the input from the stack
9300  * @tb: pointer to array of nladdr (unused)
9301  * @dev: the net device pointer
9302  * @addr: the MAC address entry being added
9303  * @flags: instructions from stack about fdb operation
9304  */
9305 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
9306                             struct net_device *dev,
9307                             const unsigned char *addr, u16 vid,
9308                             u16 flags)
9309 {
9310         struct i40e_netdev_priv *np = netdev_priv(dev);
9311         struct i40e_pf *pf = np->vsi->back;
9312         int err = 0;
9313
9314         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
9315                 return -EOPNOTSUPP;
9316
9317         if (vid) {
9318                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
9319                 return -EINVAL;
9320         }
9321
9322         /* Hardware does not support aging addresses so if a
9323          * ndm_state is given only allow permanent addresses
9324          */
9325         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
9326                 netdev_info(dev, "FDB only supports static addresses\n");
9327                 return -EINVAL;
9328         }
9329
9330         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
9331                 err = dev_uc_add_excl(dev, addr);
9332         else if (is_multicast_ether_addr(addr))
9333                 err = dev_mc_add_excl(dev, addr);
9334         else
9335                 err = -EINVAL;
9336
9337         /* Only return duplicate errors if NLM_F_EXCL is set */
9338         if (err == -EEXIST && !(flags & NLM_F_EXCL))
9339                 err = 0;
9340
9341         return err;
9342 }
9343
9344 /**
9345  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
9346  * @dev: the netdev being configured
9347  * @nlh: RTNL message
9348  *
9349  * Inserts a new hardware bridge if not already created and
9350  * enables the bridging mode requested (VEB or VEPA). If the
9351  * hardware bridge has already been inserted and the request
9352  * is to change the mode then that requires a PF reset to
9353  * allow rebuild of the components with required hardware
9354  * bridge mode enabled.
9355  *
9356  * Note: expects to be called while under rtnl_lock()
9357  **/
9358 static int i40e_ndo_bridge_setlink(struct net_device *dev,
9359                                    struct nlmsghdr *nlh,
9360                                    u16 flags)
9361 {
9362         struct i40e_netdev_priv *np = netdev_priv(dev);
9363         struct i40e_vsi *vsi = np->vsi;
9364         struct i40e_pf *pf = vsi->back;
9365         struct i40e_veb *veb = NULL;
9366         struct nlattr *attr, *br_spec;
9367         int i, rem;
9368
9369         /* Only for PF VSI for now */
9370         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9371                 return -EOPNOTSUPP;
9372
9373         /* Find the HW bridge for PF VSI */
9374         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9375                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9376                         veb = pf->veb[i];
9377         }
9378
9379         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
9380
9381         nla_for_each_nested(attr, br_spec, rem) {
9382                 __u16 mode;
9383
9384                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
9385                         continue;
9386
9387                 mode = nla_get_u16(attr);
9388                 if ((mode != BRIDGE_MODE_VEPA) &&
9389                     (mode != BRIDGE_MODE_VEB))
9390                         return -EINVAL;
9391
9392                 /* Insert a new HW bridge */
9393                 if (!veb) {
9394                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
9395                                              vsi->tc_config.enabled_tc);
9396                         if (veb) {
9397                                 veb->bridge_mode = mode;
9398                                 i40e_config_bridge_mode(veb);
9399                         } else {
9400                                 /* No Bridge HW offload available */
9401                                 return -ENOENT;
9402                         }
9403                         break;
9404                 } else if (mode != veb->bridge_mode) {
9405                         /* Existing HW bridge but different mode needs reset */
9406                         veb->bridge_mode = mode;
9407                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
9408                         if (mode == BRIDGE_MODE_VEB)
9409                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
9410                         else
9411                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
9412                         i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED),
9413                                       true);
9414                         break;
9415                 }
9416         }
9417
9418         return 0;
9419 }
9420
9421 /**
9422  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
9423  * @skb: skb buff
9424  * @pid: process id
9425  * @seq: RTNL message seq #
9426  * @dev: the netdev being configured
9427  * @filter_mask: unused
9428  * @nlflags: netlink flags passed in
9429  *
9430  * Return the mode in which the hardware bridge is operating in
9431  * i.e VEB or VEPA.
9432  **/
9433 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
9434                                    struct net_device *dev,
9435                                    u32 __always_unused filter_mask,
9436                                    int nlflags)
9437 {
9438         struct i40e_netdev_priv *np = netdev_priv(dev);
9439         struct i40e_vsi *vsi = np->vsi;
9440         struct i40e_pf *pf = vsi->back;
9441         struct i40e_veb *veb = NULL;
9442         int i;
9443
9444         /* Only for PF VSI for now */
9445         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9446                 return -EOPNOTSUPP;
9447
9448         /* Find the HW bridge for the PF VSI */
9449         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9450                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9451                         veb = pf->veb[i];
9452         }
9453
9454         if (!veb)
9455                 return 0;
9456
9457         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
9458                                        0, 0, nlflags, filter_mask, NULL);
9459 }
9460
9461 /**
9462  * i40e_features_check - Validate encapsulated packet conforms to limits
9463  * @skb: skb buff
9464  * @dev: This physical port's netdev
9465  * @features: Offload features that the stack believes apply
9466  **/
9467 static netdev_features_t i40e_features_check(struct sk_buff *skb,
9468                                              struct net_device *dev,
9469                                              netdev_features_t features)
9470 {
9471         size_t len;
9472
9473         /* No point in doing any of this if neither checksum nor GSO are
9474          * being requested for this frame.  We can rule out both by just
9475          * checking for CHECKSUM_PARTIAL
9476          */
9477         if (skb->ip_summed != CHECKSUM_PARTIAL)
9478                 return features;
9479
9480         /* We cannot support GSO if the MSS is going to be less than
9481          * 64 bytes.  If it is then we need to drop support for GSO.
9482          */
9483         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
9484                 features &= ~NETIF_F_GSO_MASK;
9485
9486         /* MACLEN can support at most 63 words */
9487         len = skb_network_header(skb) - skb->data;
9488         if (len & ~(63 * 2))
9489                 goto out_err;
9490
9491         /* IPLEN and EIPLEN can support at most 127 dwords */
9492         len = skb_transport_header(skb) - skb_network_header(skb);
9493         if (len & ~(127 * 4))
9494                 goto out_err;
9495
9496         if (skb->encapsulation) {
9497                 /* L4TUNLEN can support 127 words */
9498                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
9499                 if (len & ~(127 * 2))
9500                         goto out_err;
9501
9502                 /* IPLEN can support at most 127 dwords */
9503                 len = skb_inner_transport_header(skb) -
9504                       skb_inner_network_header(skb);
9505                 if (len & ~(127 * 4))
9506                         goto out_err;
9507         }
9508
9509         /* No need to validate L4LEN as TCP is the only protocol with a
9510          * a flexible value and we support all possible values supported
9511          * by TCP, which is at most 15 dwords
9512          */
9513
9514         return features;
9515 out_err:
9516         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
9517 }
9518
9519 /**
9520  * i40e_xdp_setup - add/remove an XDP program
9521  * @vsi: VSI to changed
9522  * @prog: XDP program
9523  **/
9524 static int i40e_xdp_setup(struct i40e_vsi *vsi,
9525                           struct bpf_prog *prog)
9526 {
9527         int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9528         struct i40e_pf *pf = vsi->back;
9529         struct bpf_prog *old_prog;
9530         bool need_reset;
9531         int i;
9532
9533         /* Don't allow frames that span over multiple buffers */
9534         if (frame_size > vsi->rx_buf_len)
9535                 return -EINVAL;
9536
9537         if (!i40e_enabled_xdp_vsi(vsi) && !prog)
9538                 return 0;
9539
9540         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
9541         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
9542
9543         if (need_reset)
9544                 i40e_prep_for_reset(pf, true);
9545
9546         old_prog = xchg(&vsi->xdp_prog, prog);
9547
9548         if (need_reset)
9549                 i40e_reset_and_rebuild(pf, true, true);
9550
9551         for (i = 0; i < vsi->num_queue_pairs; i++)
9552                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
9553
9554         if (old_prog)
9555                 bpf_prog_put(old_prog);
9556
9557         return 0;
9558 }
9559
9560 /**
9561  * i40e_xdp - implements ndo_xdp for i40e
9562  * @dev: netdevice
9563  * @xdp: XDP command
9564  **/
9565 static int i40e_xdp(struct net_device *dev,
9566                     struct netdev_xdp *xdp)
9567 {
9568         struct i40e_netdev_priv *np = netdev_priv(dev);
9569         struct i40e_vsi *vsi = np->vsi;
9570
9571         if (vsi->type != I40E_VSI_MAIN)
9572                 return -EINVAL;
9573
9574         switch (xdp->command) {
9575         case XDP_SETUP_PROG:
9576                 return i40e_xdp_setup(vsi, xdp->prog);
9577         case XDP_QUERY_PROG:
9578                 xdp->prog_attached = i40e_enabled_xdp_vsi(vsi);
9579                 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
9580                 return 0;
9581         default:
9582                 return -EINVAL;
9583         }
9584 }
9585
9586 static const struct net_device_ops i40e_netdev_ops = {
9587         .ndo_open               = i40e_open,
9588         .ndo_stop               = i40e_close,
9589         .ndo_start_xmit         = i40e_lan_xmit_frame,
9590         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
9591         .ndo_set_rx_mode        = i40e_set_rx_mode,
9592         .ndo_validate_addr      = eth_validate_addr,
9593         .ndo_set_mac_address    = i40e_set_mac,
9594         .ndo_change_mtu         = i40e_change_mtu,
9595         .ndo_do_ioctl           = i40e_ioctl,
9596         .ndo_tx_timeout         = i40e_tx_timeout,
9597         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
9598         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
9599 #ifdef CONFIG_NET_POLL_CONTROLLER
9600         .ndo_poll_controller    = i40e_netpoll,
9601 #endif
9602         .ndo_setup_tc           = __i40e_setup_tc,
9603         .ndo_set_features       = i40e_set_features,
9604         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
9605         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
9606         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
9607         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
9608         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
9609         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
9610         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
9611         .ndo_udp_tunnel_add     = i40e_udp_tunnel_add,
9612         .ndo_udp_tunnel_del     = i40e_udp_tunnel_del,
9613         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
9614         .ndo_fdb_add            = i40e_ndo_fdb_add,
9615         .ndo_features_check     = i40e_features_check,
9616         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
9617         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
9618         .ndo_xdp                = i40e_xdp,
9619 };
9620
9621 /**
9622  * i40e_config_netdev - Setup the netdev flags
9623  * @vsi: the VSI being configured
9624  *
9625  * Returns 0 on success, negative value on failure
9626  **/
9627 static int i40e_config_netdev(struct i40e_vsi *vsi)
9628 {
9629         struct i40e_pf *pf = vsi->back;
9630         struct i40e_hw *hw = &pf->hw;
9631         struct i40e_netdev_priv *np;
9632         struct net_device *netdev;
9633         u8 broadcast[ETH_ALEN];
9634         u8 mac_addr[ETH_ALEN];
9635         int etherdev_size;
9636         netdev_features_t hw_enc_features;
9637         netdev_features_t hw_features;
9638
9639         etherdev_size = sizeof(struct i40e_netdev_priv);
9640         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
9641         if (!netdev)
9642                 return -ENOMEM;
9643
9644         vsi->netdev = netdev;
9645         np = netdev_priv(netdev);
9646         np->vsi = vsi;
9647
9648         hw_enc_features = NETIF_F_SG                    |
9649                           NETIF_F_IP_CSUM               |
9650                           NETIF_F_IPV6_CSUM             |
9651                           NETIF_F_HIGHDMA               |
9652                           NETIF_F_SOFT_FEATURES         |
9653                           NETIF_F_TSO                   |
9654                           NETIF_F_TSO_ECN               |
9655                           NETIF_F_TSO6                  |
9656                           NETIF_F_GSO_GRE               |
9657                           NETIF_F_GSO_GRE_CSUM          |
9658                           NETIF_F_GSO_PARTIAL           |
9659                           NETIF_F_GSO_UDP_TUNNEL        |
9660                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
9661                           NETIF_F_SCTP_CRC              |
9662                           NETIF_F_RXHASH                |
9663                           NETIF_F_RXCSUM                |
9664                           0;
9665
9666         if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
9667                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
9668
9669         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
9670
9671         netdev->hw_enc_features |= hw_enc_features;
9672
9673         /* record features VLANs can make use of */
9674         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
9675
9676         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
9677                 netdev->hw_features |= NETIF_F_NTUPLE;
9678         hw_features = hw_enc_features           |
9679                       NETIF_F_HW_VLAN_CTAG_TX   |
9680                       NETIF_F_HW_VLAN_CTAG_RX;
9681
9682         netdev->hw_features |= hw_features;
9683
9684         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
9685         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
9686
9687         if (vsi->type == I40E_VSI_MAIN) {
9688                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
9689                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
9690                 /* The following steps are necessary for two reasons. First,
9691                  * some older NVM configurations load a default MAC-VLAN
9692                  * filter that will accept any tagged packet, and we want to
9693                  * replace this with a normal filter. Additionally, it is
9694                  * possible our MAC address was provided by the platform using
9695                  * Open Firmware or similar.
9696                  *
9697                  * Thus, we need to remove the default filter and install one
9698                  * specific to the MAC address.
9699                  */
9700                 i40e_rm_default_mac_filter(vsi, mac_addr);
9701                 spin_lock_bh(&vsi->mac_filter_hash_lock);
9702                 i40e_add_mac_filter(vsi, mac_addr);
9703                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9704         } else {
9705                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
9706                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
9707                  * the end, which is 4 bytes long, so force truncation of the
9708                  * original name by IFNAMSIZ - 4
9709                  */
9710                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
9711                          IFNAMSIZ - 4,
9712                          pf->vsi[pf->lan_vsi]->netdev->name);
9713                 random_ether_addr(mac_addr);
9714
9715                 spin_lock_bh(&vsi->mac_filter_hash_lock);
9716                 i40e_add_mac_filter(vsi, mac_addr);
9717                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9718         }
9719
9720         /* Add the broadcast filter so that we initially will receive
9721          * broadcast packets. Note that when a new VLAN is first added the
9722          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
9723          * specific filters as part of transitioning into "vlan" operation.
9724          * When more VLANs are added, the driver will copy each existing MAC
9725          * filter and add it for the new VLAN.
9726          *
9727          * Broadcast filters are handled specially by
9728          * i40e_sync_filters_subtask, as the driver must to set the broadcast
9729          * promiscuous bit instead of adding this directly as a MAC/VLAN
9730          * filter. The subtask will update the correct broadcast promiscuous
9731          * bits as VLANs become active or inactive.
9732          */
9733         eth_broadcast_addr(broadcast);
9734         spin_lock_bh(&vsi->mac_filter_hash_lock);
9735         i40e_add_mac_filter(vsi, broadcast);
9736         spin_unlock_bh(&vsi->mac_filter_hash_lock);
9737
9738         ether_addr_copy(netdev->dev_addr, mac_addr);
9739         ether_addr_copy(netdev->perm_addr, mac_addr);
9740
9741         netdev->priv_flags |= IFF_UNICAST_FLT;
9742         netdev->priv_flags |= IFF_SUPP_NOFCS;
9743         /* Setup netdev TC information */
9744         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
9745
9746         netdev->netdev_ops = &i40e_netdev_ops;
9747         netdev->watchdog_timeo = 5 * HZ;
9748         i40e_set_ethtool_ops(netdev);
9749
9750         /* MTU range: 68 - 9706 */
9751         netdev->min_mtu = ETH_MIN_MTU;
9752         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
9753
9754         return 0;
9755 }
9756
9757 /**
9758  * i40e_vsi_delete - Delete a VSI from the switch
9759  * @vsi: the VSI being removed
9760  *
9761  * Returns 0 on success, negative value on failure
9762  **/
9763 static void i40e_vsi_delete(struct i40e_vsi *vsi)
9764 {
9765         /* remove default VSI is not allowed */
9766         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
9767                 return;
9768
9769         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
9770 }
9771
9772 /**
9773  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
9774  * @vsi: the VSI being queried
9775  *
9776  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
9777  **/
9778 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
9779 {
9780         struct i40e_veb *veb;
9781         struct i40e_pf *pf = vsi->back;
9782
9783         /* Uplink is not a bridge so default to VEB */
9784         if (vsi->veb_idx == I40E_NO_VEB)
9785                 return 1;
9786
9787         veb = pf->veb[vsi->veb_idx];
9788         if (!veb) {
9789                 dev_info(&pf->pdev->dev,
9790                          "There is no veb associated with the bridge\n");
9791                 return -ENOENT;
9792         }
9793
9794         /* Uplink is a bridge in VEPA mode */
9795         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
9796                 return 0;
9797         } else {
9798                 /* Uplink is a bridge in VEB mode */
9799                 return 1;
9800         }
9801
9802         /* VEPA is now default bridge, so return 0 */
9803         return 0;
9804 }
9805
9806 /**
9807  * i40e_add_vsi - Add a VSI to the switch
9808  * @vsi: the VSI being configured
9809  *
9810  * This initializes a VSI context depending on the VSI type to be added and
9811  * passes it down to the add_vsi aq command.
9812  **/
9813 static int i40e_add_vsi(struct i40e_vsi *vsi)
9814 {
9815         int ret = -ENODEV;
9816         struct i40e_pf *pf = vsi->back;
9817         struct i40e_hw *hw = &pf->hw;
9818         struct i40e_vsi_context ctxt;
9819         struct i40e_mac_filter *f;
9820         struct hlist_node *h;
9821         int bkt;
9822
9823         u8 enabled_tc = 0x1; /* TC0 enabled */
9824         int f_count = 0;
9825
9826         memset(&ctxt, 0, sizeof(ctxt));
9827         switch (vsi->type) {
9828         case I40E_VSI_MAIN:
9829                 /* The PF's main VSI is already setup as part of the
9830                  * device initialization, so we'll not bother with
9831                  * the add_vsi call, but we will retrieve the current
9832                  * VSI context.
9833                  */
9834                 ctxt.seid = pf->main_vsi_seid;
9835                 ctxt.pf_num = pf->hw.pf_id;
9836                 ctxt.vf_num = 0;
9837                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9838                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9839                 if (ret) {
9840                         dev_info(&pf->pdev->dev,
9841                                  "couldn't get PF vsi config, err %s aq_err %s\n",
9842                                  i40e_stat_str(&pf->hw, ret),
9843                                  i40e_aq_str(&pf->hw,
9844                                              pf->hw.aq.asq_last_status));
9845                         return -ENOENT;
9846                 }
9847                 vsi->info = ctxt.info;
9848                 vsi->info.valid_sections = 0;
9849
9850                 vsi->seid = ctxt.seid;
9851                 vsi->id = ctxt.vsi_number;
9852
9853                 enabled_tc = i40e_pf_get_tc_map(pf);
9854
9855                 /* MFP mode setup queue map and update VSI */
9856                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
9857                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
9858                         memset(&ctxt, 0, sizeof(ctxt));
9859                         ctxt.seid = pf->main_vsi_seid;
9860                         ctxt.pf_num = pf->hw.pf_id;
9861                         ctxt.vf_num = 0;
9862                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
9863                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
9864                         if (ret) {
9865                                 dev_info(&pf->pdev->dev,
9866                                          "update vsi failed, err %s aq_err %s\n",
9867                                          i40e_stat_str(&pf->hw, ret),
9868                                          i40e_aq_str(&pf->hw,
9869                                                     pf->hw.aq.asq_last_status));
9870                                 ret = -ENOENT;
9871                                 goto err;
9872                         }
9873                         /* update the local VSI info queue map */
9874                         i40e_vsi_update_queue_map(vsi, &ctxt);
9875                         vsi->info.valid_sections = 0;
9876                 } else {
9877                         /* Default/Main VSI is only enabled for TC0
9878                          * reconfigure it to enable all TCs that are
9879                          * available on the port in SFP mode.
9880                          * For MFP case the iSCSI PF would use this
9881                          * flow to enable LAN+iSCSI TC.
9882                          */
9883                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
9884                         if (ret) {
9885                                 /* Single TC condition is not fatal,
9886                                  * message and continue
9887                                  */
9888                                 dev_info(&pf->pdev->dev,
9889                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
9890                                          enabled_tc,
9891                                          i40e_stat_str(&pf->hw, ret),
9892                                          i40e_aq_str(&pf->hw,
9893                                                     pf->hw.aq.asq_last_status));
9894                         }
9895                 }
9896                 break;
9897
9898         case I40E_VSI_FDIR:
9899                 ctxt.pf_num = hw->pf_id;
9900                 ctxt.vf_num = 0;
9901                 ctxt.uplink_seid = vsi->uplink_seid;
9902                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9903                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9904                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
9905                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
9906                         ctxt.info.valid_sections |=
9907                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9908                         ctxt.info.switch_id =
9909                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9910                 }
9911                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9912                 break;
9913
9914         case I40E_VSI_VMDQ2:
9915                 ctxt.pf_num = hw->pf_id;
9916                 ctxt.vf_num = 0;
9917                 ctxt.uplink_seid = vsi->uplink_seid;
9918                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9919                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
9920
9921                 /* This VSI is connected to VEB so the switch_id
9922                  * should be set to zero by default.
9923                  */
9924                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9925                         ctxt.info.valid_sections |=
9926                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9927                         ctxt.info.switch_id =
9928                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9929                 }
9930
9931                 /* Setup the VSI tx/rx queue map for TC0 only for now */
9932                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9933                 break;
9934
9935         case I40E_VSI_SRIOV:
9936                 ctxt.pf_num = hw->pf_id;
9937                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
9938                 ctxt.uplink_seid = vsi->uplink_seid;
9939                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9940                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
9941
9942                 /* This VSI is connected to VEB so the switch_id
9943                  * should be set to zero by default.
9944                  */
9945                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9946                         ctxt.info.valid_sections |=
9947                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9948                         ctxt.info.switch_id =
9949                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9950                 }
9951
9952                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
9953                         ctxt.info.valid_sections |=
9954                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
9955                         ctxt.info.queueing_opt_flags |=
9956                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
9957                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
9958                 }
9959
9960                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
9961                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
9962                 if (pf->vf[vsi->vf_id].spoofchk) {
9963                         ctxt.info.valid_sections |=
9964                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
9965                         ctxt.info.sec_flags |=
9966                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
9967                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
9968                 }
9969                 /* Setup the VSI tx/rx queue map for TC0 only for now */
9970                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9971                 break;
9972
9973         case I40E_VSI_IWARP:
9974                 /* send down message to iWARP */
9975                 break;
9976
9977         default:
9978                 return -ENODEV;
9979         }
9980
9981         if (vsi->type != I40E_VSI_MAIN) {
9982                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
9983                 if (ret) {
9984                         dev_info(&vsi->back->pdev->dev,
9985                                  "add vsi failed, err %s aq_err %s\n",
9986                                  i40e_stat_str(&pf->hw, ret),
9987                                  i40e_aq_str(&pf->hw,
9988                                              pf->hw.aq.asq_last_status));
9989                         ret = -ENOENT;
9990                         goto err;
9991                 }
9992                 vsi->info = ctxt.info;
9993                 vsi->info.valid_sections = 0;
9994                 vsi->seid = ctxt.seid;
9995                 vsi->id = ctxt.vsi_number;
9996         }
9997
9998         vsi->active_filters = 0;
9999         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
10000         spin_lock_bh(&vsi->mac_filter_hash_lock);
10001         /* If macvlan filters already exist, force them to get loaded */
10002         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
10003                 f->state = I40E_FILTER_NEW;
10004                 f_count++;
10005         }
10006         spin_unlock_bh(&vsi->mac_filter_hash_lock);
10007
10008         if (f_count) {
10009                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
10010                 pf->flags |= I40E_FLAG_FILTER_SYNC;
10011         }
10012
10013         /* Update VSI BW information */
10014         ret = i40e_vsi_get_bw_info(vsi);
10015         if (ret) {
10016                 dev_info(&pf->pdev->dev,
10017                          "couldn't get vsi bw info, err %s aq_err %s\n",
10018                          i40e_stat_str(&pf->hw, ret),
10019                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10020                 /* VSI is already added so not tearing that up */
10021                 ret = 0;
10022         }
10023
10024 err:
10025         return ret;
10026 }
10027
10028 /**
10029  * i40e_vsi_release - Delete a VSI and free its resources
10030  * @vsi: the VSI being removed
10031  *
10032  * Returns 0 on success or < 0 on error
10033  **/
10034 int i40e_vsi_release(struct i40e_vsi *vsi)
10035 {
10036         struct i40e_mac_filter *f;
10037         struct hlist_node *h;
10038         struct i40e_veb *veb = NULL;
10039         struct i40e_pf *pf;
10040         u16 uplink_seid;
10041         int i, n, bkt;
10042
10043         pf = vsi->back;
10044
10045         /* release of a VEB-owner or last VSI is not allowed */
10046         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
10047                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
10048                          vsi->seid, vsi->uplink_seid);
10049                 return -ENODEV;
10050         }
10051         if (vsi == pf->vsi[pf->lan_vsi] &&
10052             !test_bit(__I40E_DOWN, pf->state)) {
10053                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
10054                 return -ENODEV;
10055         }
10056
10057         uplink_seid = vsi->uplink_seid;
10058         if (vsi->type != I40E_VSI_SRIOV) {
10059                 if (vsi->netdev_registered) {
10060                         vsi->netdev_registered = false;
10061                         if (vsi->netdev) {
10062                                 /* results in a call to i40e_close() */
10063                                 unregister_netdev(vsi->netdev);
10064                         }
10065                 } else {
10066                         i40e_vsi_close(vsi);
10067                 }
10068                 i40e_vsi_disable_irq(vsi);
10069         }
10070
10071         spin_lock_bh(&vsi->mac_filter_hash_lock);
10072
10073         /* clear the sync flag on all filters */
10074         if (vsi->netdev) {
10075                 __dev_uc_unsync(vsi->netdev, NULL);
10076                 __dev_mc_unsync(vsi->netdev, NULL);
10077         }
10078
10079         /* make sure any remaining filters are marked for deletion */
10080         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
10081                 __i40e_del_filter(vsi, f);
10082
10083         spin_unlock_bh(&vsi->mac_filter_hash_lock);
10084
10085         i40e_sync_vsi_filters(vsi);
10086
10087         i40e_vsi_delete(vsi);
10088         i40e_vsi_free_q_vectors(vsi);
10089         if (vsi->netdev) {
10090                 free_netdev(vsi->netdev);
10091                 vsi->netdev = NULL;
10092         }
10093         i40e_vsi_clear_rings(vsi);
10094         i40e_vsi_clear(vsi);
10095
10096         /* If this was the last thing on the VEB, except for the
10097          * controlling VSI, remove the VEB, which puts the controlling
10098          * VSI onto the next level down in the switch.
10099          *
10100          * Well, okay, there's one more exception here: don't remove
10101          * the orphan VEBs yet.  We'll wait for an explicit remove request
10102          * from up the network stack.
10103          */
10104         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
10105                 if (pf->vsi[i] &&
10106                     pf->vsi[i]->uplink_seid == uplink_seid &&
10107                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10108                         n++;      /* count the VSIs */
10109                 }
10110         }
10111         for (i = 0; i < I40E_MAX_VEB; i++) {
10112                 if (!pf->veb[i])
10113                         continue;
10114                 if (pf->veb[i]->uplink_seid == uplink_seid)
10115                         n++;     /* count the VEBs */
10116                 if (pf->veb[i]->seid == uplink_seid)
10117                         veb = pf->veb[i];
10118         }
10119         if (n == 0 && veb && veb->uplink_seid != 0)
10120                 i40e_veb_release(veb);
10121
10122         return 0;
10123 }
10124
10125 /**
10126  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
10127  * @vsi: ptr to the VSI
10128  *
10129  * This should only be called after i40e_vsi_mem_alloc() which allocates the
10130  * corresponding SW VSI structure and initializes num_queue_pairs for the
10131  * newly allocated VSI.
10132  *
10133  * Returns 0 on success or negative on failure
10134  **/
10135 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
10136 {
10137         int ret = -ENOENT;
10138         struct i40e_pf *pf = vsi->back;
10139
10140         if (vsi->q_vectors[0]) {
10141                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
10142                          vsi->seid);
10143                 return -EEXIST;
10144         }
10145
10146         if (vsi->base_vector) {
10147                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
10148                          vsi->seid, vsi->base_vector);
10149                 return -EEXIST;
10150         }
10151
10152         ret = i40e_vsi_alloc_q_vectors(vsi);
10153         if (ret) {
10154                 dev_info(&pf->pdev->dev,
10155                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
10156                          vsi->num_q_vectors, vsi->seid, ret);
10157                 vsi->num_q_vectors = 0;
10158                 goto vector_setup_out;
10159         }
10160
10161         /* In Legacy mode, we do not have to get any other vector since we
10162          * piggyback on the misc/ICR0 for queue interrupts.
10163         */
10164         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
10165                 return ret;
10166         if (vsi->num_q_vectors)
10167                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
10168                                                  vsi->num_q_vectors, vsi->idx);
10169         if (vsi->base_vector < 0) {
10170                 dev_info(&pf->pdev->dev,
10171                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
10172                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
10173                 i40e_vsi_free_q_vectors(vsi);
10174                 ret = -ENOENT;
10175                 goto vector_setup_out;
10176         }
10177
10178 vector_setup_out:
10179         return ret;
10180 }
10181
10182 /**
10183  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
10184  * @vsi: pointer to the vsi.
10185  *
10186  * This re-allocates a vsi's queue resources.
10187  *
10188  * Returns pointer to the successfully allocated and configured VSI sw struct
10189  * on success, otherwise returns NULL on failure.
10190  **/
10191 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
10192 {
10193         u16 alloc_queue_pairs;
10194         struct i40e_pf *pf;
10195         u8 enabled_tc;
10196         int ret;
10197
10198         if (!vsi)
10199                 return NULL;
10200
10201         pf = vsi->back;
10202
10203         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
10204         i40e_vsi_clear_rings(vsi);
10205
10206         i40e_vsi_free_arrays(vsi, false);
10207         i40e_set_num_rings_in_vsi(vsi);
10208         ret = i40e_vsi_alloc_arrays(vsi, false);
10209         if (ret)
10210                 goto err_vsi;
10211
10212         alloc_queue_pairs = vsi->alloc_queue_pairs *
10213                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
10214
10215         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
10216         if (ret < 0) {
10217                 dev_info(&pf->pdev->dev,
10218                          "failed to get tracking for %d queues for VSI %d err %d\n",
10219                          alloc_queue_pairs, vsi->seid, ret);
10220                 goto err_vsi;
10221         }
10222         vsi->base_queue = ret;
10223
10224         /* Update the FW view of the VSI. Force a reset of TC and queue
10225          * layout configurations.
10226          */
10227         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
10228         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
10229         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
10230         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
10231         if (vsi->type == I40E_VSI_MAIN)
10232                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
10233
10234         /* assign it some queues */
10235         ret = i40e_alloc_rings(vsi);
10236         if (ret)
10237                 goto err_rings;
10238
10239         /* map all of the rings to the q_vectors */
10240         i40e_vsi_map_rings_to_vectors(vsi);
10241         return vsi;
10242
10243 err_rings:
10244         i40e_vsi_free_q_vectors(vsi);
10245         if (vsi->netdev_registered) {
10246                 vsi->netdev_registered = false;
10247                 unregister_netdev(vsi->netdev);
10248                 free_netdev(vsi->netdev);
10249                 vsi->netdev = NULL;
10250         }
10251         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
10252 err_vsi:
10253         i40e_vsi_clear(vsi);
10254         return NULL;
10255 }
10256
10257 /**
10258  * i40e_vsi_setup - Set up a VSI by a given type
10259  * @pf: board private structure
10260  * @type: VSI type
10261  * @uplink_seid: the switch element to link to
10262  * @param1: usage depends upon VSI type. For VF types, indicates VF id
10263  *
10264  * This allocates the sw VSI structure and its queue resources, then add a VSI
10265  * to the identified VEB.
10266  *
10267  * Returns pointer to the successfully allocated and configure VSI sw struct on
10268  * success, otherwise returns NULL on failure.
10269  **/
10270 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
10271                                 u16 uplink_seid, u32 param1)
10272 {
10273         struct i40e_vsi *vsi = NULL;
10274         struct i40e_veb *veb = NULL;
10275         u16 alloc_queue_pairs;
10276         int ret, i;
10277         int v_idx;
10278
10279         /* The requested uplink_seid must be either
10280          *     - the PF's port seid
10281          *              no VEB is needed because this is the PF
10282          *              or this is a Flow Director special case VSI
10283          *     - seid of an existing VEB
10284          *     - seid of a VSI that owns an existing VEB
10285          *     - seid of a VSI that doesn't own a VEB
10286          *              a new VEB is created and the VSI becomes the owner
10287          *     - seid of the PF VSI, which is what creates the first VEB
10288          *              this is a special case of the previous
10289          *
10290          * Find which uplink_seid we were given and create a new VEB if needed
10291          */
10292         for (i = 0; i < I40E_MAX_VEB; i++) {
10293                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
10294                         veb = pf->veb[i];
10295                         break;
10296                 }
10297         }
10298
10299         if (!veb && uplink_seid != pf->mac_seid) {
10300
10301                 for (i = 0; i < pf->num_alloc_vsi; i++) {
10302                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
10303                                 vsi = pf->vsi[i];
10304                                 break;
10305                         }
10306                 }
10307                 if (!vsi) {
10308                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
10309                                  uplink_seid);
10310                         return NULL;
10311                 }
10312
10313                 if (vsi->uplink_seid == pf->mac_seid)
10314                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
10315                                              vsi->tc_config.enabled_tc);
10316                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
10317                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
10318                                              vsi->tc_config.enabled_tc);
10319                 if (veb) {
10320                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
10321                                 dev_info(&vsi->back->pdev->dev,
10322                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
10323                                 return NULL;
10324                         }
10325                         /* We come up by default in VEPA mode if SRIOV is not
10326                          * already enabled, in which case we can't force VEPA
10327                          * mode.
10328                          */
10329                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
10330                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
10331                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
10332                         }
10333                         i40e_config_bridge_mode(veb);
10334                 }
10335                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
10336                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
10337                                 veb = pf->veb[i];
10338                 }
10339                 if (!veb) {
10340                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
10341                         return NULL;
10342                 }
10343
10344                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10345                 uplink_seid = veb->seid;
10346         }
10347
10348         /* get vsi sw struct */
10349         v_idx = i40e_vsi_mem_alloc(pf, type);
10350         if (v_idx < 0)
10351                 goto err_alloc;
10352         vsi = pf->vsi[v_idx];
10353         if (!vsi)
10354                 goto err_alloc;
10355         vsi->type = type;
10356         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
10357
10358         if (type == I40E_VSI_MAIN)
10359                 pf->lan_vsi = v_idx;
10360         else if (type == I40E_VSI_SRIOV)
10361                 vsi->vf_id = param1;
10362         /* assign it some queues */
10363         alloc_queue_pairs = vsi->alloc_queue_pairs *
10364                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
10365
10366         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
10367         if (ret < 0) {
10368                 dev_info(&pf->pdev->dev,
10369                          "failed to get tracking for %d queues for VSI %d err=%d\n",
10370                          alloc_queue_pairs, vsi->seid, ret);
10371                 goto err_vsi;
10372         }
10373         vsi->base_queue = ret;
10374
10375         /* get a VSI from the hardware */
10376         vsi->uplink_seid = uplink_seid;
10377         ret = i40e_add_vsi(vsi);
10378         if (ret)
10379                 goto err_vsi;
10380
10381         switch (vsi->type) {
10382         /* setup the netdev if needed */
10383         case I40E_VSI_MAIN:
10384         case I40E_VSI_VMDQ2:
10385                 ret = i40e_config_netdev(vsi);
10386                 if (ret)
10387                         goto err_netdev;
10388                 ret = register_netdev(vsi->netdev);
10389                 if (ret)
10390                         goto err_netdev;
10391                 vsi->netdev_registered = true;
10392                 netif_carrier_off(vsi->netdev);
10393 #ifdef CONFIG_I40E_DCB
10394                 /* Setup DCB netlink interface */
10395                 i40e_dcbnl_setup(vsi);
10396 #endif /* CONFIG_I40E_DCB */
10397                 /* fall through */
10398
10399         case I40E_VSI_FDIR:
10400                 /* set up vectors and rings if needed */
10401                 ret = i40e_vsi_setup_vectors(vsi);
10402                 if (ret)
10403                         goto err_msix;
10404
10405                 ret = i40e_alloc_rings(vsi);
10406                 if (ret)
10407                         goto err_rings;
10408
10409                 /* map all of the rings to the q_vectors */
10410                 i40e_vsi_map_rings_to_vectors(vsi);
10411
10412                 i40e_vsi_reset_stats(vsi);
10413                 break;
10414
10415         default:
10416                 /* no netdev or rings for the other VSI types */
10417                 break;
10418         }
10419
10420         if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
10421             (vsi->type == I40E_VSI_VMDQ2)) {
10422                 ret = i40e_vsi_config_rss(vsi);
10423         }
10424         return vsi;
10425
10426 err_rings:
10427         i40e_vsi_free_q_vectors(vsi);
10428 err_msix:
10429         if (vsi->netdev_registered) {
10430                 vsi->netdev_registered = false;
10431                 unregister_netdev(vsi->netdev);
10432                 free_netdev(vsi->netdev);
10433                 vsi->netdev = NULL;
10434         }
10435 err_netdev:
10436         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
10437 err_vsi:
10438         i40e_vsi_clear(vsi);
10439 err_alloc:
10440         return NULL;
10441 }
10442
10443 /**
10444  * i40e_veb_get_bw_info - Query VEB BW information
10445  * @veb: the veb to query
10446  *
10447  * Query the Tx scheduler BW configuration data for given VEB
10448  **/
10449 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
10450 {
10451         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
10452         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
10453         struct i40e_pf *pf = veb->pf;
10454         struct i40e_hw *hw = &pf->hw;
10455         u32 tc_bw_max;
10456         int ret = 0;
10457         int i;
10458
10459         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
10460                                                   &bw_data, NULL);
10461         if (ret) {
10462                 dev_info(&pf->pdev->dev,
10463                          "query veb bw config failed, err %s aq_err %s\n",
10464                          i40e_stat_str(&pf->hw, ret),
10465                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10466                 goto out;
10467         }
10468
10469         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
10470                                                    &ets_data, NULL);
10471         if (ret) {
10472                 dev_info(&pf->pdev->dev,
10473                          "query veb bw ets config failed, err %s aq_err %s\n",
10474                          i40e_stat_str(&pf->hw, ret),
10475                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10476                 goto out;
10477         }
10478
10479         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
10480         veb->bw_max_quanta = ets_data.tc_bw_max;
10481         veb->is_abs_credits = bw_data.absolute_credits_enable;
10482         veb->enabled_tc = ets_data.tc_valid_bits;
10483         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
10484                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
10485         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
10486                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
10487                 veb->bw_tc_limit_credits[i] =
10488                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
10489                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
10490         }
10491
10492 out:
10493         return ret;
10494 }
10495
10496 /**
10497  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
10498  * @pf: board private structure
10499  *
10500  * On error: returns error code (negative)
10501  * On success: returns vsi index in PF (positive)
10502  **/
10503 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
10504 {
10505         int ret = -ENOENT;
10506         struct i40e_veb *veb;
10507         int i;
10508
10509         /* Need to protect the allocation of switch elements at the PF level */
10510         mutex_lock(&pf->switch_mutex);
10511
10512         /* VEB list may be fragmented if VEB creation/destruction has
10513          * been happening.  We can afford to do a quick scan to look
10514          * for any free slots in the list.
10515          *
10516          * find next empty veb slot, looping back around if necessary
10517          */
10518         i = 0;
10519         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
10520                 i++;
10521         if (i >= I40E_MAX_VEB) {
10522                 ret = -ENOMEM;
10523                 goto err_alloc_veb;  /* out of VEB slots! */
10524         }
10525
10526         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
10527         if (!veb) {
10528                 ret = -ENOMEM;
10529                 goto err_alloc_veb;
10530         }
10531         veb->pf = pf;
10532         veb->idx = i;
10533         veb->enabled_tc = 1;
10534
10535         pf->veb[i] = veb;
10536         ret = i;
10537 err_alloc_veb:
10538         mutex_unlock(&pf->switch_mutex);
10539         return ret;
10540 }
10541
10542 /**
10543  * i40e_switch_branch_release - Delete a branch of the switch tree
10544  * @branch: where to start deleting
10545  *
10546  * This uses recursion to find the tips of the branch to be
10547  * removed, deleting until we get back to and can delete this VEB.
10548  **/
10549 static void i40e_switch_branch_release(struct i40e_veb *branch)
10550 {
10551         struct i40e_pf *pf = branch->pf;
10552         u16 branch_seid = branch->seid;
10553         u16 veb_idx = branch->idx;
10554         int i;
10555
10556         /* release any VEBs on this VEB - RECURSION */
10557         for (i = 0; i < I40E_MAX_VEB; i++) {
10558                 if (!pf->veb[i])
10559                         continue;
10560                 if (pf->veb[i]->uplink_seid == branch->seid)
10561                         i40e_switch_branch_release(pf->veb[i]);
10562         }
10563
10564         /* Release the VSIs on this VEB, but not the owner VSI.
10565          *
10566          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
10567          *       the VEB itself, so don't use (*branch) after this loop.
10568          */
10569         for (i = 0; i < pf->num_alloc_vsi; i++) {
10570                 if (!pf->vsi[i])
10571                         continue;
10572                 if (pf->vsi[i]->uplink_seid == branch_seid &&
10573                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10574                         i40e_vsi_release(pf->vsi[i]);
10575                 }
10576         }
10577
10578         /* There's one corner case where the VEB might not have been
10579          * removed, so double check it here and remove it if needed.
10580          * This case happens if the veb was created from the debugfs
10581          * commands and no VSIs were added to it.
10582          */
10583         if (pf->veb[veb_idx])
10584                 i40e_veb_release(pf->veb[veb_idx]);
10585 }
10586
10587 /**
10588  * i40e_veb_clear - remove veb struct
10589  * @veb: the veb to remove
10590  **/
10591 static void i40e_veb_clear(struct i40e_veb *veb)
10592 {
10593         if (!veb)
10594                 return;
10595
10596         if (veb->pf) {
10597                 struct i40e_pf *pf = veb->pf;
10598
10599                 mutex_lock(&pf->switch_mutex);
10600                 if (pf->veb[veb->idx] == veb)
10601                         pf->veb[veb->idx] = NULL;
10602                 mutex_unlock(&pf->switch_mutex);
10603         }
10604
10605         kfree(veb);
10606 }
10607
10608 /**
10609  * i40e_veb_release - Delete a VEB and free its resources
10610  * @veb: the VEB being removed
10611  **/
10612 void i40e_veb_release(struct i40e_veb *veb)
10613 {
10614         struct i40e_vsi *vsi = NULL;
10615         struct i40e_pf *pf;
10616         int i, n = 0;
10617
10618         pf = veb->pf;
10619
10620         /* find the remaining VSI and check for extras */
10621         for (i = 0; i < pf->num_alloc_vsi; i++) {
10622                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
10623                         n++;
10624                         vsi = pf->vsi[i];
10625                 }
10626         }
10627         if (n != 1) {
10628                 dev_info(&pf->pdev->dev,
10629                          "can't remove VEB %d with %d VSIs left\n",
10630                          veb->seid, n);
10631                 return;
10632         }
10633
10634         /* move the remaining VSI to uplink veb */
10635         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
10636         if (veb->uplink_seid) {
10637                 vsi->uplink_seid = veb->uplink_seid;
10638                 if (veb->uplink_seid == pf->mac_seid)
10639                         vsi->veb_idx = I40E_NO_VEB;
10640                 else
10641                         vsi->veb_idx = veb->veb_idx;
10642         } else {
10643                 /* floating VEB */
10644                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10645                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
10646         }
10647
10648         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10649         i40e_veb_clear(veb);
10650 }
10651
10652 /**
10653  * i40e_add_veb - create the VEB in the switch
10654  * @veb: the VEB to be instantiated
10655  * @vsi: the controlling VSI
10656  **/
10657 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
10658 {
10659         struct i40e_pf *pf = veb->pf;
10660         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
10661         int ret;
10662
10663         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
10664                               veb->enabled_tc, false,
10665                               &veb->seid, enable_stats, NULL);
10666
10667         /* get a VEB from the hardware */
10668         if (ret) {
10669                 dev_info(&pf->pdev->dev,
10670                          "couldn't add VEB, err %s aq_err %s\n",
10671                          i40e_stat_str(&pf->hw, ret),
10672                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10673                 return -EPERM;
10674         }
10675
10676         /* get statistics counter */
10677         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
10678                                          &veb->stats_idx, NULL, NULL, NULL);
10679         if (ret) {
10680                 dev_info(&pf->pdev->dev,
10681                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
10682                          i40e_stat_str(&pf->hw, ret),
10683                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10684                 return -EPERM;
10685         }
10686         ret = i40e_veb_get_bw_info(veb);
10687         if (ret) {
10688                 dev_info(&pf->pdev->dev,
10689                          "couldn't get VEB bw info, err %s aq_err %s\n",
10690                          i40e_stat_str(&pf->hw, ret),
10691                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10692                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10693                 return -ENOENT;
10694         }
10695
10696         vsi->uplink_seid = veb->seid;
10697         vsi->veb_idx = veb->idx;
10698         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10699
10700         return 0;
10701 }
10702
10703 /**
10704  * i40e_veb_setup - Set up a VEB
10705  * @pf: board private structure
10706  * @flags: VEB setup flags
10707  * @uplink_seid: the switch element to link to
10708  * @vsi_seid: the initial VSI seid
10709  * @enabled_tc: Enabled TC bit-map
10710  *
10711  * This allocates the sw VEB structure and links it into the switch
10712  * It is possible and legal for this to be a duplicate of an already
10713  * existing VEB.  It is also possible for both uplink and vsi seids
10714  * to be zero, in order to create a floating VEB.
10715  *
10716  * Returns pointer to the successfully allocated VEB sw struct on
10717  * success, otherwise returns NULL on failure.
10718  **/
10719 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
10720                                 u16 uplink_seid, u16 vsi_seid,
10721                                 u8 enabled_tc)
10722 {
10723         struct i40e_veb *veb, *uplink_veb = NULL;
10724         int vsi_idx, veb_idx;
10725         int ret;
10726
10727         /* if one seid is 0, the other must be 0 to create a floating relay */
10728         if ((uplink_seid == 0 || vsi_seid == 0) &&
10729             (uplink_seid + vsi_seid != 0)) {
10730                 dev_info(&pf->pdev->dev,
10731                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
10732                          uplink_seid, vsi_seid);
10733                 return NULL;
10734         }
10735
10736         /* make sure there is such a vsi and uplink */
10737         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
10738                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
10739                         break;
10740         if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
10741                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
10742                          vsi_seid);
10743                 return NULL;
10744         }
10745
10746         if (uplink_seid && uplink_seid != pf->mac_seid) {
10747                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10748                         if (pf->veb[veb_idx] &&
10749                             pf->veb[veb_idx]->seid == uplink_seid) {
10750                                 uplink_veb = pf->veb[veb_idx];
10751                                 break;
10752                         }
10753                 }
10754                 if (!uplink_veb) {
10755                         dev_info(&pf->pdev->dev,
10756                                  "uplink seid %d not found\n", uplink_seid);
10757                         return NULL;
10758                 }
10759         }
10760
10761         /* get veb sw struct */
10762         veb_idx = i40e_veb_mem_alloc(pf);
10763         if (veb_idx < 0)
10764                 goto err_alloc;
10765         veb = pf->veb[veb_idx];
10766         veb->flags = flags;
10767         veb->uplink_seid = uplink_seid;
10768         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
10769         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
10770
10771         /* create the VEB in the switch */
10772         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
10773         if (ret)
10774                 goto err_veb;
10775         if (vsi_idx == pf->lan_vsi)
10776                 pf->lan_veb = veb->idx;
10777
10778         return veb;
10779
10780 err_veb:
10781         i40e_veb_clear(veb);
10782 err_alloc:
10783         return NULL;
10784 }
10785
10786 /**
10787  * i40e_setup_pf_switch_element - set PF vars based on switch type
10788  * @pf: board private structure
10789  * @ele: element we are building info from
10790  * @num_reported: total number of elements
10791  * @printconfig: should we print the contents
10792  *
10793  * helper function to assist in extracting a few useful SEID values.
10794  **/
10795 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
10796                                 struct i40e_aqc_switch_config_element_resp *ele,
10797                                 u16 num_reported, bool printconfig)
10798 {
10799         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
10800         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
10801         u8 element_type = ele->element_type;
10802         u16 seid = le16_to_cpu(ele->seid);
10803
10804         if (printconfig)
10805                 dev_info(&pf->pdev->dev,
10806                          "type=%d seid=%d uplink=%d downlink=%d\n",
10807                          element_type, seid, uplink_seid, downlink_seid);
10808
10809         switch (element_type) {
10810         case I40E_SWITCH_ELEMENT_TYPE_MAC:
10811                 pf->mac_seid = seid;
10812                 break;
10813         case I40E_SWITCH_ELEMENT_TYPE_VEB:
10814                 /* Main VEB? */
10815                 if (uplink_seid != pf->mac_seid)
10816                         break;
10817                 if (pf->lan_veb == I40E_NO_VEB) {
10818                         int v;
10819
10820                         /* find existing or else empty VEB */
10821                         for (v = 0; v < I40E_MAX_VEB; v++) {
10822                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
10823                                         pf->lan_veb = v;
10824                                         break;
10825                                 }
10826                         }
10827                         if (pf->lan_veb == I40E_NO_VEB) {
10828                                 v = i40e_veb_mem_alloc(pf);
10829                                 if (v < 0)
10830                                         break;
10831                                 pf->lan_veb = v;
10832                         }
10833                 }
10834
10835                 pf->veb[pf->lan_veb]->seid = seid;
10836                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
10837                 pf->veb[pf->lan_veb]->pf = pf;
10838                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
10839                 break;
10840         case I40E_SWITCH_ELEMENT_TYPE_VSI:
10841                 if (num_reported != 1)
10842                         break;
10843                 /* This is immediately after a reset so we can assume this is
10844                  * the PF's VSI
10845                  */
10846                 pf->mac_seid = uplink_seid;
10847                 pf->pf_seid = downlink_seid;
10848                 pf->main_vsi_seid = seid;
10849                 if (printconfig)
10850                         dev_info(&pf->pdev->dev,
10851                                  "pf_seid=%d main_vsi_seid=%d\n",
10852                                  pf->pf_seid, pf->main_vsi_seid);
10853                 break;
10854         case I40E_SWITCH_ELEMENT_TYPE_PF:
10855         case I40E_SWITCH_ELEMENT_TYPE_VF:
10856         case I40E_SWITCH_ELEMENT_TYPE_EMP:
10857         case I40E_SWITCH_ELEMENT_TYPE_BMC:
10858         case I40E_SWITCH_ELEMENT_TYPE_PE:
10859         case I40E_SWITCH_ELEMENT_TYPE_PA:
10860                 /* ignore these for now */
10861                 break;
10862         default:
10863                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
10864                          element_type, seid);
10865                 break;
10866         }
10867 }
10868
10869 /**
10870  * i40e_fetch_switch_configuration - Get switch config from firmware
10871  * @pf: board private structure
10872  * @printconfig: should we print the contents
10873  *
10874  * Get the current switch configuration from the device and
10875  * extract a few useful SEID values.
10876  **/
10877 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
10878 {
10879         struct i40e_aqc_get_switch_config_resp *sw_config;
10880         u16 next_seid = 0;
10881         int ret = 0;
10882         u8 *aq_buf;
10883         int i;
10884
10885         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
10886         if (!aq_buf)
10887                 return -ENOMEM;
10888
10889         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
10890         do {
10891                 u16 num_reported, num_total;
10892
10893                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
10894                                                 I40E_AQ_LARGE_BUF,
10895                                                 &next_seid, NULL);
10896                 if (ret) {
10897                         dev_info(&pf->pdev->dev,
10898                                  "get switch config failed err %s aq_err %s\n",
10899                                  i40e_stat_str(&pf->hw, ret),
10900                                  i40e_aq_str(&pf->hw,
10901                                              pf->hw.aq.asq_last_status));
10902                         kfree(aq_buf);
10903                         return -ENOENT;
10904                 }
10905
10906                 num_reported = le16_to_cpu(sw_config->header.num_reported);
10907                 num_total = le16_to_cpu(sw_config->header.num_total);
10908
10909                 if (printconfig)
10910                         dev_info(&pf->pdev->dev,
10911                                  "header: %d reported %d total\n",
10912                                  num_reported, num_total);
10913
10914                 for (i = 0; i < num_reported; i++) {
10915                         struct i40e_aqc_switch_config_element_resp *ele =
10916                                 &sw_config->element[i];
10917
10918                         i40e_setup_pf_switch_element(pf, ele, num_reported,
10919                                                      printconfig);
10920                 }
10921         } while (next_seid != 0);
10922
10923         kfree(aq_buf);
10924         return ret;
10925 }
10926
10927 /**
10928  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
10929  * @pf: board private structure
10930  * @reinit: if the Main VSI needs to re-initialized.
10931  *
10932  * Returns 0 on success, negative value on failure
10933  **/
10934 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
10935 {
10936         u16 flags = 0;
10937         int ret;
10938
10939         /* find out what's out there already */
10940         ret = i40e_fetch_switch_configuration(pf, false);
10941         if (ret) {
10942                 dev_info(&pf->pdev->dev,
10943                          "couldn't fetch switch config, err %s aq_err %s\n",
10944                          i40e_stat_str(&pf->hw, ret),
10945                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10946                 return ret;
10947         }
10948         i40e_pf_reset_stats(pf);
10949
10950         /* set the switch config bit for the whole device to
10951          * support limited promisc or true promisc
10952          * when user requests promisc. The default is limited
10953          * promisc.
10954         */
10955
10956         if ((pf->hw.pf_id == 0) &&
10957             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT))
10958                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
10959
10960         if (pf->hw.pf_id == 0) {
10961                 u16 valid_flags;
10962
10963                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
10964                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags,
10965                                                 NULL);
10966                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
10967                         dev_info(&pf->pdev->dev,
10968                                  "couldn't set switch config bits, err %s aq_err %s\n",
10969                                  i40e_stat_str(&pf->hw, ret),
10970                                  i40e_aq_str(&pf->hw,
10971                                              pf->hw.aq.asq_last_status));
10972                         /* not a fatal problem, just keep going */
10973                 }
10974         }
10975
10976         /* first time setup */
10977         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
10978                 struct i40e_vsi *vsi = NULL;
10979                 u16 uplink_seid;
10980
10981                 /* Set up the PF VSI associated with the PF's main VSI
10982                  * that is already in the HW switch
10983                  */
10984                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
10985                         uplink_seid = pf->veb[pf->lan_veb]->seid;
10986                 else
10987                         uplink_seid = pf->mac_seid;
10988                 if (pf->lan_vsi == I40E_NO_VSI)
10989                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
10990                 else if (reinit)
10991                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
10992                 if (!vsi) {
10993                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
10994                         i40e_fdir_teardown(pf);
10995                         return -EAGAIN;
10996                 }
10997         } else {
10998                 /* force a reset of TC and queue layout configurations */
10999                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
11000
11001                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
11002                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
11003                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
11004         }
11005         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
11006
11007         i40e_fdir_sb_setup(pf);
11008
11009         /* Setup static PF queue filter control settings */
11010         ret = i40e_setup_pf_filter_control(pf);
11011         if (ret) {
11012                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
11013                          ret);
11014                 /* Failure here should not stop continuing other steps */
11015         }
11016
11017         /* enable RSS in the HW, even for only one queue, as the stack can use
11018          * the hash
11019          */
11020         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
11021                 i40e_pf_config_rss(pf);
11022
11023         /* fill in link information and enable LSE reporting */
11024         i40e_link_event(pf);
11025
11026         /* Initialize user-specific link properties */
11027         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
11028                                   I40E_AQ_AN_COMPLETED) ? true : false);
11029
11030         i40e_ptp_init(pf);
11031
11032         /* repopulate tunnel port filters */
11033         i40e_sync_udp_filters(pf);
11034
11035         return ret;
11036 }
11037
11038 /**
11039  * i40e_determine_queue_usage - Work out queue distribution
11040  * @pf: board private structure
11041  **/
11042 static void i40e_determine_queue_usage(struct i40e_pf *pf)
11043 {
11044         int queues_left;
11045
11046         pf->num_lan_qps = 0;
11047
11048         /* Find the max queues to be put into basic use.  We'll always be
11049          * using TC0, whether or not DCB is running, and TC0 will get the
11050          * big RSS set.
11051          */
11052         queues_left = pf->hw.func_caps.num_tx_qp;
11053
11054         if ((queues_left == 1) ||
11055             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
11056                 /* one qp for PF, no queues for anything else */
11057                 queues_left = 0;
11058                 pf->alloc_rss_size = pf->num_lan_qps = 1;
11059
11060                 /* make sure all the fancies are disabled */
11061                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
11062                                I40E_FLAG_IWARP_ENABLED  |
11063                                I40E_FLAG_FD_SB_ENABLED  |
11064                                I40E_FLAG_FD_ATR_ENABLED |
11065                                I40E_FLAG_DCB_CAPABLE    |
11066                                I40E_FLAG_DCB_ENABLED    |
11067                                I40E_FLAG_SRIOV_ENABLED  |
11068                                I40E_FLAG_VMDQ_ENABLED);
11069         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
11070                                   I40E_FLAG_FD_SB_ENABLED |
11071                                   I40E_FLAG_FD_ATR_ENABLED |
11072                                   I40E_FLAG_DCB_CAPABLE))) {
11073                 /* one qp for PF */
11074                 pf->alloc_rss_size = pf->num_lan_qps = 1;
11075                 queues_left -= pf->num_lan_qps;
11076
11077                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
11078                                I40E_FLAG_IWARP_ENABLED  |
11079                                I40E_FLAG_FD_SB_ENABLED  |
11080                                I40E_FLAG_FD_ATR_ENABLED |
11081                                I40E_FLAG_DCB_ENABLED    |
11082                                I40E_FLAG_VMDQ_ENABLED);
11083         } else {
11084                 /* Not enough queues for all TCs */
11085                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
11086                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
11087                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
11088                                         I40E_FLAG_DCB_ENABLED);
11089                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
11090                 }
11091                 pf->num_lan_qps = max_t(int, pf->rss_size_max,
11092                                         num_online_cpus());
11093                 pf->num_lan_qps = min_t(int, pf->num_lan_qps,
11094                                         pf->hw.func_caps.num_tx_qp);
11095
11096                 queues_left -= pf->num_lan_qps;
11097         }
11098
11099         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11100                 if (queues_left > 1) {
11101                         queues_left -= 1; /* save 1 queue for FD */
11102                 } else {
11103                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11104                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
11105                 }
11106         }
11107
11108         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11109             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
11110                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
11111                                         (queues_left / pf->num_vf_qps));
11112                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
11113         }
11114
11115         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11116             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
11117                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
11118                                           (queues_left / pf->num_vmdq_qps));
11119                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
11120         }
11121
11122         pf->queues_left = queues_left;
11123         dev_dbg(&pf->pdev->dev,
11124                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
11125                 pf->hw.func_caps.num_tx_qp,
11126                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
11127                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
11128                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
11129                 queues_left);
11130 }
11131
11132 /**
11133  * i40e_setup_pf_filter_control - Setup PF static filter control
11134  * @pf: PF to be setup
11135  *
11136  * i40e_setup_pf_filter_control sets up a PF's initial filter control
11137  * settings. If PE/FCoE are enabled then it will also set the per PF
11138  * based filter sizes required for them. It also enables Flow director,
11139  * ethertype and macvlan type filter settings for the pf.
11140  *
11141  * Returns 0 on success, negative on failure
11142  **/
11143 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
11144 {
11145         struct i40e_filter_control_settings *settings = &pf->filter_settings;
11146
11147         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
11148
11149         /* Flow Director is enabled */
11150         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
11151                 settings->enable_fdir = true;
11152
11153         /* Ethtype and MACVLAN filters enabled for PF */
11154         settings->enable_ethtype = true;
11155         settings->enable_macvlan = true;
11156
11157         if (i40e_set_filter_control(&pf->hw, settings))
11158                 return -ENOENT;
11159
11160         return 0;
11161 }
11162
11163 #define INFO_STRING_LEN 255
11164 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
11165 static void i40e_print_features(struct i40e_pf *pf)
11166 {
11167         struct i40e_hw *hw = &pf->hw;
11168         char *buf;
11169         int i;
11170
11171         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
11172         if (!buf)
11173                 return;
11174
11175         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
11176 #ifdef CONFIG_PCI_IOV
11177         i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
11178 #endif
11179         i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
11180                       pf->hw.func_caps.num_vsis,
11181                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
11182         if (pf->flags & I40E_FLAG_RSS_ENABLED)
11183                 i += snprintf(&buf[i], REMAIN(i), " RSS");
11184         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
11185                 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
11186         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11187                 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
11188                 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
11189         }
11190         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
11191                 i += snprintf(&buf[i], REMAIN(i), " DCB");
11192         i += snprintf(&buf[i], REMAIN(i), " VxLAN");
11193         i += snprintf(&buf[i], REMAIN(i), " Geneve");
11194         if (pf->flags & I40E_FLAG_PTP)
11195                 i += snprintf(&buf[i], REMAIN(i), " PTP");
11196         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
11197                 i += snprintf(&buf[i], REMAIN(i), " VEB");
11198         else
11199                 i += snprintf(&buf[i], REMAIN(i), " VEPA");
11200
11201         dev_info(&pf->pdev->dev, "%s\n", buf);
11202         kfree(buf);
11203         WARN_ON(i > INFO_STRING_LEN);
11204 }
11205
11206 /**
11207  * i40e_get_platform_mac_addr - get platform-specific MAC address
11208  * @pdev: PCI device information struct
11209  * @pf: board private structure
11210  *
11211  * Look up the MAC address for the device. First we'll try
11212  * eth_platform_get_mac_address, which will check Open Firmware, or arch
11213  * specific fallback. Otherwise, we'll default to the stored value in
11214  * firmware.
11215  **/
11216 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
11217 {
11218         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
11219                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
11220 }
11221
11222 /**
11223  * i40e_probe - Device initialization routine
11224  * @pdev: PCI device information struct
11225  * @ent: entry in i40e_pci_tbl
11226  *
11227  * i40e_probe initializes a PF identified by a pci_dev structure.
11228  * The OS initialization, configuring of the PF private structure,
11229  * and a hardware reset occur.
11230  *
11231  * Returns 0 on success, negative on failure
11232  **/
11233 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
11234 {
11235         struct i40e_aq_get_phy_abilities_resp abilities;
11236         struct i40e_pf *pf;
11237         struct i40e_hw *hw;
11238         static u16 pfs_found;
11239         u16 wol_nvm_bits;
11240         u16 link_status;
11241         int err;
11242         u32 val;
11243         u32 i;
11244         u8 set_fc_aq_fail;
11245
11246         err = pci_enable_device_mem(pdev);
11247         if (err)
11248                 return err;
11249
11250         /* set up for high or low dma */
11251         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
11252         if (err) {
11253                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
11254                 if (err) {
11255                         dev_err(&pdev->dev,
11256                                 "DMA configuration failed: 0x%x\n", err);
11257                         goto err_dma;
11258                 }
11259         }
11260
11261         /* set up pci connections */
11262         err = pci_request_mem_regions(pdev, i40e_driver_name);
11263         if (err) {
11264                 dev_info(&pdev->dev,
11265                          "pci_request_selected_regions failed %d\n", err);
11266                 goto err_pci_reg;
11267         }
11268
11269         pci_enable_pcie_error_reporting(pdev);
11270         pci_set_master(pdev);
11271
11272         /* Now that we have a PCI connection, we need to do the
11273          * low level device setup.  This is primarily setting up
11274          * the Admin Queue structures and then querying for the
11275          * device's current profile information.
11276          */
11277         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
11278         if (!pf) {
11279                 err = -ENOMEM;
11280                 goto err_pf_alloc;
11281         }
11282         pf->next_vsi = 0;
11283         pf->pdev = pdev;
11284         set_bit(__I40E_DOWN, pf->state);
11285
11286         hw = &pf->hw;
11287         hw->back = pf;
11288
11289         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
11290                                 I40E_MAX_CSR_SPACE);
11291
11292         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
11293         if (!hw->hw_addr) {
11294                 err = -EIO;
11295                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
11296                          (unsigned int)pci_resource_start(pdev, 0),
11297                          pf->ioremap_len, err);
11298                 goto err_ioremap;
11299         }
11300         hw->vendor_id = pdev->vendor;
11301         hw->device_id = pdev->device;
11302         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
11303         hw->subsystem_vendor_id = pdev->subsystem_vendor;
11304         hw->subsystem_device_id = pdev->subsystem_device;
11305         hw->bus.device = PCI_SLOT(pdev->devfn);
11306         hw->bus.func = PCI_FUNC(pdev->devfn);
11307         hw->bus.bus_id = pdev->bus->number;
11308         pf->instance = pfs_found;
11309
11310         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
11311         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
11312
11313         /* set up the locks for the AQ, do this only once in probe
11314          * and destroy them only once in remove
11315          */
11316         mutex_init(&hw->aq.asq_mutex);
11317         mutex_init(&hw->aq.arq_mutex);
11318
11319         pf->msg_enable = netif_msg_init(debug,
11320                                         NETIF_MSG_DRV |
11321                                         NETIF_MSG_PROBE |
11322                                         NETIF_MSG_LINK);
11323         if (debug < -1)
11324                 pf->hw.debug_mask = debug;
11325
11326         /* do a special CORER for clearing PXE mode once at init */
11327         if (hw->revision_id == 0 &&
11328             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
11329                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
11330                 i40e_flush(hw);
11331                 msleep(200);
11332                 pf->corer_count++;
11333
11334                 i40e_clear_pxe_mode(hw);
11335         }
11336
11337         /* Reset here to make sure all is clean and to define PF 'n' */
11338         i40e_clear_hw(hw);
11339         err = i40e_pf_reset(hw);
11340         if (err) {
11341                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
11342                 goto err_pf_reset;
11343         }
11344         pf->pfr_count++;
11345
11346         hw->aq.num_arq_entries = I40E_AQ_LEN;
11347         hw->aq.num_asq_entries = I40E_AQ_LEN;
11348         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
11349         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
11350         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
11351
11352         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
11353                  "%s-%s:misc",
11354                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
11355
11356         err = i40e_init_shared_code(hw);
11357         if (err) {
11358                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
11359                          err);
11360                 goto err_pf_reset;
11361         }
11362
11363         /* set up a default setting for link flow control */
11364         pf->hw.fc.requested_mode = I40E_FC_NONE;
11365
11366         err = i40e_init_adminq(hw);
11367         if (err) {
11368                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
11369                         dev_info(&pdev->dev,
11370                                  "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
11371                 else
11372                         dev_info(&pdev->dev,
11373                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
11374
11375                 goto err_pf_reset;
11376         }
11377         i40e_get_oem_version(hw);
11378
11379         /* provide nvm, fw, api versions */
11380         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n",
11381                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
11382                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
11383                  i40e_nvm_version_str(hw));
11384
11385         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
11386             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
11387                 dev_info(&pdev->dev,
11388                          "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
11389         else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
11390                 dev_info(&pdev->dev,
11391                          "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
11392
11393         i40e_verify_eeprom(pf);
11394
11395         /* Rev 0 hardware was never productized */
11396         if (hw->revision_id < 1)
11397                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
11398
11399         i40e_clear_pxe_mode(hw);
11400         err = i40e_get_capabilities(pf);
11401         if (err)
11402                 goto err_adminq_setup;
11403
11404         err = i40e_sw_init(pf);
11405         if (err) {
11406                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
11407                 goto err_sw_init;
11408         }
11409
11410         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
11411                                 hw->func_caps.num_rx_qp, 0, 0);
11412         if (err) {
11413                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
11414                 goto err_init_lan_hmc;
11415         }
11416
11417         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
11418         if (err) {
11419                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
11420                 err = -ENOENT;
11421                 goto err_configure_lan_hmc;
11422         }
11423
11424         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
11425          * Ignore error return codes because if it was already disabled via
11426          * hardware settings this will fail
11427          */
11428         if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
11429                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
11430                 i40e_aq_stop_lldp(hw, true, NULL);
11431         }
11432
11433         /* allow a platform config to override the HW addr */
11434         i40e_get_platform_mac_addr(pdev, pf);
11435
11436         if (!is_valid_ether_addr(hw->mac.addr)) {
11437                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
11438                 err = -EIO;
11439                 goto err_mac_addr;
11440         }
11441         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
11442         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
11443         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
11444         if (is_valid_ether_addr(hw->mac.port_addr))
11445                 pf->hw_features |= I40E_HW_PORT_ID_VALID;
11446
11447         pci_set_drvdata(pdev, pf);
11448         pci_save_state(pdev);
11449 #ifdef CONFIG_I40E_DCB
11450         err = i40e_init_pf_dcb(pf);
11451         if (err) {
11452                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
11453                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
11454                 /* Continue without DCB enabled */
11455         }
11456 #endif /* CONFIG_I40E_DCB */
11457
11458         /* set up periodic task facility */
11459         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
11460         pf->service_timer_period = HZ;
11461
11462         INIT_WORK(&pf->service_task, i40e_service_task);
11463         clear_bit(__I40E_SERVICE_SCHED, pf->state);
11464
11465         /* NVM bit on means WoL disabled for the port */
11466         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
11467         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
11468                 pf->wol_en = false;
11469         else
11470                 pf->wol_en = true;
11471         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
11472
11473         /* set up the main switch operations */
11474         i40e_determine_queue_usage(pf);
11475         err = i40e_init_interrupt_scheme(pf);
11476         if (err)
11477                 goto err_switch_setup;
11478
11479         /* The number of VSIs reported by the FW is the minimum guaranteed
11480          * to us; HW supports far more and we share the remaining pool with
11481          * the other PFs. We allocate space for more than the guarantee with
11482          * the understanding that we might not get them all later.
11483          */
11484         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
11485                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
11486         else
11487                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
11488
11489         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
11490         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
11491                           GFP_KERNEL);
11492         if (!pf->vsi) {
11493                 err = -ENOMEM;
11494                 goto err_switch_setup;
11495         }
11496
11497 #ifdef CONFIG_PCI_IOV
11498         /* prep for VF support */
11499         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11500             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11501             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
11502                 if (pci_num_vf(pdev))
11503                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
11504         }
11505 #endif
11506         err = i40e_setup_pf_switch(pf, false);
11507         if (err) {
11508                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
11509                 goto err_vsis;
11510         }
11511
11512         /* Make sure flow control is set according to current settings */
11513         err = i40e_set_fc(hw, &set_fc_aq_fail, true);
11514         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET)
11515                 dev_dbg(&pf->pdev->dev,
11516                         "Set fc with err %s aq_err %s on get_phy_cap\n",
11517                         i40e_stat_str(hw, err),
11518                         i40e_aq_str(hw, hw->aq.asq_last_status));
11519         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET)
11520                 dev_dbg(&pf->pdev->dev,
11521                         "Set fc with err %s aq_err %s on set_phy_config\n",
11522                         i40e_stat_str(hw, err),
11523                         i40e_aq_str(hw, hw->aq.asq_last_status));
11524         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE)
11525                 dev_dbg(&pf->pdev->dev,
11526                         "Set fc with err %s aq_err %s on get_link_info\n",
11527                         i40e_stat_str(hw, err),
11528                         i40e_aq_str(hw, hw->aq.asq_last_status));
11529
11530         /* if FDIR VSI was set up, start it now */
11531         for (i = 0; i < pf->num_alloc_vsi; i++) {
11532                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
11533                         i40e_vsi_open(pf->vsi[i]);
11534                         break;
11535                 }
11536         }
11537
11538         /* The driver only wants link up/down and module qualification
11539          * reports from firmware.  Note the negative logic.
11540          */
11541         err = i40e_aq_set_phy_int_mask(&pf->hw,
11542                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
11543                                          I40E_AQ_EVENT_MEDIA_NA |
11544                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
11545         if (err)
11546                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
11547                          i40e_stat_str(&pf->hw, err),
11548                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11549
11550         /* Reconfigure hardware for allowing smaller MSS in the case
11551          * of TSO, so that we avoid the MDD being fired and causing
11552          * a reset in the case of small MSS+TSO.
11553          */
11554         val = rd32(hw, I40E_REG_MSS);
11555         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11556                 val &= ~I40E_REG_MSS_MIN_MASK;
11557                 val |= I40E_64BYTE_MSS;
11558                 wr32(hw, I40E_REG_MSS, val);
11559         }
11560
11561         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
11562                 msleep(75);
11563                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11564                 if (err)
11565                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
11566                                  i40e_stat_str(&pf->hw, err),
11567                                  i40e_aq_str(&pf->hw,
11568                                              pf->hw.aq.asq_last_status));
11569         }
11570         /* The main driver is (mostly) up and happy. We need to set this state
11571          * before setting up the misc vector or we get a race and the vector
11572          * ends up disabled forever.
11573          */
11574         clear_bit(__I40E_DOWN, pf->state);
11575
11576         /* In case of MSIX we are going to setup the misc vector right here
11577          * to handle admin queue events etc. In case of legacy and MSI
11578          * the misc functionality and queue processing is combined in
11579          * the same vector and that gets setup at open.
11580          */
11581         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11582                 err = i40e_setup_misc_vector(pf);
11583                 if (err) {
11584                         dev_info(&pdev->dev,
11585                                  "setup of misc vector failed: %d\n", err);
11586                         goto err_vsis;
11587                 }
11588         }
11589
11590 #ifdef CONFIG_PCI_IOV
11591         /* prep for VF support */
11592         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11593             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11594             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
11595                 /* disable link interrupts for VFs */
11596                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
11597                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
11598                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
11599                 i40e_flush(hw);
11600
11601                 if (pci_num_vf(pdev)) {
11602                         dev_info(&pdev->dev,
11603                                  "Active VFs found, allocating resources.\n");
11604                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
11605                         if (err)
11606                                 dev_info(&pdev->dev,
11607                                          "Error %d allocating resources for existing VFs\n",
11608                                          err);
11609                 }
11610         }
11611 #endif /* CONFIG_PCI_IOV */
11612
11613         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11614                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
11615                                                       pf->num_iwarp_msix,
11616                                                       I40E_IWARP_IRQ_PILE_ID);
11617                 if (pf->iwarp_base_vector < 0) {
11618                         dev_info(&pdev->dev,
11619                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
11620                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
11621                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11622                 }
11623         }
11624
11625         i40e_dbg_pf_init(pf);
11626
11627         /* tell the firmware that we're starting */
11628         i40e_send_version(pf);
11629
11630         /* since everything's happy, start the service_task timer */
11631         mod_timer(&pf->service_timer,
11632                   round_jiffies(jiffies + pf->service_timer_period));
11633
11634         /* add this PF to client device list and launch a client service task */
11635         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11636                 err = i40e_lan_add_device(pf);
11637                 if (err)
11638                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
11639                                  err);
11640         }
11641
11642 #define PCI_SPEED_SIZE 8
11643 #define PCI_WIDTH_SIZE 8
11644         /* Devices on the IOSF bus do not have this information
11645          * and will report PCI Gen 1 x 1 by default so don't bother
11646          * checking them.
11647          */
11648         if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
11649                 char speed[PCI_SPEED_SIZE] = "Unknown";
11650                 char width[PCI_WIDTH_SIZE] = "Unknown";
11651
11652                 /* Get the negotiated link width and speed from PCI config
11653                  * space
11654                  */
11655                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
11656                                           &link_status);
11657
11658                 i40e_set_pci_config_data(hw, link_status);
11659
11660                 switch (hw->bus.speed) {
11661                 case i40e_bus_speed_8000:
11662                         strncpy(speed, "8.0", PCI_SPEED_SIZE); break;
11663                 case i40e_bus_speed_5000:
11664                         strncpy(speed, "5.0", PCI_SPEED_SIZE); break;
11665                 case i40e_bus_speed_2500:
11666                         strncpy(speed, "2.5", PCI_SPEED_SIZE); break;
11667                 default:
11668                         break;
11669                 }
11670                 switch (hw->bus.width) {
11671                 case i40e_bus_width_pcie_x8:
11672                         strncpy(width, "8", PCI_WIDTH_SIZE); break;
11673                 case i40e_bus_width_pcie_x4:
11674                         strncpy(width, "4", PCI_WIDTH_SIZE); break;
11675                 case i40e_bus_width_pcie_x2:
11676                         strncpy(width, "2", PCI_WIDTH_SIZE); break;
11677                 case i40e_bus_width_pcie_x1:
11678                         strncpy(width, "1", PCI_WIDTH_SIZE); break;
11679                 default:
11680                         break;
11681                 }
11682
11683                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
11684                          speed, width);
11685
11686                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
11687                     hw->bus.speed < i40e_bus_speed_8000) {
11688                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
11689                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
11690                 }
11691         }
11692
11693         /* get the requested speeds from the fw */
11694         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
11695         if (err)
11696                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
11697                         i40e_stat_str(&pf->hw, err),
11698                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11699         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
11700
11701         /* get the supported phy types from the fw */
11702         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
11703         if (err)
11704                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
11705                         i40e_stat_str(&pf->hw, err),
11706                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11707
11708         /* Add a filter to drop all Flow control frames from any VSI from being
11709          * transmitted. By doing so we stop a malicious VF from sending out
11710          * PAUSE or PFC frames and potentially controlling traffic for other
11711          * PF/VF VSIs.
11712          * The FW can still send Flow control frames if enabled.
11713          */
11714         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11715                                                        pf->main_vsi_seid);
11716
11717         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
11718                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
11719                 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
11720         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
11721                 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
11722         /* print a string summarizing features */
11723         i40e_print_features(pf);
11724
11725         return 0;
11726
11727         /* Unwind what we've done if something failed in the setup */
11728 err_vsis:
11729         set_bit(__I40E_DOWN, pf->state);
11730         i40e_clear_interrupt_scheme(pf);
11731         kfree(pf->vsi);
11732 err_switch_setup:
11733         i40e_reset_interrupt_capability(pf);
11734         del_timer_sync(&pf->service_timer);
11735 err_mac_addr:
11736 err_configure_lan_hmc:
11737         (void)i40e_shutdown_lan_hmc(hw);
11738 err_init_lan_hmc:
11739         kfree(pf->qp_pile);
11740 err_sw_init:
11741 err_adminq_setup:
11742 err_pf_reset:
11743         iounmap(hw->hw_addr);
11744 err_ioremap:
11745         kfree(pf);
11746 err_pf_alloc:
11747         pci_disable_pcie_error_reporting(pdev);
11748         pci_release_mem_regions(pdev);
11749 err_pci_reg:
11750 err_dma:
11751         pci_disable_device(pdev);
11752         return err;
11753 }
11754
11755 /**
11756  * i40e_remove - Device removal routine
11757  * @pdev: PCI device information struct
11758  *
11759  * i40e_remove is called by the PCI subsystem to alert the driver
11760  * that is should release a PCI device.  This could be caused by a
11761  * Hot-Plug event, or because the driver is going to be removed from
11762  * memory.
11763  **/
11764 static void i40e_remove(struct pci_dev *pdev)
11765 {
11766         struct i40e_pf *pf = pci_get_drvdata(pdev);
11767         struct i40e_hw *hw = &pf->hw;
11768         i40e_status ret_code;
11769         int i;
11770
11771         i40e_dbg_pf_exit(pf);
11772
11773         i40e_ptp_stop(pf);
11774
11775         /* Disable RSS in hw */
11776         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
11777         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
11778
11779         /* no more scheduling of any task */
11780         set_bit(__I40E_SUSPENDED, pf->state);
11781         set_bit(__I40E_DOWN, pf->state);
11782         if (pf->service_timer.data)
11783                 del_timer_sync(&pf->service_timer);
11784         if (pf->service_task.func)
11785                 cancel_work_sync(&pf->service_task);
11786
11787         /* Client close must be called explicitly here because the timer
11788          * has been stopped.
11789          */
11790         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
11791
11792         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
11793                 i40e_free_vfs(pf);
11794                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
11795         }
11796
11797         i40e_fdir_teardown(pf);
11798
11799         /* If there is a switch structure or any orphans, remove them.
11800          * This will leave only the PF's VSI remaining.
11801          */
11802         for (i = 0; i < I40E_MAX_VEB; i++) {
11803                 if (!pf->veb[i])
11804                         continue;
11805
11806                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
11807                     pf->veb[i]->uplink_seid == 0)
11808                         i40e_switch_branch_release(pf->veb[i]);
11809         }
11810
11811         /* Now we can shutdown the PF's VSI, just before we kill
11812          * adminq and hmc.
11813          */
11814         if (pf->vsi[pf->lan_vsi])
11815                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
11816
11817         /* remove attached clients */
11818         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11819                 ret_code = i40e_lan_del_device(pf);
11820                 if (ret_code)
11821                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
11822                                  ret_code);
11823         }
11824
11825         /* shutdown and destroy the HMC */
11826         if (hw->hmc.hmc_obj) {
11827                 ret_code = i40e_shutdown_lan_hmc(hw);
11828                 if (ret_code)
11829                         dev_warn(&pdev->dev,
11830                                  "Failed to destroy the HMC resources: %d\n",
11831                                  ret_code);
11832         }
11833
11834         /* shutdown the adminq */
11835         i40e_shutdown_adminq(hw);
11836
11837         /* destroy the locks only once, here */
11838         mutex_destroy(&hw->aq.arq_mutex);
11839         mutex_destroy(&hw->aq.asq_mutex);
11840
11841         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
11842         i40e_clear_interrupt_scheme(pf);
11843         for (i = 0; i < pf->num_alloc_vsi; i++) {
11844                 if (pf->vsi[i]) {
11845                         i40e_vsi_clear_rings(pf->vsi[i]);
11846                         i40e_vsi_clear(pf->vsi[i]);
11847                         pf->vsi[i] = NULL;
11848                 }
11849         }
11850
11851         for (i = 0; i < I40E_MAX_VEB; i++) {
11852                 kfree(pf->veb[i]);
11853                 pf->veb[i] = NULL;
11854         }
11855
11856         kfree(pf->qp_pile);
11857         kfree(pf->vsi);
11858
11859         iounmap(hw->hw_addr);
11860         kfree(pf);
11861         pci_release_mem_regions(pdev);
11862
11863         pci_disable_pcie_error_reporting(pdev);
11864         pci_disable_device(pdev);
11865 }
11866
11867 /**
11868  * i40e_pci_error_detected - warning that something funky happened in PCI land
11869  * @pdev: PCI device information struct
11870  *
11871  * Called to warn that something happened and the error handling steps
11872  * are in progress.  Allows the driver to quiesce things, be ready for
11873  * remediation.
11874  **/
11875 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
11876                                                 enum pci_channel_state error)
11877 {
11878         struct i40e_pf *pf = pci_get_drvdata(pdev);
11879
11880         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
11881
11882         if (!pf) {
11883                 dev_info(&pdev->dev,
11884                          "Cannot recover - error happened during device probe\n");
11885                 return PCI_ERS_RESULT_DISCONNECT;
11886         }
11887
11888         /* shutdown all operations */
11889         if (!test_bit(__I40E_SUSPENDED, pf->state))
11890                 i40e_prep_for_reset(pf, false);
11891
11892         /* Request a slot reset */
11893         return PCI_ERS_RESULT_NEED_RESET;
11894 }
11895
11896 /**
11897  * i40e_pci_error_slot_reset - a PCI slot reset just happened
11898  * @pdev: PCI device information struct
11899  *
11900  * Called to find if the driver can work with the device now that
11901  * the pci slot has been reset.  If a basic connection seems good
11902  * (registers are readable and have sane content) then return a
11903  * happy little PCI_ERS_RESULT_xxx.
11904  **/
11905 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
11906 {
11907         struct i40e_pf *pf = pci_get_drvdata(pdev);
11908         pci_ers_result_t result;
11909         int err;
11910         u32 reg;
11911
11912         dev_dbg(&pdev->dev, "%s\n", __func__);
11913         if (pci_enable_device_mem(pdev)) {
11914                 dev_info(&pdev->dev,
11915                          "Cannot re-enable PCI device after reset.\n");
11916                 result = PCI_ERS_RESULT_DISCONNECT;
11917         } else {
11918                 pci_set_master(pdev);
11919                 pci_restore_state(pdev);
11920                 pci_save_state(pdev);
11921                 pci_wake_from_d3(pdev, false);
11922
11923                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
11924                 if (reg == 0)
11925                         result = PCI_ERS_RESULT_RECOVERED;
11926                 else
11927                         result = PCI_ERS_RESULT_DISCONNECT;
11928         }
11929
11930         err = pci_cleanup_aer_uncorrect_error_status(pdev);
11931         if (err) {
11932                 dev_info(&pdev->dev,
11933                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
11934                          err);
11935                 /* non-fatal, continue */
11936         }
11937
11938         return result;
11939 }
11940
11941 /**
11942  * i40e_pci_error_resume - restart operations after PCI error recovery
11943  * @pdev: PCI device information struct
11944  *
11945  * Called to allow the driver to bring things back up after PCI error
11946  * and/or reset recovery has finished.
11947  **/
11948 static void i40e_pci_error_resume(struct pci_dev *pdev)
11949 {
11950         struct i40e_pf *pf = pci_get_drvdata(pdev);
11951
11952         dev_dbg(&pdev->dev, "%s\n", __func__);
11953         if (test_bit(__I40E_SUSPENDED, pf->state))
11954                 return;
11955
11956         i40e_handle_reset_warning(pf, false);
11957 }
11958
11959 /**
11960  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
11961  * using the mac_address_write admin q function
11962  * @pf: pointer to i40e_pf struct
11963  **/
11964 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
11965 {
11966         struct i40e_hw *hw = &pf->hw;
11967         i40e_status ret;
11968         u8 mac_addr[6];
11969         u16 flags = 0;
11970
11971         /* Get current MAC address in case it's an LAA */
11972         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
11973                 ether_addr_copy(mac_addr,
11974                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
11975         } else {
11976                 dev_err(&pf->pdev->dev,
11977                         "Failed to retrieve MAC address; using default\n");
11978                 ether_addr_copy(mac_addr, hw->mac.addr);
11979         }
11980
11981         /* The FW expects the mac address write cmd to first be called with
11982          * one of these flags before calling it again with the multicast
11983          * enable flags.
11984          */
11985         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
11986
11987         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
11988                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
11989
11990         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
11991         if (ret) {
11992                 dev_err(&pf->pdev->dev,
11993                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
11994                 return;
11995         }
11996
11997         flags = I40E_AQC_MC_MAG_EN
11998                         | I40E_AQC_WOL_PRESERVE_ON_PFR
11999                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
12000         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
12001         if (ret)
12002                 dev_err(&pf->pdev->dev,
12003                         "Failed to enable Multicast Magic Packet wake up\n");
12004 }
12005
12006 /**
12007  * i40e_shutdown - PCI callback for shutting down
12008  * @pdev: PCI device information struct
12009  **/
12010 static void i40e_shutdown(struct pci_dev *pdev)
12011 {
12012         struct i40e_pf *pf = pci_get_drvdata(pdev);
12013         struct i40e_hw *hw = &pf->hw;
12014
12015         set_bit(__I40E_SUSPENDED, pf->state);
12016         set_bit(__I40E_DOWN, pf->state);
12017         rtnl_lock();
12018         i40e_prep_for_reset(pf, true);
12019         rtnl_unlock();
12020
12021         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12022         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12023
12024         del_timer_sync(&pf->service_timer);
12025         cancel_work_sync(&pf->service_task);
12026         i40e_fdir_teardown(pf);
12027
12028         /* Client close must be called explicitly here because the timer
12029          * has been stopped.
12030          */
12031         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
12032
12033         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
12034                 i40e_enable_mc_magic_wake(pf);
12035
12036         i40e_prep_for_reset(pf, false);
12037
12038         wr32(hw, I40E_PFPM_APM,
12039              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12040         wr32(hw, I40E_PFPM_WUFC,
12041              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12042
12043         i40e_clear_interrupt_scheme(pf);
12044
12045         if (system_state == SYSTEM_POWER_OFF) {
12046                 pci_wake_from_d3(pdev, pf->wol_en);
12047                 pci_set_power_state(pdev, PCI_D3hot);
12048         }
12049 }
12050
12051 #ifdef CONFIG_PM
12052 /**
12053  * i40e_suspend - PM callback for moving to D3
12054  * @dev: generic device information structure
12055  **/
12056 static int i40e_suspend(struct device *dev)
12057 {
12058         struct pci_dev *pdev = to_pci_dev(dev);
12059         struct i40e_pf *pf = pci_get_drvdata(pdev);
12060         struct i40e_hw *hw = &pf->hw;
12061
12062         set_bit(__I40E_SUSPENDED, pf->state);
12063         set_bit(__I40E_DOWN, pf->state);
12064
12065         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
12066                 i40e_enable_mc_magic_wake(pf);
12067
12068         i40e_prep_for_reset(pf, false);
12069
12070         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12071         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12072
12073         i40e_free_misc_vector(pf);
12074
12075         return 0;
12076 }
12077
12078 /**
12079  * i40e_resume - PM callback for waking up from D3
12080  * @dev: generic device information structure
12081  **/
12082 static int i40e_resume(struct device *dev)
12083 {
12084         struct pci_dev *pdev = to_pci_dev(dev);
12085         struct i40e_pf *pf = pci_get_drvdata(pdev);
12086
12087         /* handling the reset will rebuild the device state */
12088         if (test_and_clear_bit(__I40E_SUSPENDED, pf->state)) {
12089                 clear_bit(__I40E_DOWN, pf->state);
12090                 i40e_reset_and_rebuild(pf, false, false);
12091         }
12092
12093         return 0;
12094 }
12095
12096 #endif /* CONFIG_PM */
12097
12098 static const struct pci_error_handlers i40e_err_handler = {
12099         .error_detected = i40e_pci_error_detected,
12100         .slot_reset = i40e_pci_error_slot_reset,
12101         .resume = i40e_pci_error_resume,
12102 };
12103
12104 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
12105
12106 static struct pci_driver i40e_driver = {
12107         .name     = i40e_driver_name,
12108         .id_table = i40e_pci_tbl,
12109         .probe    = i40e_probe,
12110         .remove   = i40e_remove,
12111 #ifdef CONFIG_PM
12112         .driver   = {
12113                 .pm = &i40e_pm_ops,
12114         },
12115 #endif /* CONFIG_PM */
12116         .shutdown = i40e_shutdown,
12117         .err_handler = &i40e_err_handler,
12118         .sriov_configure = i40e_pci_sriov_configure,
12119 };
12120
12121 /**
12122  * i40e_init_module - Driver registration routine
12123  *
12124  * i40e_init_module is the first routine called when the driver is
12125  * loaded. All it does is register with the PCI subsystem.
12126  **/
12127 static int __init i40e_init_module(void)
12128 {
12129         pr_info("%s: %s - version %s\n", i40e_driver_name,
12130                 i40e_driver_string, i40e_driver_version_str);
12131         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
12132
12133         /* There is no need to throttle the number of active tasks because
12134          * each device limits its own task using a state bit for scheduling
12135          * the service task, and the device tasks do not interfere with each
12136          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
12137          * since we need to be able to guarantee forward progress even under
12138          * memory pressure.
12139          */
12140         i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
12141         if (!i40e_wq) {
12142                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
12143                 return -ENOMEM;
12144         }
12145
12146         i40e_dbg_init();
12147         return pci_register_driver(&i40e_driver);
12148 }
12149 module_init(i40e_init_module);
12150
12151 /**
12152  * i40e_exit_module - Driver exit cleanup routine
12153  *
12154  * i40e_exit_module is called just before the driver is removed
12155  * from memory.
12156  **/
12157 static void __exit i40e_exit_module(void)
12158 {
12159         pci_unregister_driver(&i40e_driver);
12160         destroy_workqueue(i40e_wq);
12161         i40e_dbg_exit();
12162 }
12163 module_exit(i40e_exit_module);