drivers/net/ethernet/intel/i40e/i40e_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2021 Intel Corporation. */
   3
   4 #include <linux/etherdevice.h>
   5 #include <linux/of_net.h>
   6 #include <linux/pci.h>
   7 #include <linux/bpf.h>
   8 #include <generated/utsrelease.h>
   9
  10 /* Local includes */
  11 #include "i40e.h"
  12 #include "i40e_diag.h"
  13 #include "i40e_xsk.h"
  14 #include <net/udp_tunnel.h>
  15 #include <net/xdp_sock_drv.h>
  16 /* All i40e tracepoints are defined by the include below, which
  17  * must be included exactly once across the whole kernel with
  18  * CREATE_TRACE_POINTS defined
  19  */
  20 #define CREATE_TRACE_POINTS
  21 #include "i40e_trace.h"
  22
  23 const char i40e_driver_name[] = "i40e";
  24 static const char i40e_driver_string[] =
  25                         "Intel(R) Ethernet Connection XL710 Network Driver";
  26
  27 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
  28
  29 /* a bit of forward declarations */
  30 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  31 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  32 static int i40e_add_vsi(struct i40e_vsi *vsi);
  33 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  34 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  35 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  36 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  37 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  38 static void i40e_prep_for_reset(struct i40e_pf *pf);
  39 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
  40                                    bool lock_acquired);
  41 static int i40e_reset(struct i40e_pf *pf);
  42 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  43 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
  44 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
  45 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
  46 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
  47 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  48 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  49 static int i40e_get_capabilities(struct i40e_pf *pf,
  50                                  enum i40e_admin_queue_opc list_type);
  51 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
  52
  53 /* i40e_pci_tbl - PCI Device ID Table
  54  *
  55  * Last entry must be all 0s
  56  *
  57  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  58  *   Class, Class Mask, private data (not used) }
  59  */
  60 static const struct pci_device_id i40e_pci_tbl[] = {
  61         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  62         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  63         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  64         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  65         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  66         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  67         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  68         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  69         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  70         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
  71         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
  72         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
  73         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  77         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  78         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  79         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  80         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
  85         /* required last entry */
  86         {0, }
  87 };
  88 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  89
  90 #define I40E_MAX_VF_COUNT 128
  91 static int debug = -1;
  92 module_param(debug, uint, 0);
  93 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
  94
  95 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  96 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
  97 MODULE_LICENSE("GPL v2");
  98
  99 static struct workqueue_struct *i40e_wq;
 100
 101 /**
 102  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 103  * @hw:   pointer to the HW structure
 104  * @mem:  ptr to mem struct to fill out
 105  * @size: size of memory requested
 106  * @alignment: what to align the allocation to
 107  **/
 108 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 109                             u64 size, u32 alignment)
 110 {
 111         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 112
 113         mem->size = ALIGN(size, alignment);
 114         mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
 115                                      GFP_KERNEL);
 116         if (!mem->va)
 117                 return -ENOMEM;
 118
 119         return 0;
 120 }
 121
 122 /**
 123  * i40e_free_dma_mem_d - OS specific memory free for shared code
 124  * @hw:   pointer to the HW structure
 125  * @mem:  ptr to mem struct to free
 126  **/
 127 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 128 {
 129         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 130
 131         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 132         mem->va = NULL;
 133         mem->pa = 0;
 134         mem->size = 0;
 135
 136         return 0;
 137 }
 138
 139 /**
 140  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 141  * @hw:   pointer to the HW structure
 142  * @mem:  ptr to mem struct to fill out
 143  * @size: size of memory requested
 144  **/
 145 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 146                              u32 size)
 147 {
 148         mem->size = size;
 149         mem->va = kzalloc(size, GFP_KERNEL);
 150
 151         if (!mem->va)
 152                 return -ENOMEM;
 153
 154         return 0;
 155 }
 156
 157 /**
 158  * i40e_free_virt_mem_d - OS specific memory free for shared code
 159  * @hw:   pointer to the HW structure
 160  * @mem:  ptr to mem struct to free
 161  **/
 162 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 163 {
 164         /* it's ok to kfree a NULL pointer */
 165         kfree(mem->va);
 166         mem->va = NULL;
 167         mem->size = 0;
 168
 169         return 0;
 170 }
 171
 172 /**
 173  * i40e_get_lump - find a lump of free generic resource
 174  * @pf: board private structure
 175  * @pile: the pile of resource to search
 176  * @needed: the number of items needed
 177  * @id: an owner id to stick on the items assigned
 178  *
 179  * Returns the base item index of the lump, or negative for error
 180  *
 181  * The search_hint trick and lack of advanced fit-finding only work
 182  * because we're highly likely to have all the same size lump requests.
 183  * Linear search time and any fragmentation should be minimal.
 184  **/
 185 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 186                          u16 needed, u16 id)
 187 {
 188         int ret = -ENOMEM;
 189         int i, j;
 190
 191         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 192                 dev_info(&pf->pdev->dev,
 193                          "param err: pile=%s needed=%d id=0x%04x\n",
 194                          pile ? "<valid>" : "<null>", needed, id);
 195                 return -EINVAL;
 196         }
 197
 198         /* start the linear search with an imperfect hint */
 199         i = pile->search_hint;
 200         while (i < pile->num_entries) {
 201                 /* skip already allocated entries */
 202                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 203                         i++;
 204                         continue;
 205                 }
 206
 207                 /* do we have enough in this lump? */
 208                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 209                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 210                                 break;
 211                 }
 212
 213                 if (j == needed) {
 214                         /* there was enough, so assign it to the requestor */
 215                         for (j = 0; j < needed; j++)
 216                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 217                         ret = i;
 218                         pile->search_hint = i + j;
 219                         break;
 220                 }
 221
 222                 /* not enough, so skip over it and continue looking */
 223                 i += j;
 224         }
 225
 226         return ret;
 227 }
 228
 229 /**
 230  * i40e_put_lump - return a lump of generic resource
 231  * @pile: the pile of resource to search
 232  * @index: the base item index
 233  * @id: the owner id of the items assigned
 234  *
 235  * Returns the count of items in the lump
 236  **/
 237 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 238 {
 239         int valid_id = (id | I40E_PILE_VALID_BIT);
 240         int count = 0;
 241         int i;
 242
 243         if (!pile || index >= pile->num_entries)
 244                 return -EINVAL;
 245
 246         for (i = index;
 247              i < pile->num_entries && pile->list[i] == valid_id;
 248              i++) {
 249                 pile->list[i] = 0;
 250                 count++;
 251         }
 252
 253         if (count && index < pile->search_hint)
 254                 pile->search_hint = index;
 255
 256         return count;
 257 }
 258
 259 /**
 260  * i40e_find_vsi_from_id - searches for the vsi with the given id
 261  * @pf: the pf structure to search for the vsi
 262  * @id: id of the vsi it is searching for
 263  **/
 264 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 265 {
 266         int i;
 267
 268         for (i = 0; i < pf->num_alloc_vsi; i++)
 269                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 270                         return pf->vsi[i];
 271
 272         return NULL;
 273 }
 274
 275 /**
 276  * i40e_service_event_schedule - Schedule the service task to wake up
 277  * @pf: board private structure
 278  *
 279  * If not already scheduled, this puts the task into the work queue
 280  **/
 281 void i40e_service_event_schedule(struct i40e_pf *pf)
 282 {
 283         if ((!test_bit(__I40E_DOWN, pf->state) &&
 284              !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
 285               test_bit(__I40E_RECOVERY_MODE, pf->state))
 286                 queue_work(i40e_wq, &pf->service_task);
 287 }
 288
 289 /**
 290  * i40e_tx_timeout - Respond to a Tx Hang
 291  * @netdev: network interface device structure
 292  * @txqueue: queue number timing out
 293  *
 294  * If any port has noticed a Tx timeout, it is likely that the whole
 295  * device is munged, not just the one netdev port, so go for the full
 296  * reset.
 297  **/
 298 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 299 {
 300         struct i40e_netdev_priv *np = netdev_priv(netdev);
 301         struct i40e_vsi *vsi = np->vsi;
 302         struct i40e_pf *pf = vsi->back;
 303         struct i40e_ring *tx_ring = NULL;
 304         unsigned int i;
 305         u32 head, val;
 306
 307         pf->tx_timeout_count++;
 308
 309         /* with txqueue index, find the tx_ring struct */
 310         for (i = 0; i < vsi->num_queue_pairs; i++) {
 311                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 312                         if (txqueue ==
 313                             vsi->tx_rings[i]->queue_index) {
 314                                 tx_ring = vsi->tx_rings[i];
 315                                 break;
 316                         }
 317                 }
 318         }
 319
 320         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 321                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 322         else if (time_before(jiffies,
 323                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 324                 return;   /* don't do any new action before the next timeout */
 325
 326         /* don't kick off another recovery if one is already pending */
 327         if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
 328                 return;
 329
 330         if (tx_ring) {
 331                 head = i40e_get_head(tx_ring);
 332                 /* Read interrupt register */
 333                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 334                         val = rd32(&pf->hw,
 335                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 336                                                 tx_ring->vsi->base_vector - 1));
 337                 else
 338                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 339
 340                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 341                             vsi->seid, txqueue, tx_ring->next_to_clean,
 342                             head, tx_ring->next_to_use,
 343                             readl(tx_ring->tail), val);
 344         }
 345
 346         pf->tx_timeout_last_recovery = jiffies;
 347         netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",
 348                     pf->tx_timeout_recovery_level, txqueue);
 349
 350         switch (pf->tx_timeout_recovery_level) {
 351         case 1:
 352                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
 353                 break;
 354         case 2:
 355                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
 356                 break;
 357         case 3:
 358                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
 359                 break;
 360         default:
 361                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 362                 break;
 363         }
 364
 365         i40e_service_event_schedule(pf);
 366         pf->tx_timeout_recovery_level++;
 367 }
 368
 369 /**
 370  * i40e_get_vsi_stats_struct - Get System Network Statistics
 371  * @vsi: the VSI we care about
 372  *
 373  * Returns the address of the device statistics structure.
 374  * The statistics are actually updated from the service task.
 375  **/
 376 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 377 {
 378         return &vsi->net_stats;
 379 }
 380
 381 /**
 382  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
 383  * @ring: Tx ring to get statistics from
 384  * @stats: statistics entry to be updated
 385  **/
 386 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 387                                             struct rtnl_link_stats64 *stats)
 388 {
 389         u64 bytes, packets;
 390         unsigned int start;
 391
 392         do {
 393                 start = u64_stats_fetch_begin_irq(&ring->syncp);
 394                 packets = ring->stats.packets;
 395                 bytes   = ring->stats.bytes;
 396         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 397
 398         stats->tx_packets += packets;
 399         stats->tx_bytes   += bytes;
 400 }
 401
 402 /**
 403  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 404  * @netdev: network interface device structure
 405  * @stats: data structure to store statistics
 406  *
 407  * Returns the address of the device statistics structure.
 408  * The statistics are actually updated from the service task.
 409  **/
 410 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 411                                   struct rtnl_link_stats64 *stats)
 412 {
 413         struct i40e_netdev_priv *np = netdev_priv(netdev);
 414         struct i40e_vsi *vsi = np->vsi;
 415         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 416         struct i40e_ring *ring;
 417         int i;
 418
 419         if (test_bit(__I40E_VSI_DOWN, vsi->state))
 420                 return;
 421
 422         if (!vsi->tx_rings)
 423                 return;
 424
 425         rcu_read_lock();
 426         for (i = 0; i < vsi->num_queue_pairs; i++) {
 427                 u64 bytes, packets;
 428                 unsigned int start;
 429
 430                 ring = READ_ONCE(vsi->tx_rings[i]);
 431                 if (!ring)
 432                         continue;
 433                 i40e_get_netdev_stats_struct_tx(ring, stats);
 434
 435                 if (i40e_enabled_xdp_vsi(vsi)) {
 436                         ring = READ_ONCE(vsi->xdp_rings[i]);
 437                         if (!ring)
 438                                 continue;
 439                         i40e_get_netdev_stats_struct_tx(ring, stats);
 440                 }
 441
 442                 ring = READ_ONCE(vsi->rx_rings[i]);
 443                 if (!ring)
 444                         continue;
 445                 do {
 446                         start   = u64_stats_fetch_begin_irq(&ring->syncp);
 447                         packets = ring->stats.packets;
 448                         bytes   = ring->stats.bytes;
 449                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 450
 451                 stats->rx_packets += packets;
 452                 stats->rx_bytes   += bytes;
 453
 454         }
 455         rcu_read_unlock();
 456
 457         /* following stats updated by i40e_watchdog_subtask() */
 458         stats->multicast        = vsi_stats->multicast;
 459         stats->tx_errors        = vsi_stats->tx_errors;
 460         stats->tx_dropped       = vsi_stats->tx_dropped;
 461         stats->rx_errors        = vsi_stats->rx_errors;
 462         stats->rx_dropped       = vsi_stats->rx_dropped;
 463         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 464         stats->rx_length_errors = vsi_stats->rx_length_errors;
 465 }
 466
 467 /**
 468  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 469  * @vsi: the VSI to have its stats reset
 470  **/
 471 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 472 {
 473         struct rtnl_link_stats64 *ns;
 474         int i;
 475
 476         if (!vsi)
 477                 return;
 478
 479         ns = i40e_get_vsi_stats_struct(vsi);
 480         memset(ns, 0, sizeof(*ns));
 481         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 482         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 483         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 484         if (vsi->rx_rings && vsi->rx_rings[0]) {
 485                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 486                         memset(&vsi->rx_rings[i]->stats, 0,
 487                                sizeof(vsi->rx_rings[i]->stats));
 488                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 489                                sizeof(vsi->rx_rings[i]->rx_stats));
 490                         memset(&vsi->tx_rings[i]->stats, 0,
 491                                sizeof(vsi->tx_rings[i]->stats));
 492                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 493                                sizeof(vsi->tx_rings[i]->tx_stats));
 494                 }
 495         }
 496         vsi->stat_offsets_loaded = false;
 497 }
 498
 499 /**
 500  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 501  * @pf: the PF to be reset
 502  **/
 503 void i40e_pf_reset_stats(struct i40e_pf *pf)
 504 {
 505         int i;
 506
 507         memset(&pf->stats, 0, sizeof(pf->stats));
 508         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 509         pf->stat_offsets_loaded = false;
 510
 511         for (i = 0; i < I40E_MAX_VEB; i++) {
 512                 if (pf->veb[i]) {
 513                         memset(&pf->veb[i]->stats, 0,
 514                                sizeof(pf->veb[i]->stats));
 515                         memset(&pf->veb[i]->stats_offsets, 0,
 516                                sizeof(pf->veb[i]->stats_offsets));
 517                         memset(&pf->veb[i]->tc_stats, 0,
 518                                sizeof(pf->veb[i]->tc_stats));
 519                         memset(&pf->veb[i]->tc_stats_offsets, 0,
 520                                sizeof(pf->veb[i]->tc_stats_offsets));
 521                         pf->veb[i]->stat_offsets_loaded = false;
 522                 }
 523         }
 524         pf->hw_csum_rx_error = 0;
 525 }
 526
 527 /**
 528  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 529  * @hw: ptr to the hardware info
 530  * @hireg: the high 32 bit reg to read
 531  * @loreg: the low 32 bit reg to read
 532  * @offset_loaded: has the initial offset been loaded yet
 533  * @offset: ptr to current offset value
 534  * @stat: ptr to the stat
 535  *
 536  * Since the device stats are not reset at PFReset, they likely will not
 537  * be zeroed when the driver starts.  We'll save the first values read
 538  * and use them as offsets to be subtracted from the raw values in order
 539  * to report stats that count from zero.  In the process, we also manage
 540  * the potential roll-over.
 541  **/
 542 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 543                                bool offset_loaded, u64 *offset, u64 *stat)
 544 {
 545         u64 new_data;
 546
 547         if (hw->device_id == I40E_DEV_ID_QEMU) {
 548                 new_data = rd32(hw, loreg);
 549                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 550         } else {
 551                 new_data = rd64(hw, loreg);
 552         }
 553         if (!offset_loaded)
 554                 *offset = new_data;
 555         if (likely(new_data >= *offset))
 556                 *stat = new_data - *offset;
 557         else
 558                 *stat = (new_data + BIT_ULL(48)) - *offset;
 559         *stat &= 0xFFFFFFFFFFFFULL;
 560 }
 561
 562 /**
 563  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 564  * @hw: ptr to the hardware info
 565  * @reg: the hw reg to read
 566  * @offset_loaded: has the initial offset been loaded yet
 567  * @offset: ptr to current offset value
 568  * @stat: ptr to the stat
 569  **/
 570 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 571                                bool offset_loaded, u64 *offset, u64 *stat)
 572 {
 573         u32 new_data;
 574
 575         new_data = rd32(hw, reg);
 576         if (!offset_loaded)
 577                 *offset = new_data;
 578         if (likely(new_data >= *offset))
 579                 *stat = (u32)(new_data - *offset);
 580         else
 581                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 582 }
 583
 584 /**
 585  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
 586  * @hw: ptr to the hardware info
 587  * @reg: the hw reg to read and clear
 588  * @stat: ptr to the stat
 589  **/
 590 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
 591 {
 592         u32 new_data = rd32(hw, reg);
 593
 594         wr32(hw, reg, 1); /* must write a nonzero value to clear register */
 595         *stat += new_data;
 596 }
 597
 598 /**
 599  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 600  * @vsi: the VSI to be updated
 601  **/
 602 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 603 {
 604         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 605         struct i40e_pf *pf = vsi->back;
 606         struct i40e_hw *hw = &pf->hw;
 607         struct i40e_eth_stats *oes;
 608         struct i40e_eth_stats *es;     /* device's eth stats */
 609
 610         es = &vsi->eth_stats;
 611         oes = &vsi->eth_stats_offsets;
 612
 613         /* Gather up the stats that the hw collects */
 614         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 615                            vsi->stat_offsets_loaded,
 616                            &oes->tx_errors, &es->tx_errors);
 617         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 618                            vsi->stat_offsets_loaded,
 619                            &oes->rx_discards, &es->rx_discards);
 620         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 621                            vsi->stat_offsets_loaded,
 622                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 623
 624         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 625                            I40E_GLV_GORCL(stat_idx),
 626                            vsi->stat_offsets_loaded,
 627                            &oes->rx_bytes, &es->rx_bytes);
 628         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 629                            I40E_GLV_UPRCL(stat_idx),
 630                            vsi->stat_offsets_loaded,
 631                            &oes->rx_unicast, &es->rx_unicast);
 632         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 633                            I40E_GLV_MPRCL(stat_idx),
 634                            vsi->stat_offsets_loaded,
 635                            &oes->rx_multicast, &es->rx_multicast);
 636         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 637                            I40E_GLV_BPRCL(stat_idx),
 638                            vsi->stat_offsets_loaded,
 639                            &oes->rx_broadcast, &es->rx_broadcast);
 640
 641         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 642                            I40E_GLV_GOTCL(stat_idx),
 643                            vsi->stat_offsets_loaded,
 644                            &oes->tx_bytes, &es->tx_bytes);
 645         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 646                            I40E_GLV_UPTCL(stat_idx),
 647                            vsi->stat_offsets_loaded,
 648                            &oes->tx_unicast, &es->tx_unicast);
 649         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 650                            I40E_GLV_MPTCL(stat_idx),
 651                            vsi->stat_offsets_loaded,
 652                            &oes->tx_multicast, &es->tx_multicast);
 653         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 654                            I40E_GLV_BPTCL(stat_idx),
 655                            vsi->stat_offsets_loaded,
 656                            &oes->tx_broadcast, &es->tx_broadcast);
 657         vsi->stat_offsets_loaded = true;
 658 }
 659
 660 /**
 661  * i40e_update_veb_stats - Update Switch component statistics
 662  * @veb: the VEB being updated
 663  **/
 664 void i40e_update_veb_stats(struct i40e_veb *veb)
 665 {
 666         struct i40e_pf *pf = veb->pf;
 667         struct i40e_hw *hw = &pf->hw;
 668         struct i40e_eth_stats *oes;
 669         struct i40e_eth_stats *es;     /* device's eth stats */
 670         struct i40e_veb_tc_stats *veb_oes;
 671         struct i40e_veb_tc_stats *veb_es;
 672         int i, idx = 0;
 673
 674         idx = veb->stats_idx;
 675         es = &veb->stats;
 676         oes = &veb->stats_offsets;
 677         veb_es = &veb->tc_stats;
 678         veb_oes = &veb->tc_stats_offsets;
 679
 680         /* Gather up the stats that the hw collects */
 681         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 682                            veb->stat_offsets_loaded,
 683                            &oes->tx_discards, &es->tx_discards);
 684         if (hw->revision_id > 0)
 685                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 686                                    veb->stat_offsets_loaded,
 687                                    &oes->rx_unknown_protocol,
 688                                    &es->rx_unknown_protocol);
 689         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 690                            veb->stat_offsets_loaded,
 691                            &oes->rx_bytes, &es->rx_bytes);
 692         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 693                            veb->stat_offsets_loaded,
 694                            &oes->rx_unicast, &es->rx_unicast);
 695         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 696                            veb->stat_offsets_loaded,
 697                            &oes->rx_multicast, &es->rx_multicast);
 698         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 699                            veb->stat_offsets_loaded,
 700                            &oes->rx_broadcast, &es->rx_broadcast);
 701
 702         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 703                            veb->stat_offsets_loaded,
 704                            &oes->tx_bytes, &es->tx_bytes);
 705         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 706                            veb->stat_offsets_loaded,
 707                            &oes->tx_unicast, &es->tx_unicast);
 708         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 709                            veb->stat_offsets_loaded,
 710                            &oes->tx_multicast, &es->tx_multicast);
 711         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 712                            veb->stat_offsets_loaded,
 713                            &oes->tx_broadcast, &es->tx_broadcast);
 714         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 715                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 716                                    I40E_GLVEBTC_RPCL(i, idx),
 717                                    veb->stat_offsets_loaded,
 718                                    &veb_oes->tc_rx_packets[i],
 719                                    &veb_es->tc_rx_packets[i]);
 720                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 721                                    I40E_GLVEBTC_RBCL(i, idx),
 722                                    veb->stat_offsets_loaded,
 723                                    &veb_oes->tc_rx_bytes[i],
 724                                    &veb_es->tc_rx_bytes[i]);
 725                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 726                                    I40E_GLVEBTC_TPCL(i, idx),
 727                                    veb->stat_offsets_loaded,
 728                                    &veb_oes->tc_tx_packets[i],
 729                                    &veb_es->tc_tx_packets[i]);
 730                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 731                                    I40E_GLVEBTC_TBCL(i, idx),
 732                                    veb->stat_offsets_loaded,
 733                                    &veb_oes->tc_tx_bytes[i],
 734                                    &veb_es->tc_tx_bytes[i]);
 735         }
 736         veb->stat_offsets_loaded = true;
 737 }
 738
 739 /**
 740  * i40e_update_vsi_stats - Update the vsi statistics counters.
 741  * @vsi: the VSI to be updated
 742  *
 743  * There are a few instances where we store the same stat in a
 744  * couple of different structs.  This is partly because we have
 745  * the netdev stats that need to be filled out, which is slightly
 746  * different from the "eth_stats" defined by the chip and used in
 747  * VF communications.  We sort it out here.
 748  **/
 749 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 750 {
 751         struct i40e_pf *pf = vsi->back;
 752         struct rtnl_link_stats64 *ons;
 753         struct rtnl_link_stats64 *ns;   /* netdev stats */
 754         struct i40e_eth_stats *oes;
 755         struct i40e_eth_stats *es;     /* device's eth stats */
 756         u32 tx_restart, tx_busy;
 757         struct i40e_ring *p;
 758         u32 rx_page, rx_buf;
 759         u64 bytes, packets;
 760         unsigned int start;
 761         u64 tx_linearize;
 762         u64 tx_force_wb;
 763         u64 rx_p, rx_b;
 764         u64 tx_p, tx_b;
 765         u16 q;
 766
 767         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
 768             test_bit(__I40E_CONFIG_BUSY, pf->state))
 769                 return;
 770
 771         ns = i40e_get_vsi_stats_struct(vsi);
 772         ons = &vsi->net_stats_offsets;
 773         es = &vsi->eth_stats;
 774         oes = &vsi->eth_stats_offsets;
 775
 776         /* Gather up the netdev and vsi stats that the driver collects
 777          * on the fly during packet processing
 778          */
 779         rx_b = rx_p = 0;
 780         tx_b = tx_p = 0;
 781         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 782         rx_page = 0;
 783         rx_buf = 0;
 784         rcu_read_lock();
 785         for (q = 0; q < vsi->num_queue_pairs; q++) {
 786                 /* locate Tx ring */
 787                 p = READ_ONCE(vsi->tx_rings[q]);
 788                 if (!p)
 789                         continue;
 790
 791                 do {
 792                         start = u64_stats_fetch_begin_irq(&p->syncp);
 793                         packets = p->stats.packets;
 794                         bytes = p->stats.bytes;
 795                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 796                 tx_b += bytes;
 797                 tx_p += packets;
 798                 tx_restart += p->tx_stats.restart_queue;
 799                 tx_busy += p->tx_stats.tx_busy;
 800                 tx_linearize += p->tx_stats.tx_linearize;
 801                 tx_force_wb += p->tx_stats.tx_force_wb;
 802
 803                 /* locate Rx ring */
 804                 p = READ_ONCE(vsi->rx_rings[q]);
 805                 if (!p)
 806                         continue;
 807
 808                 do {
 809                         start = u64_stats_fetch_begin_irq(&p->syncp);
 810                         packets = p->stats.packets;
 811                         bytes = p->stats.bytes;
 812                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 813                 rx_b += bytes;
 814                 rx_p += packets;
 815                 rx_buf += p->rx_stats.alloc_buff_failed;
 816                 rx_page += p->rx_stats.alloc_page_failed;
 817
 818                 if (i40e_enabled_xdp_vsi(vsi)) {
 819                         /* locate XDP ring */
 820                         p = READ_ONCE(vsi->xdp_rings[q]);
 821                         if (!p)
 822                                 continue;
 823
 824                         do {
 825                                 start = u64_stats_fetch_begin_irq(&p->syncp);
 826                                 packets = p->stats.packets;
 827                                 bytes = p->stats.bytes;
 828                         } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 829                         tx_b += bytes;
 830                         tx_p += packets;
 831                         tx_restart += p->tx_stats.restart_queue;
 832                         tx_busy += p->tx_stats.tx_busy;
 833                         tx_linearize += p->tx_stats.tx_linearize;
 834                         tx_force_wb += p->tx_stats.tx_force_wb;
 835                 }
 836         }
 837         rcu_read_unlock();
 838         vsi->tx_restart = tx_restart;
 839         vsi->tx_busy = tx_busy;
 840         vsi->tx_linearize = tx_linearize;
 841         vsi->tx_force_wb = tx_force_wb;
 842         vsi->rx_page_failed = rx_page;
 843         vsi->rx_buf_failed = rx_buf;
 844
 845         ns->rx_packets = rx_p;
 846         ns->rx_bytes = rx_b;
 847         ns->tx_packets = tx_p;
 848         ns->tx_bytes = tx_b;
 849
 850         /* update netdev stats from eth stats */
 851         i40e_update_eth_stats(vsi);
 852         ons->tx_errors = oes->tx_errors;
 853         ns->tx_errors = es->tx_errors;
 854         ons->multicast = oes->rx_multicast;
 855         ns->multicast = es->rx_multicast;
 856         ons->rx_dropped = oes->rx_discards;
 857         ns->rx_dropped = es->rx_discards;
 858         ons->tx_dropped = oes->tx_discards;
 859         ns->tx_dropped = es->tx_discards;
 860
 861         /* pull in a couple PF stats if this is the main vsi */
 862         if (vsi == pf->vsi[pf->lan_vsi]) {
 863                 ns->rx_crc_errors = pf->stats.crc_errors;
 864                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 865                 ns->rx_length_errors = pf->stats.rx_length_errors;
 866         }
 867 }
 868
 869 /**
 870  * i40e_update_pf_stats - Update the PF statistics counters.
 871  * @pf: the PF to be updated
 872  **/
 873 static void i40e_update_pf_stats(struct i40e_pf *pf)
 874 {
 875         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 876         struct i40e_hw_port_stats *nsd = &pf->stats;
 877         struct i40e_hw *hw = &pf->hw;
 878         u32 val;
 879         int i;
 880
 881         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 882                            I40E_GLPRT_GORCL(hw->port),
 883                            pf->stat_offsets_loaded,
 884                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 885         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 886                            I40E_GLPRT_GOTCL(hw->port),
 887                            pf->stat_offsets_loaded,
 888                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 889         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 890                            pf->stat_offsets_loaded,
 891                            &osd->eth.rx_discards,
 892                            &nsd->eth.rx_discards);
 893         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 894                            I40E_GLPRT_UPRCL(hw->port),
 895                            pf->stat_offsets_loaded,
 896                            &osd->eth.rx_unicast,
 897                            &nsd->eth.rx_unicast);
 898         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 899                            I40E_GLPRT_MPRCL(hw->port),
 900                            pf->stat_offsets_loaded,
 901                            &osd->eth.rx_multicast,
 902                            &nsd->eth.rx_multicast);
 903         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 904                            I40E_GLPRT_BPRCL(hw->port),
 905                            pf->stat_offsets_loaded,
 906                            &osd->eth.rx_broadcast,
 907                            &nsd->eth.rx_broadcast);
 908         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 909                            I40E_GLPRT_UPTCL(hw->port),
 910                            pf->stat_offsets_loaded,
 911                            &osd->eth.tx_unicast,
 912                            &nsd->eth.tx_unicast);
 913         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 914                            I40E_GLPRT_MPTCL(hw->port),
 915                            pf->stat_offsets_loaded,
 916                            &osd->eth.tx_multicast,
 917                            &nsd->eth.tx_multicast);
 918         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 919                            I40E_GLPRT_BPTCL(hw->port),
 920                            pf->stat_offsets_loaded,
 921                            &osd->eth.tx_broadcast,
 922                            &nsd->eth.tx_broadcast);
 923
 924         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 925                            pf->stat_offsets_loaded,
 926                            &osd->tx_dropped_link_down,
 927                            &nsd->tx_dropped_link_down);
 928
 929         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 930                            pf->stat_offsets_loaded,
 931                            &osd->crc_errors, &nsd->crc_errors);
 932
 933         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 934                            pf->stat_offsets_loaded,
 935                            &osd->illegal_bytes, &nsd->illegal_bytes);
 936
 937         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 938                            pf->stat_offsets_loaded,
 939                            &osd->mac_local_faults,
 940                            &nsd->mac_local_faults);
 941         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 942                            pf->stat_offsets_loaded,
 943                            &osd->mac_remote_faults,
 944                            &nsd->mac_remote_faults);
 945
 946         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 947                            pf->stat_offsets_loaded,
 948                            &osd->rx_length_errors,
 949                            &nsd->rx_length_errors);
 950
 951         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 952                            pf->stat_offsets_loaded,
 953                            &osd->link_xon_rx, &nsd->link_xon_rx);
 954         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 955                            pf->stat_offsets_loaded,
 956                            &osd->link_xon_tx, &nsd->link_xon_tx);
 957         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 958                            pf->stat_offsets_loaded,
 959                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 960         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 961                            pf->stat_offsets_loaded,
 962                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 963
 964         for (i = 0; i < 8; i++) {
 965                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 966                                    pf->stat_offsets_loaded,
 967                                    &osd->priority_xoff_rx[i],
 968                                    &nsd->priority_xoff_rx[i]);
 969                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 970                                    pf->stat_offsets_loaded,
 971                                    &osd->priority_xon_rx[i],
 972                                    &nsd->priority_xon_rx[i]);
 973                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 974                                    pf->stat_offsets_loaded,
 975                                    &osd->priority_xon_tx[i],
 976                                    &nsd->priority_xon_tx[i]);
 977                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 978                                    pf->stat_offsets_loaded,
 979                                    &osd->priority_xoff_tx[i],
 980                                    &nsd->priority_xoff_tx[i]);
 981                 i40e_stat_update32(hw,
 982                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 983                                    pf->stat_offsets_loaded,
 984                                    &osd->priority_xon_2_xoff[i],
 985                                    &nsd->priority_xon_2_xoff[i]);
 986         }
 987
 988         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 989                            I40E_GLPRT_PRC64L(hw->port),
 990                            pf->stat_offsets_loaded,
 991                            &osd->rx_size_64, &nsd->rx_size_64);
 992         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 993                            I40E_GLPRT_PRC127L(hw->port),
 994                            pf->stat_offsets_loaded,
 995                            &osd->rx_size_127, &nsd->rx_size_127);
 996         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 997                            I40E_GLPRT_PRC255L(hw->port),
 998                            pf->stat_offsets_loaded,
 999                            &osd->rx_size_255, &nsd->rx_size_255);
1000         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1001                            I40E_GLPRT_PRC511L(hw->port),
1002                            pf->stat_offsets_loaded,
1003                            &osd->rx_size_511, &nsd->rx_size_511);
1004         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1005                            I40E_GLPRT_PRC1023L(hw->port),
1006                            pf->stat_offsets_loaded,
1007                            &osd->rx_size_1023, &nsd->rx_size_1023);
1008         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1009                            I40E_GLPRT_PRC1522L(hw->port),
1010                            pf->stat_offsets_loaded,
1011                            &osd->rx_size_1522, &nsd->rx_size_1522);
1012         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1013                            I40E_GLPRT_PRC9522L(hw->port),
1014                            pf->stat_offsets_loaded,
1015                            &osd->rx_size_big, &nsd->rx_size_big);
1016
1017         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1018                            I40E_GLPRT_PTC64L(hw->port),
1019                            pf->stat_offsets_loaded,
1020                            &osd->tx_size_64, &nsd->tx_size_64);
1021         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1022                            I40E_GLPRT_PTC127L(hw->port),
1023                            pf->stat_offsets_loaded,
1024                            &osd->tx_size_127, &nsd->tx_size_127);
1025         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1026                            I40E_GLPRT_PTC255L(hw->port),
1027                            pf->stat_offsets_loaded,
1028                            &osd->tx_size_255, &nsd->tx_size_255);
1029         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1030                            I40E_GLPRT_PTC511L(hw->port),
1031                            pf->stat_offsets_loaded,
1032                            &osd->tx_size_511, &nsd->tx_size_511);
1033         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1034                            I40E_GLPRT_PTC1023L(hw->port),
1035                            pf->stat_offsets_loaded,
1036                            &osd->tx_size_1023, &nsd->tx_size_1023);
1037         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1038                            I40E_GLPRT_PTC1522L(hw->port),
1039                            pf->stat_offsets_loaded,
1040                            &osd->tx_size_1522, &nsd->tx_size_1522);
1041         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1042                            I40E_GLPRT_PTC9522L(hw->port),
1043                            pf->stat_offsets_loaded,
1044                            &osd->tx_size_big, &nsd->tx_size_big);
1045
1046         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1047                            pf->stat_offsets_loaded,
1048                            &osd->rx_undersize, &nsd->rx_undersize);
1049         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1050                            pf->stat_offsets_loaded,
1051                            &osd->rx_fragments, &nsd->rx_fragments);
1052         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1053                            pf->stat_offsets_loaded,
1054                            &osd->rx_oversize, &nsd->rx_oversize);
1055         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1056                            pf->stat_offsets_loaded,
1057                            &osd->rx_jabber, &nsd->rx_jabber);
1058
1059         /* FDIR stats */
1060         i40e_stat_update_and_clear32(hw,
1061                         I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1062                         &nsd->fd_atr_match);
1063         i40e_stat_update_and_clear32(hw,
1064                         I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1065                         &nsd->fd_sb_match);
1066         i40e_stat_update_and_clear32(hw,
1067                         I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1068                         &nsd->fd_atr_tunnel_match);
1069
1070         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1071         nsd->tx_lpi_status =
1072                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1073                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1074         nsd->rx_lpi_status =
1075                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1076                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1077         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1078                            pf->stat_offsets_loaded,
1079                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1080         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1081                            pf->stat_offsets_loaded,
1082                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1083
1084         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1085             !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1086                 nsd->fd_sb_status = true;
1087         else
1088                 nsd->fd_sb_status = false;
1089
1090         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1091             !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1092                 nsd->fd_atr_status = true;
1093         else
1094                 nsd->fd_atr_status = false;
1095
1096         pf->stat_offsets_loaded = true;
1097 }
1098
1099 /**
1100  * i40e_update_stats - Update the various statistics counters.
1101  * @vsi: the VSI to be updated
1102  *
1103  * Update the various stats for this VSI and its related entities.
1104  **/
1105 void i40e_update_stats(struct i40e_vsi *vsi)
1106 {
1107         struct i40e_pf *pf = vsi->back;
1108
1109         if (vsi == pf->vsi[pf->lan_vsi])
1110                 i40e_update_pf_stats(pf);
1111
1112         i40e_update_vsi_stats(vsi);
1113 }
1114
1115 /**
1116  * i40e_count_filters - counts VSI mac filters
1117  * @vsi: the VSI to be searched
1118  *
1119  * Returns count of mac filters
1120  **/
1121 int i40e_count_filters(struct i40e_vsi *vsi)
1122 {
1123         struct i40e_mac_filter *f;
1124         struct hlist_node *h;
1125         int bkt;
1126         int cnt = 0;
1127
1128         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
1129                 ++cnt;
1130
1131         return cnt;
1132 }
1133
1134 /**
1135  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1136  * @vsi: the VSI to be searched
1137  * @macaddr: the MAC address
1138  * @vlan: the vlan
1139  *
1140  * Returns ptr to the filter object or NULL
1141  **/
1142 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1143                                                 const u8 *macaddr, s16 vlan)
1144 {
1145         struct i40e_mac_filter *f;
1146         u64 key;
1147
1148         if (!vsi || !macaddr)
1149                 return NULL;
1150
1151         key = i40e_addr_to_hkey(macaddr);
1152         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1153                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1154                     (vlan == f->vlan))
1155                         return f;
1156         }
1157         return NULL;
1158 }
1159
1160 /**
1161  * i40e_find_mac - Find a mac addr in the macvlan filters list
1162  * @vsi: the VSI to be searched
1163  * @macaddr: the MAC address we are searching for
1164  *
1165  * Returns the first filter with the provided MAC address or NULL if
1166  * MAC address was not found
1167  **/
1168 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1169 {
1170         struct i40e_mac_filter *f;
1171         u64 key;
1172
1173         if (!vsi || !macaddr)
1174                 return NULL;
1175
1176         key = i40e_addr_to_hkey(macaddr);
1177         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1178                 if ((ether_addr_equal(macaddr, f->macaddr)))
1179                         return f;
1180         }
1181         return NULL;
1182 }
1183
1184 /**
1185  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1186  * @vsi: the VSI to be searched
1187  *
1188  * Returns true if VSI is in vlan mode or false otherwise
1189  **/
1190 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1191 {
1192         /* If we have a PVID, always operate in VLAN mode */
1193         if (vsi->info.pvid)
1194                 return true;
1195
1196         /* We need to operate in VLAN mode whenever we have any filters with
1197          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1198          * time, incurring search cost repeatedly. However, we can notice two
1199          * things:
1200          *
1201          * 1) the only place where we can gain a VLAN filter is in
1202          *    i40e_add_filter.
1203          *
1204          * 2) the only place where filters are actually removed is in
1205          *    i40e_sync_filters_subtask.
1206          *
1207          * Thus, we can simply use a boolean value, has_vlan_filters which we
1208          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1209          * we have to perform the full search after deleting filters in
1210          * i40e_sync_filters_subtask, but we already have to search
1211          * filters here and can perform the check at the same time. This
1212          * results in avoiding embedding a loop for VLAN mode inside another
1213          * loop over all the filters, and should maintain correctness as noted
1214          * above.
1215          */
1216         return vsi->has_vlan_filter;
1217 }
1218
1219 /**
1220  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1221  * @vsi: the VSI to configure
1222  * @tmp_add_list: list of filters ready to be added
1223  * @tmp_del_list: list of filters ready to be deleted
1224  * @vlan_filters: the number of active VLAN filters
1225  *
1226  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1227  * behave as expected. If we have any active VLAN filters remaining or about
1228  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1229  * so that they only match against untagged traffic. If we no longer have any
1230  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1231  * so that they match against both tagged and untagged traffic. In this way,
1232  * we ensure that we correctly receive the desired traffic. This ensures that
1233  * when we have an active VLAN we will receive only untagged traffic and
1234  * traffic matching active VLANs. If we have no active VLANs then we will
1235  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1236  *
1237  * Finally, in a similar fashion, this function also corrects filters when
1238  * there is an active PVID assigned to this VSI.
1239  *
1240  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1241  *
1242  * This function is only expected to be called from within
1243  * i40e_sync_vsi_filters.
1244  *
1245  * NOTE: This function expects to be called while under the
1246  * mac_filter_hash_lock
1247  */
1248 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1249                                          struct hlist_head *tmp_add_list,
1250                                          struct hlist_head *tmp_del_list,
1251                                          int vlan_filters)
1252 {
1253         s16 pvid = le16_to_cpu(vsi->info.pvid);
1254         struct i40e_mac_filter *f, *add_head;
1255         struct i40e_new_mac_filter *new;
1256         struct hlist_node *h;
1257         int bkt, new_vlan;
1258
1259         /* To determine if a particular filter needs to be replaced we
1260          * have the three following conditions:
1261          *
1262          * a) if we have a PVID assigned, then all filters which are
1263          *    not marked as VLAN=PVID must be replaced with filters that
1264          *    are.
1265          * b) otherwise, if we have any active VLANS, all filters
1266          *    which are marked as VLAN=-1 must be replaced with
1267          *    filters marked as VLAN=0
1268          * c) finally, if we do not have any active VLANS, all filters
1269          *    which are marked as VLAN=0 must be replaced with filters
1270          *    marked as VLAN=-1
1271          */
1272
1273         /* Update the filters about to be added in place */
1274         hlist_for_each_entry(new, tmp_add_list, hlist) {
1275                 if (pvid && new->f->vlan != pvid)
1276                         new->f->vlan = pvid;
1277                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1278                         new->f->vlan = 0;
1279                 else if (!vlan_filters && new->f->vlan == 0)
1280                         new->f->vlan = I40E_VLAN_ANY;
1281         }
1282
1283         /* Update the remaining active filters */
1284         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1285                 /* Combine the checks for whether a filter needs to be changed
1286                  * and then determine the new VLAN inside the if block, in
1287                  * order to avoid duplicating code for adding the new filter
1288                  * then deleting the old filter.
1289                  */
1290                 if ((pvid && f->vlan != pvid) ||
1291                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1292                     (!vlan_filters && f->vlan == 0)) {
1293                         /* Determine the new vlan we will be adding */
1294                         if (pvid)
1295                                 new_vlan = pvid;
1296                         else if (vlan_filters)
1297                                 new_vlan = 0;
1298                         else
1299                                 new_vlan = I40E_VLAN_ANY;
1300
1301                         /* Create the new filter */
1302                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1303                         if (!add_head)
1304                                 return -ENOMEM;
1305
1306                         /* Create a temporary i40e_new_mac_filter */
1307                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1308                         if (!new)
1309                                 return -ENOMEM;
1310
1311                         new->f = add_head;
1312                         new->state = add_head->state;
1313
1314                         /* Add the new filter to the tmp list */
1315                         hlist_add_head(&new->hlist, tmp_add_list);
1316
1317                         /* Put the original filter into the delete list */
1318                         f->state = I40E_FILTER_REMOVE;
1319                         hash_del(&f->hlist);
1320                         hlist_add_head(&f->hlist, tmp_del_list);
1321                 }
1322         }
1323
1324         vsi->has_vlan_filter = !!vlan_filters;
1325
1326         return 0;
1327 }
1328
1329 /**
1330  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1331  * @vsi: the PF Main VSI - inappropriate for any other VSI
1332  * @macaddr: the MAC address
1333  *
1334  * Remove whatever filter the firmware set up so the driver can manage
1335  * its own filtering intelligently.
1336  **/
1337 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1338 {
1339         struct i40e_aqc_remove_macvlan_element_data element;
1340         struct i40e_pf *pf = vsi->back;
1341
1342         /* Only appropriate for the PF main VSI */
1343         if (vsi->type != I40E_VSI_MAIN)
1344                 return;
1345
1346         memset(&element, 0, sizeof(element));
1347         ether_addr_copy(element.mac_addr, macaddr);
1348         element.vlan_tag = 0;
1349         /* Ignore error returns, some firmware does it this way... */
1350         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1351         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1352
1353         memset(&element, 0, sizeof(element));
1354         ether_addr_copy(element.mac_addr, macaddr);
1355         element.vlan_tag = 0;
1356         /* ...and some firmware does it this way. */
1357         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1358                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1359         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1360 }
1361
1362 /**
1363  * i40e_add_filter - Add a mac/vlan filter to the VSI
1364  * @vsi: the VSI to be searched
1365  * @macaddr: the MAC address
1366  * @vlan: the vlan
1367  *
1368  * Returns ptr to the filter object or NULL when no memory available.
1369  *
1370  * NOTE: This function is expected to be called with mac_filter_hash_lock
1371  * being held.
1372  **/
1373 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1374                                         const u8 *macaddr, s16 vlan)
1375 {
1376         struct i40e_mac_filter *f;
1377         u64 key;
1378
1379         if (!vsi || !macaddr)
1380                 return NULL;
1381
1382         f = i40e_find_filter(vsi, macaddr, vlan);
1383         if (!f) {
1384                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1385                 if (!f)
1386                         return NULL;
1387
1388                 /* Update the boolean indicating if we need to function in
1389                  * VLAN mode.
1390                  */
1391                 if (vlan >= 0)
1392                         vsi->has_vlan_filter = true;
1393
1394                 ether_addr_copy(f->macaddr, macaddr);
1395                 f->vlan = vlan;
1396                 f->state = I40E_FILTER_NEW;
1397                 INIT_HLIST_NODE(&f->hlist);
1398
1399                 key = i40e_addr_to_hkey(macaddr);
1400                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1401
1402                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1403                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1404         }
1405
1406         /* If we're asked to add a filter that has been marked for removal, it
1407          * is safe to simply restore it to active state. __i40e_del_filter
1408          * will have simply deleted any filters which were previously marked
1409          * NEW or FAILED, so if it is currently marked REMOVE it must have
1410          * previously been ACTIVE. Since we haven't yet run the sync filters
1411          * task, just restore this filter to the ACTIVE state so that the
1412          * sync task leaves it in place
1413          */
1414         if (f->state == I40E_FILTER_REMOVE)
1415                 f->state = I40E_FILTER_ACTIVE;
1416
1417         return f;
1418 }
1419
1420 /**
1421  * __i40e_del_filter - Remove a specific filter from the VSI
1422  * @vsi: VSI to remove from
1423  * @f: the filter to remove from the list
1424  *
1425  * This function should be called instead of i40e_del_filter only if you know
1426  * the exact filter you will remove already, such as via i40e_find_filter or
1427  * i40e_find_mac.
1428  *
1429  * NOTE: This function is expected to be called with mac_filter_hash_lock
1430  * being held.
1431  * ANOTHER NOTE: This function MUST be called from within the context of
1432  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1433  * instead of list_for_each_entry().
1434  **/
1435 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1436 {
1437         if (!f)
1438                 return;
1439
1440         /* If the filter was never added to firmware then we can just delete it
1441          * directly and we don't want to set the status to remove or else an
1442          * admin queue command will unnecessarily fire.
1443          */
1444         if ((f->state == I40E_FILTER_FAILED) ||
1445             (f->state == I40E_FILTER_NEW)) {
1446                 hash_del(&f->hlist);
1447                 kfree(f);
1448         } else {
1449                 f->state = I40E_FILTER_REMOVE;
1450         }
1451
1452         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1453         set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1454 }
1455
1456 /**
1457  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1458  * @vsi: the VSI to be searched
1459  * @macaddr: the MAC address
1460  * @vlan: the VLAN
1461  *
1462  * NOTE: This function is expected to be called with mac_filter_hash_lock
1463  * being held.
1464  * ANOTHER NOTE: This function MUST be called from within the context of
1465  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1466  * instead of list_for_each_entry().
1467  **/
1468 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1469 {
1470         struct i40e_mac_filter *f;
1471
1472         if (!vsi || !macaddr)
1473                 return;
1474
1475         f = i40e_find_filter(vsi, macaddr, vlan);
1476         __i40e_del_filter(vsi, f);
1477 }
1478
1479 /**
1480  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1481  * @vsi: the VSI to be searched
1482  * @macaddr: the mac address to be filtered
1483  *
1484  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1485  * go through all the macvlan filters and add a macvlan filter for each
1486  * unique vlan that already exists. If a PVID has been assigned, instead only
1487  * add the macaddr to that VLAN.
1488  *
1489  * Returns last filter added on success, else NULL
1490  **/
1491 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1492                                             const u8 *macaddr)
1493 {
1494         struct i40e_mac_filter *f, *add = NULL;
1495         struct hlist_node *h;
1496         int bkt;
1497
1498         if (vsi->info.pvid)
1499                 return i40e_add_filter(vsi, macaddr,
1500                                        le16_to_cpu(vsi->info.pvid));
1501
1502         if (!i40e_is_vsi_in_vlan(vsi))
1503                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1504
1505         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1506                 if (f->state == I40E_FILTER_REMOVE)
1507                         continue;
1508                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1509                 if (!add)
1510                         return NULL;
1511         }
1512
1513         return add;
1514 }
1515
1516 /**
1517  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1518  * @vsi: the VSI to be searched
1519  * @macaddr: the mac address to be removed
1520  *
1521  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1522  * associated with.
1523  *
1524  * Returns 0 for success, or error
1525  **/
1526 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1527 {
1528         struct i40e_mac_filter *f;
1529         struct hlist_node *h;
1530         bool found = false;
1531         int bkt;
1532
1533         lockdep_assert_held(&vsi->mac_filter_hash_lock);
1534         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1535                 if (ether_addr_equal(macaddr, f->macaddr)) {
1536                         __i40e_del_filter(vsi, f);
1537                         found = true;
1538                 }
1539         }
1540
1541         if (found)
1542                 return 0;
1543         else
1544                 return -ENOENT;
1545 }
1546
1547 /**
1548  * i40e_set_mac - NDO callback to set mac address
1549  * @netdev: network interface device structure
1550  * @p: pointer to an address structure
1551  *
1552  * Returns 0 on success, negative on failure
1553  **/
1554 static int i40e_set_mac(struct net_device *netdev, void *p)
1555 {
1556         struct i40e_netdev_priv *np = netdev_priv(netdev);
1557         struct i40e_vsi *vsi = np->vsi;
1558         struct i40e_pf *pf = vsi->back;
1559         struct i40e_hw *hw = &pf->hw;
1560         struct sockaddr *addr = p;
1561
1562         if (!is_valid_ether_addr(addr->sa_data))
1563                 return -EADDRNOTAVAIL;
1564
1565         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1566                 netdev_info(netdev, "already using mac address %pM\n",
1567                             addr->sa_data);
1568                 return 0;
1569         }
1570
1571         if (test_bit(__I40E_DOWN, pf->state) ||
1572             test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1573                 return -EADDRNOTAVAIL;
1574
1575         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1576                 netdev_info(netdev, "returning to hw mac address %pM\n",
1577                             hw->mac.addr);
1578         else
1579                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1580
1581         /* Copy the address first, so that we avoid a possible race with
1582          * .set_rx_mode().
1583          * - Remove old address from MAC filter
1584          * - Copy new address
1585          * - Add new address to MAC filter
1586          */
1587         spin_lock_bh(&vsi->mac_filter_hash_lock);
1588         i40e_del_mac_filter(vsi, netdev->dev_addr);
1589         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1590         i40e_add_mac_filter(vsi, netdev->dev_addr);
1591         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1592
1593         if (vsi->type == I40E_VSI_MAIN) {
1594                 i40e_status ret;
1595
1596                 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1597                                                 addr->sa_data, NULL);
1598                 if (ret)
1599                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1600                                     i40e_stat_str(hw, ret),
1601                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1602         }
1603
1604         /* schedule our worker thread which will take care of
1605          * applying the new filter changes
1606          */
1607         i40e_service_event_schedule(pf);
1608         return 0;
1609 }
1610
1611 /**
1612  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1613  * @vsi: vsi structure
1614  * @seed: RSS hash seed
1615  * @lut: pointer to lookup table of lut_size
1616  * @lut_size: size of the lookup table
1617  **/
1618 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1619                               u8 *lut, u16 lut_size)
1620 {
1621         struct i40e_pf *pf = vsi->back;
1622         struct i40e_hw *hw = &pf->hw;
1623         int ret = 0;
1624
1625         if (seed) {
1626                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1627                         (struct i40e_aqc_get_set_rss_key_data *)seed;
1628                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1629                 if (ret) {
1630                         dev_info(&pf->pdev->dev,
1631                                  "Cannot set RSS key, err %s aq_err %s\n",
1632                                  i40e_stat_str(hw, ret),
1633                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1634                         return ret;
1635                 }
1636         }
1637         if (lut) {
1638                 bool pf_lut = vsi->type == I40E_VSI_MAIN;
1639
1640                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1641                 if (ret) {
1642                         dev_info(&pf->pdev->dev,
1643                                  "Cannot set RSS lut, err %s aq_err %s\n",
1644                                  i40e_stat_str(hw, ret),
1645                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1646                         return ret;
1647                 }
1648         }
1649         return ret;
1650 }
1651
1652 /**
1653  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1654  * @vsi: VSI structure
1655  **/
1656 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1657 {
1658         struct i40e_pf *pf = vsi->back;
1659         u8 seed[I40E_HKEY_ARRAY_SIZE];
1660         u8 *lut;
1661         int ret;
1662
1663         if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
1664                 return 0;
1665         if (!vsi->rss_size)
1666                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1667                                       vsi->num_queue_pairs);
1668         if (!vsi->rss_size)
1669                 return -EINVAL;
1670         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1671         if (!lut)
1672                 return -ENOMEM;
1673
1674         /* Use the user configured hash keys and lookup table if there is one,
1675          * otherwise use default
1676          */
1677         if (vsi->rss_lut_user)
1678                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1679         else
1680                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1681         if (vsi->rss_hkey_user)
1682                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1683         else
1684                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1685         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1686         kfree(lut);
1687         return ret;
1688 }
1689
1690 /**
1691  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1692  * @vsi: the VSI being configured,
1693  * @ctxt: VSI context structure
1694  * @enabled_tc: number of traffic classes to enable
1695  *
1696  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1697  **/
1698 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1699                                            struct i40e_vsi_context *ctxt,
1700                                            u8 enabled_tc)
1701 {
1702         u16 qcount = 0, max_qcount, qmap, sections = 0;
1703         int i, override_q, pow, num_qps, ret;
1704         u8 netdev_tc = 0, offset = 0;
1705
1706         if (vsi->type != I40E_VSI_MAIN)
1707                 return -EINVAL;
1708         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1709         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1710         vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1711         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1712         num_qps = vsi->mqprio_qopt.qopt.count[0];
1713
1714         /* find the next higher power-of-2 of num queue pairs */
1715         pow = ilog2(num_qps);
1716         if (!is_power_of_2(num_qps))
1717                 pow++;
1718         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1719                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1720
1721         /* Setup queue offset/count for all TCs for given VSI */
1722         max_qcount = vsi->mqprio_qopt.qopt.count[0];
1723         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1724                 /* See if the given TC is enabled for the given VSI */
1725                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1726                         offset = vsi->mqprio_qopt.qopt.offset[i];
1727                         qcount = vsi->mqprio_qopt.qopt.count[i];
1728                         if (qcount > max_qcount)
1729                                 max_qcount = qcount;
1730                         vsi->tc_config.tc_info[i].qoffset = offset;
1731                         vsi->tc_config.tc_info[i].qcount = qcount;
1732                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1733                 } else {
1734                         /* TC is not enabled so set the offset to
1735                          * default queue and allocate one queue
1736                          * for the given TC.
1737                          */
1738                         vsi->tc_config.tc_info[i].qoffset = 0;
1739                         vsi->tc_config.tc_info[i].qcount = 1;
1740                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1741                 }
1742         }
1743
1744         /* Set actual Tx/Rx queue pairs */
1745         vsi->num_queue_pairs = offset + qcount;
1746
1747         /* Setup queue TC[0].qmap for given VSI context */
1748         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1749         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1750         ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1751         ctxt->info.valid_sections |= cpu_to_le16(sections);
1752
1753         /* Reconfigure RSS for main VSI with max queue count */
1754         vsi->rss_size = max_qcount;
1755         ret = i40e_vsi_config_rss(vsi);
1756         if (ret) {
1757                 dev_info(&vsi->back->pdev->dev,
1758                          "Failed to reconfig rss for num_queues (%u)\n",
1759                          max_qcount);
1760                 return ret;
1761         }
1762         vsi->reconfig_rss = true;
1763         dev_dbg(&vsi->back->pdev->dev,
1764                 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1765
1766         /* Find queue count available for channel VSIs and starting offset
1767          * for channel VSIs
1768          */
1769         override_q = vsi->mqprio_qopt.qopt.count[0];
1770         if (override_q && override_q < vsi->num_queue_pairs) {
1771                 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1772                 vsi->next_base_queue = override_q;
1773         }
1774         return 0;
1775 }
1776
1777 /**
1778  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1779  * @vsi: the VSI being setup
1780  * @ctxt: VSI context structure
1781  * @enabled_tc: Enabled TCs bitmap
1782  * @is_add: True if called before Add VSI
1783  *
1784  * Setup VSI queue mapping for enabled traffic classes.
1785  **/
1786 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1787                                      struct i40e_vsi_context *ctxt,
1788                                      u8 enabled_tc,
1789                                      bool is_add)
1790 {
1791         struct i40e_pf *pf = vsi->back;
1792         u16 sections = 0;
1793         u8 netdev_tc = 0;
1794         u16 numtc = 1;
1795         u16 qcount;
1796         u8 offset;
1797         u16 qmap;
1798         int i;
1799         u16 num_tc_qps = 0;
1800
1801         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1802         offset = 0;
1803
1804         /* Number of queues per enabled TC */
1805         num_tc_qps = vsi->alloc_queue_pairs;
1806         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1807                 /* Find numtc from enabled TC bitmap */
1808                 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1809                         if (enabled_tc & BIT(i)) /* TC is enabled */
1810                                 numtc++;
1811                 }
1812                 if (!numtc) {
1813                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1814                         numtc = 1;
1815                 }
1816                 num_tc_qps = num_tc_qps / numtc;
1817                 num_tc_qps = min_t(int, num_tc_qps,
1818                                    i40e_pf_get_max_q_per_tc(pf));
1819         }
1820
1821         vsi->tc_config.numtc = numtc;
1822         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1823
1824         /* Do not allow use more TC queue pairs than MSI-X vectors exist */
1825         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1826                 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
1827
1828         /* Setup queue offset/count for all TCs for given VSI */
1829         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1830                 /* See if the given TC is enabled for the given VSI */
1831                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1832                         /* TC is enabled */
1833                         int pow, num_qps;
1834
1835                         switch (vsi->type) {
1836                         case I40E_VSI_MAIN:
1837                                 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
1838                                     I40E_FLAG_FD_ATR_ENABLED)) ||
1839                                     vsi->tc_config.enabled_tc != 1) {
1840                                         qcount = min_t(int, pf->alloc_rss_size,
1841                                                        num_tc_qps);
1842                                         break;
1843                                 }
1844                                 fallthrough;
1845                         case I40E_VSI_FDIR:
1846                         case I40E_VSI_SRIOV:
1847                         case I40E_VSI_VMDQ2:
1848                         default:
1849                                 qcount = num_tc_qps;
1850                                 WARN_ON(i != 0);
1851                                 break;
1852                         }
1853                         vsi->tc_config.tc_info[i].qoffset = offset;
1854                         vsi->tc_config.tc_info[i].qcount = qcount;
1855
1856                         /* find the next higher power-of-2 of num queue pairs */
1857                         num_qps = qcount;
1858                         pow = 0;
1859                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1860                                 pow++;
1861                                 num_qps >>= 1;
1862                         }
1863
1864                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1865                         qmap =
1866                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1867                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1868
1869                         offset += qcount;
1870                 } else {
1871                         /* TC is not enabled so set the offset to
1872                          * default queue and allocate one queue
1873                          * for the given TC.
1874                          */
1875                         vsi->tc_config.tc_info[i].qoffset = 0;
1876                         vsi->tc_config.tc_info[i].qcount = 1;
1877                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1878
1879                         qmap = 0;
1880                 }
1881                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1882         }
1883
1884         /* Set actual Tx/Rx queue pairs */
1885         vsi->num_queue_pairs = offset;
1886         if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1887                 if (vsi->req_queue_pairs > 0)
1888                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1889                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1890                         vsi->num_queue_pairs = pf->num_lan_msix;
1891         }
1892
1893         /* Scheduler section valid can only be set for ADD VSI */
1894         if (is_add) {
1895                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1896
1897                 ctxt->info.up_enable_bits = enabled_tc;
1898         }
1899         if (vsi->type == I40E_VSI_SRIOV) {
1900                 ctxt->info.mapping_flags |=
1901                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1902                 for (i = 0; i < vsi->num_queue_pairs; i++)
1903                         ctxt->info.queue_mapping[i] =
1904                                                cpu_to_le16(vsi->base_queue + i);
1905         } else {
1906                 ctxt->info.mapping_flags |=
1907                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1908                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1909         }
1910         ctxt->info.valid_sections |= cpu_to_le16(sections);
1911 }
1912
1913 /**
1914  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1915  * @netdev: the netdevice
1916  * @addr: address to add
1917  *
1918  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1919  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1920  */
1921 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1922 {
1923         struct i40e_netdev_priv *np = netdev_priv(netdev);
1924         struct i40e_vsi *vsi = np->vsi;
1925
1926         if (i40e_add_mac_filter(vsi, addr))
1927                 return 0;
1928         else
1929                 return -ENOMEM;
1930 }
1931
1932 /**
1933  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1934  * @netdev: the netdevice
1935  * @addr: address to add
1936  *
1937  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1938  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1939  */
1940 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1941 {
1942         struct i40e_netdev_priv *np = netdev_priv(netdev);
1943         struct i40e_vsi *vsi = np->vsi;
1944
1945         /* Under some circumstances, we might receive a request to delete
1946          * our own device address from our uc list. Because we store the
1947          * device address in the VSI's MAC/VLAN filter list, we need to ignore
1948          * such requests and not delete our device address from this list.
1949          */
1950         if (ether_addr_equal(addr, netdev->dev_addr))
1951                 return 0;
1952
1953         i40e_del_mac_filter(vsi, addr);
1954
1955         return 0;
1956 }
1957
1958 /**
1959  * i40e_set_rx_mode - NDO callback to set the netdev filters
1960  * @netdev: network interface device structure
1961  **/
1962 static void i40e_set_rx_mode(struct net_device *netdev)
1963 {
1964         struct i40e_netdev_priv *np = netdev_priv(netdev);
1965         struct i40e_vsi *vsi = np->vsi;
1966
1967         spin_lock_bh(&vsi->mac_filter_hash_lock);
1968
1969         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1970         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1971
1972         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1973
1974         /* check for other flag changes */
1975         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1976                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1977                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1978         }
1979 }
1980
1981 /**
1982  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
1983  * @vsi: Pointer to VSI struct
1984  * @from: Pointer to list which contains MAC filter entries - changes to
1985  *        those entries needs to be undone.
1986  *
1987  * MAC filter entries from this list were slated for deletion.
1988  **/
1989 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
1990                                          struct hlist_head *from)
1991 {
1992         struct i40e_mac_filter *f;
1993         struct hlist_node *h;
1994
1995         hlist_for_each_entry_safe(f, h, from, hlist) {
1996                 u64 key = i40e_addr_to_hkey(f->macaddr);
1997
1998                 /* Move the element back into MAC filter list*/
1999                 hlist_del(&f->hlist);
2000                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
2001         }
2002 }
2003
2004 /**
2005  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2006  * @vsi: Pointer to vsi struct
2007  * @from: Pointer to list which contains MAC filter entries - changes to
2008  *        those entries needs to be undone.
2009  *
2010  * MAC filter entries from this list were slated for addition.
2011  **/
2012 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2013                                          struct hlist_head *from)
2014 {
2015         struct i40e_new_mac_filter *new;
2016         struct hlist_node *h;
2017
2018         hlist_for_each_entry_safe(new, h, from, hlist) {
2019                 /* We can simply free the wrapper structure */
2020                 hlist_del(&new->hlist);
2021                 kfree(new);
2022         }
2023 }
2024
2025 /**
2026  * i40e_next_entry - Get the next non-broadcast filter from a list
2027  * @next: pointer to filter in list
2028  *
2029  * Returns the next non-broadcast filter in the list. Required so that we
2030  * ignore broadcast filters within the list, since these are not handled via
2031  * the normal firmware update path.
2032  */
2033 static
2034 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2035 {
2036         hlist_for_each_entry_continue(next, hlist) {
2037                 if (!is_broadcast_ether_addr(next->f->macaddr))
2038                         return next;
2039         }
2040
2041         return NULL;
2042 }
2043
2044 /**
2045  * i40e_update_filter_state - Update filter state based on return data
2046  * from firmware
2047  * @count: Number of filters added
2048  * @add_list: return data from fw
2049  * @add_head: pointer to first filter in current batch
2050  *
2051  * MAC filter entries from list were slated to be added to device. Returns
2052  * number of successful filters. Note that 0 does NOT mean success!
2053  **/
2054 static int
2055 i40e_update_filter_state(int count,
2056                          struct i40e_aqc_add_macvlan_element_data *add_list,
2057                          struct i40e_new_mac_filter *add_head)
2058 {
2059         int retval = 0;
2060         int i;
2061
2062         for (i = 0; i < count; i++) {
2063                 /* Always check status of each filter. We don't need to check
2064                  * the firmware return status because we pre-set the filter
2065                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2066                  * request to the adminq. Thus, if it no longer matches then
2067                  * we know the filter is active.
2068                  */
2069                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2070                         add_head->state = I40E_FILTER_FAILED;
2071                 } else {
2072                         add_head->state = I40E_FILTER_ACTIVE;
2073                         retval++;
2074                 }
2075
2076                 add_head = i40e_next_filter(add_head);
2077                 if (!add_head)
2078                         break;
2079         }
2080
2081         return retval;
2082 }
2083
2084 /**
2085  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2086  * @vsi: ptr to the VSI
2087  * @vsi_name: name to display in messages
2088  * @list: the list of filters to send to firmware
2089  * @num_del: the number of filters to delete
2090  * @retval: Set to -EIO on failure to delete
2091  *
2092  * Send a request to firmware via AdminQ to delete a set of filters. Uses
2093  * *retval instead of a return value so that success does not force ret_val to
2094  * be set to 0. This ensures that a sequence of calls to this function
2095  * preserve the previous value of *retval on successful delete.
2096  */
2097 static
2098 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2099                           struct i40e_aqc_remove_macvlan_element_data *list,
2100                           int num_del, int *retval)
2101 {
2102         struct i40e_hw *hw = &vsi->back->hw;
2103         i40e_status aq_ret;
2104         int aq_err;
2105
2106         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
2107         aq_err = hw->aq.asq_last_status;
2108
2109         /* Explicitly ignore and do not report when firmware returns ENOENT */
2110         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
2111                 *retval = -EIO;
2112                 dev_info(&vsi->back->pdev->dev,
2113                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
2114                          vsi_name, i40e_stat_str(hw, aq_ret),
2115                          i40e_aq_str(hw, aq_err));
2116         }
2117 }
2118
2119 /**
2120  * i40e_aqc_add_filters - Request firmware to add a set of filters
2121  * @vsi: ptr to the VSI
2122  * @vsi_name: name to display in messages
2123  * @list: the list of filters to send to firmware
2124  * @add_head: Position in the add hlist
2125  * @num_add: the number of filters to add
2126  *
2127  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2128  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2129  * space for more filters.
2130  */
2131 static
2132 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2133                           struct i40e_aqc_add_macvlan_element_data *list,
2134                           struct i40e_new_mac_filter *add_head,
2135                           int num_add)
2136 {
2137         struct i40e_hw *hw = &vsi->back->hw;
2138         int aq_err, fcnt;
2139
2140         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
2141         aq_err = hw->aq.asq_last_status;
2142         fcnt = i40e_update_filter_state(num_add, list, add_head);
2143
2144         if (fcnt != num_add) {
2145                 if (vsi->type == I40E_VSI_MAIN) {
2146                         set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2147                         dev_warn(&vsi->back->pdev->dev,
2148                                  "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2149                                  i40e_aq_str(hw, aq_err), vsi_name);
2150                 } else if (vsi->type == I40E_VSI_SRIOV ||
2151                            vsi->type == I40E_VSI_VMDQ1 ||
2152                            vsi->type == I40E_VSI_VMDQ2) {
2153                         dev_warn(&vsi->back->pdev->dev,
2154                                  "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2155                                  i40e_aq_str(hw, aq_err), vsi_name, vsi_name);
2156                 } else {
2157                         dev_warn(&vsi->back->pdev->dev,
2158                                  "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2159                                  i40e_aq_str(hw, aq_err), vsi_name, vsi->type);
2160                 }
2161         }
2162 }
2163
2164 /**
2165  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2166  * @vsi: pointer to the VSI
2167  * @vsi_name: the VSI name
2168  * @f: filter data
2169  *
2170  * This function sets or clears the promiscuous broadcast flags for VLAN
2171  * filters in order to properly receive broadcast frames. Assumes that only
2172  * broadcast filters are passed.
2173  *
2174  * Returns status indicating success or failure;
2175  **/
2176 static i40e_status
2177 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2178                           struct i40e_mac_filter *f)
2179 {
2180         bool enable = f->state == I40E_FILTER_NEW;
2181         struct i40e_hw *hw = &vsi->back->hw;
2182         i40e_status aq_ret;
2183
2184         if (f->vlan == I40E_VLAN_ANY) {
2185                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2186                                                    vsi->seid,
2187                                                    enable,
2188                                                    NULL);
2189         } else {
2190                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2191                                                             vsi->seid,
2192                                                             enable,
2193                                                             f->vlan,
2194                                                             NULL);
2195         }
2196
2197         if (aq_ret) {
2198                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2199                 dev_warn(&vsi->back->pdev->dev,
2200                          "Error %s, forcing overflow promiscuous on %s\n",
2201                          i40e_aq_str(hw, hw->aq.asq_last_status),
2202                          vsi_name);
2203         }
2204
2205         return aq_ret;
2206 }
2207
2208 /**
2209  * i40e_set_promiscuous - set promiscuous mode
2210  * @pf: board private structure
2211  * @promisc: promisc on or off
2212  *
2213  * There are different ways of setting promiscuous mode on a PF depending on
2214  * what state/environment we're in.  This identifies and sets it appropriately.
2215  * Returns 0 on success.
2216  **/
2217 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2218 {
2219         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2220         struct i40e_hw *hw = &pf->hw;
2221         i40e_status aq_ret;
2222
2223         if (vsi->type == I40E_VSI_MAIN &&
2224             pf->lan_veb != I40E_NO_VEB &&
2225             !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2226                 /* set defport ON for Main VSI instead of true promisc
2227                  * this way we will get all unicast/multicast and VLAN
2228                  * promisc behavior but will not get VF or VMDq traffic
2229                  * replicated on the Main VSI.
2230                  */
2231                 if (promisc)
2232                         aq_ret = i40e_aq_set_default_vsi(hw,
2233                                                          vsi->seid,
2234                                                          NULL);
2235                 else
2236                         aq_ret = i40e_aq_clear_default_vsi(hw,
2237                                                            vsi->seid,
2238                                                            NULL);
2239                 if (aq_ret) {
2240                         dev_info(&pf->pdev->dev,
2241                                  "Set default VSI failed, err %s, aq_err %s\n",
2242                                  i40e_stat_str(hw, aq_ret),
2243                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2244                 }
2245         } else {
2246                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2247                                                   hw,
2248                                                   vsi->seid,
2249                                                   promisc, NULL,
2250                                                   true);
2251                 if (aq_ret) {
2252                         dev_info(&pf->pdev->dev,
2253                                  "set unicast promisc failed, err %s, aq_err %s\n",
2254                                  i40e_stat_str(hw, aq_ret),
2255                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2256                 }
2257                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2258                                                   hw,
2259                                                   vsi->seid,
2260                                                   promisc, NULL);
2261                 if (aq_ret) {
2262                         dev_info(&pf->pdev->dev,
2263                                  "set multicast promisc failed, err %s, aq_err %s\n",
2264                                  i40e_stat_str(hw, aq_ret),
2265                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2266                 }
2267         }
2268
2269         if (!aq_ret)
2270                 pf->cur_promisc = promisc;
2271
2272         return aq_ret;
2273 }
2274
2275 /**
2276  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2277  * @vsi: ptr to the VSI
2278  *
2279  * Push any outstanding VSI filter changes through the AdminQ.
2280  *
2281  * Returns 0 or error value
2282  **/
2283 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2284 {
2285         struct hlist_head tmp_add_list, tmp_del_list;
2286         struct i40e_mac_filter *f;
2287         struct i40e_new_mac_filter *new, *add_head = NULL;
2288         struct i40e_hw *hw = &vsi->back->hw;
2289         bool old_overflow, new_overflow;
2290         unsigned int failed_filters = 0;
2291         unsigned int vlan_filters = 0;
2292         char vsi_name[16] = "PF";
2293         int filter_list_len = 0;
2294         i40e_status aq_ret = 0;
2295         u32 changed_flags = 0;
2296         struct hlist_node *h;
2297         struct i40e_pf *pf;
2298         int num_add = 0;
2299         int num_del = 0;
2300         int retval = 0;
2301         u16 cmd_flags;
2302         int list_size;
2303         int bkt;
2304
2305         /* empty array typed pointers, kcalloc later */
2306         struct i40e_aqc_add_macvlan_element_data *add_list;
2307         struct i40e_aqc_remove_macvlan_element_data *del_list;
2308
2309         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2310                 usleep_range(1000, 2000);
2311         pf = vsi->back;
2312
2313         old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2314
2315         if (vsi->netdev) {
2316                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2317                 vsi->current_netdev_flags = vsi->netdev->flags;
2318         }
2319
2320         INIT_HLIST_HEAD(&tmp_add_list);
2321         INIT_HLIST_HEAD(&tmp_del_list);
2322
2323         if (vsi->type == I40E_VSI_SRIOV)
2324                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2325         else if (vsi->type != I40E_VSI_MAIN)
2326                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2327
2328         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2329                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2330
2331                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2332                 /* Create a list of filters to delete. */
2333                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2334                         if (f->state == I40E_FILTER_REMOVE) {
2335                                 /* Move the element into temporary del_list */
2336                                 hash_del(&f->hlist);
2337                                 hlist_add_head(&f->hlist, &tmp_del_list);
2338
2339                                 /* Avoid counting removed filters */
2340                                 continue;
2341                         }
2342                         if (f->state == I40E_FILTER_NEW) {
2343                                 /* Create a temporary i40e_new_mac_filter */
2344                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2345                                 if (!new)
2346                                         goto err_no_memory_locked;
2347
2348                                 /* Store pointer to the real filter */
2349                                 new->f = f;
2350                                 new->state = f->state;
2351
2352                                 /* Add it to the hash list */
2353                                 hlist_add_head(&new->hlist, &tmp_add_list);
2354                         }
2355
2356                         /* Count the number of active (current and new) VLAN
2357                          * filters we have now. Does not count filters which
2358                          * are marked for deletion.
2359                          */
2360                         if (f->vlan > 0)
2361                                 vlan_filters++;
2362                 }
2363
2364                 retval = i40e_correct_mac_vlan_filters(vsi,
2365                                                        &tmp_add_list,
2366                                                        &tmp_del_list,
2367                                                        vlan_filters);
2368                 if (retval)
2369                         goto err_no_memory_locked;
2370
2371                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2372         }
2373
2374         /* Now process 'del_list' outside the lock */
2375         if (!hlist_empty(&tmp_del_list)) {
2376                 filter_list_len = hw->aq.asq_buf_size /
2377                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2378                 list_size = filter_list_len *
2379                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2380                 del_list = kzalloc(list_size, GFP_ATOMIC);
2381                 if (!del_list)
2382                         goto err_no_memory;
2383
2384                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2385                         cmd_flags = 0;
2386
2387                         /* handle broadcast filters by updating the broadcast
2388                          * promiscuous flag and release filter list.
2389                          */
2390                         if (is_broadcast_ether_addr(f->macaddr)) {
2391                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2392
2393                                 hlist_del(&f->hlist);
2394                                 kfree(f);
2395                                 continue;
2396                         }
2397
2398                         /* add to delete list */
2399                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2400                         if (f->vlan == I40E_VLAN_ANY) {
2401                                 del_list[num_del].vlan_tag = 0;
2402                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2403                         } else {
2404                                 del_list[num_del].vlan_tag =
2405                                         cpu_to_le16((u16)(f->vlan));
2406                         }
2407
2408                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2409                         del_list[num_del].flags = cmd_flags;
2410                         num_del++;
2411
2412                         /* flush a full buffer */
2413                         if (num_del == filter_list_len) {
2414                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2415                                                      num_del, &retval);
2416                                 memset(del_list, 0, list_size);
2417                                 num_del = 0;
2418                         }
2419                         /* Release memory for MAC filter entries which were
2420                          * synced up with HW.
2421                          */
2422                         hlist_del(&f->hlist);
2423                         kfree(f);
2424                 }
2425
2426                 if (num_del) {
2427                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2428                                              num_del, &retval);
2429                 }
2430
2431                 kfree(del_list);
2432                 del_list = NULL;
2433         }
2434
2435         if (!hlist_empty(&tmp_add_list)) {
2436                 /* Do all the adds now. */
2437                 filter_list_len = hw->aq.asq_buf_size /
2438                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2439                 list_size = filter_list_len *
2440                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2441                 add_list = kzalloc(list_size, GFP_ATOMIC);
2442                 if (!add_list)
2443                         goto err_no_memory;
2444
2445                 num_add = 0;
2446                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2447                         /* handle broadcast filters by updating the broadcast
2448                          * promiscuous flag instead of adding a MAC filter.
2449                          */
2450                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2451                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2452                                                               new->f))
2453                                         new->state = I40E_FILTER_FAILED;
2454                                 else
2455                                         new->state = I40E_FILTER_ACTIVE;
2456                                 continue;
2457                         }
2458
2459                         /* add to add array */
2460                         if (num_add == 0)
2461                                 add_head = new;
2462                         cmd_flags = 0;
2463                         ether_addr_copy(add_list[num_add].mac_addr,
2464                                         new->f->macaddr);
2465                         if (new->f->vlan == I40E_VLAN_ANY) {
2466                                 add_list[num_add].vlan_tag = 0;
2467                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2468                         } else {
2469                                 add_list[num_add].vlan_tag =
2470                                         cpu_to_le16((u16)(new->f->vlan));
2471                         }
2472                         add_list[num_add].queue_number = 0;
2473                         /* set invalid match method for later detection */
2474                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2475                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2476                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2477                         num_add++;
2478
2479                         /* flush a full buffer */
2480                         if (num_add == filter_list_len) {
2481                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2482                                                      add_head, num_add);
2483                                 memset(add_list, 0, list_size);
2484                                 num_add = 0;
2485                         }
2486                 }
2487                 if (num_add) {
2488                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2489                                              num_add);
2490                 }
2491                 /* Now move all of the filters from the temp add list back to
2492                  * the VSI's list.
2493                  */
2494                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2495                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2496                         /* Only update the state if we're still NEW */
2497                         if (new->f->state == I40E_FILTER_NEW)
2498                                 new->f->state = new->state;
2499                         hlist_del(&new->hlist);
2500                         kfree(new);
2501                 }
2502                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2503                 kfree(add_list);
2504                 add_list = NULL;
2505         }
2506
2507         /* Determine the number of active and failed filters. */
2508         spin_lock_bh(&vsi->mac_filter_hash_lock);
2509         vsi->active_filters = 0;
2510         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2511                 if (f->state == I40E_FILTER_ACTIVE)
2512                         vsi->active_filters++;
2513                 else if (f->state == I40E_FILTER_FAILED)
2514                         failed_filters++;
2515         }
2516         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2517
2518         /* Check if we are able to exit overflow promiscuous mode. We can
2519          * safely exit if we didn't just enter, we no longer have any failed
2520          * filters, and we have reduced filters below the threshold value.
2521          */
2522         if (old_overflow && !failed_filters &&
2523             vsi->active_filters < vsi->promisc_threshold) {
2524                 dev_info(&pf->pdev->dev,
2525                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2526                          vsi_name);
2527                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2528                 vsi->promisc_threshold = 0;
2529         }
2530
2531         /* if the VF is not trusted do not do promisc */
2532         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2533                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2534                 goto out;
2535         }
2536
2537         new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2538
2539         /* If we are entering overflow promiscuous, we need to calculate a new
2540          * threshold for when we are safe to exit
2541          */
2542         if (!old_overflow && new_overflow)
2543                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2544
2545         /* check for changes in promiscuous modes */
2546         if (changed_flags & IFF_ALLMULTI) {
2547                 bool cur_multipromisc;
2548
2549                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2550                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2551                                                                vsi->seid,
2552                                                                cur_multipromisc,
2553                                                                NULL);
2554                 if (aq_ret) {
2555                         retval = i40e_aq_rc_to_posix(aq_ret,
2556                                                      hw->aq.asq_last_status);
2557                         dev_info(&pf->pdev->dev,
2558                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2559                                  vsi_name,
2560                                  i40e_stat_str(hw, aq_ret),
2561                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2562                 } else {
2563                         dev_info(&pf->pdev->dev, "%s is %s allmulti mode.\n",
2564                                  vsi->netdev->name,
2565                                  cur_multipromisc ? "entering" : "leaving");
2566                 }
2567         }
2568
2569         if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2570                 bool cur_promisc;
2571
2572                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2573                                new_overflow);
2574                 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2575                 if (aq_ret) {
2576                         retval = i40e_aq_rc_to_posix(aq_ret,
2577                                                      hw->aq.asq_last_status);
2578                         dev_info(&pf->pdev->dev,
2579                                  "Setting promiscuous %s failed on %s, err %s aq_err %s\n",
2580                                  cur_promisc ? "on" : "off",
2581                                  vsi_name,
2582                                  i40e_stat_str(hw, aq_ret),
2583                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2584                 }
2585         }
2586 out:
2587         /* if something went wrong then set the changed flag so we try again */
2588         if (retval)
2589                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2590
2591         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2592         return retval;
2593
2594 err_no_memory:
2595         /* Restore elements on the temporary add and delete lists */
2596         spin_lock_bh(&vsi->mac_filter_hash_lock);
2597 err_no_memory_locked:
2598         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2599         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2600         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2601
2602         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2603         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2604         return -ENOMEM;
2605 }
2606
2607 /**
2608  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2609  * @pf: board private structure
2610  **/
2611 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2612 {
2613         int v;
2614
2615         if (!pf)
2616                 return;
2617         if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2618                 return;
2619         if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2620                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2621                 return;
2622         }
2623
2624         for (v = 0; v < pf->num_alloc_vsi; v++) {
2625                 if (pf->vsi[v] &&
2626                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2627                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2628
2629                         if (ret) {
2630                                 /* come back and try again later */
2631                                 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2632                                         pf->state);
2633                                 break;
2634                         }
2635                 }
2636         }
2637 }
2638
2639 /**
2640  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2641  * @vsi: the vsi
2642  **/
2643 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2644 {
2645         if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2646                 return I40E_RXBUFFER_2048;
2647         else
2648                 return I40E_RXBUFFER_3072;
2649 }
2650
2651 /**
2652  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2653  * @netdev: network interface device structure
2654  * @new_mtu: new value for maximum frame size
2655  *
2656  * Returns 0 on success, negative on failure
2657  **/
2658 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2659 {
2660         struct i40e_netdev_priv *np = netdev_priv(netdev);
2661         struct i40e_vsi *vsi = np->vsi;
2662         struct i40e_pf *pf = vsi->back;
2663
2664         if (i40e_enabled_xdp_vsi(vsi)) {
2665                 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2666
2667                 if (frame_size > i40e_max_xdp_frame_size(vsi))
2668                         return -EINVAL;
2669         }
2670
2671         netdev_dbg(netdev, "changing MTU from %d to %d\n",
2672                    netdev->mtu, new_mtu);
2673         netdev->mtu = new_mtu;
2674         if (netif_running(netdev))
2675                 i40e_vsi_reinit_locked(vsi);
2676         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2677         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2678         return 0;
2679 }
2680
2681 /**
2682  * i40e_ioctl - Access the hwtstamp interface
2683  * @netdev: network interface device structure
2684  * @ifr: interface request data
2685  * @cmd: ioctl command
2686  **/
2687 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2688 {
2689         struct i40e_netdev_priv *np = netdev_priv(netdev);
2690         struct i40e_pf *pf = np->vsi->back;
2691
2692         switch (cmd) {
2693         case SIOCGHWTSTAMP:
2694                 return i40e_ptp_get_ts_config(pf, ifr);
2695         case SIOCSHWTSTAMP:
2696                 return i40e_ptp_set_ts_config(pf, ifr);
2697         default:
2698                 return -EOPNOTSUPP;
2699         }
2700 }
2701
2702 /**
2703  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2704  * @vsi: the vsi being adjusted
2705  **/
2706 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2707 {
2708         struct i40e_vsi_context ctxt;
2709         i40e_status ret;
2710
2711         /* Don't modify stripping options if a port VLAN is active */
2712         if (vsi->info.pvid)
2713                 return;
2714
2715         if ((vsi->info.valid_sections &
2716              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2717             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2718                 return;  /* already enabled */
2719
2720         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2721         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2722                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2723
2724         ctxt.seid = vsi->seid;
2725         ctxt.info = vsi->info;
2726         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2727         if (ret) {
2728                 dev_info(&vsi->back->pdev->dev,
2729                          "update vlan stripping failed, err %s aq_err %s\n",
2730                          i40e_stat_str(&vsi->back->hw, ret),
2731                          i40e_aq_str(&vsi->back->hw,
2732                                      vsi->back->hw.aq.asq_last_status));
2733         }
2734 }
2735
2736 /**
2737  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2738  * @vsi: the vsi being adjusted
2739  **/
2740 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2741 {
2742         struct i40e_vsi_context ctxt;
2743         i40e_status ret;
2744
2745         /* Don't modify stripping options if a port VLAN is active */
2746         if (vsi->info.pvid)
2747                 return;
2748
2749         if ((vsi->info.valid_sections &
2750              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2751             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2752              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2753                 return;  /* already disabled */
2754
2755         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2756         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2757                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2758
2759         ctxt.seid = vsi->seid;
2760         ctxt.info = vsi->info;
2761         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2762         if (ret) {
2763                 dev_info(&vsi->back->pdev->dev,
2764                          "update vlan stripping failed, err %s aq_err %s\n",
2765                          i40e_stat_str(&vsi->back->hw, ret),
2766                          i40e_aq_str(&vsi->back->hw,
2767                                      vsi->back->hw.aq.asq_last_status));
2768         }
2769 }
2770
2771 /**
2772  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2773  * @vsi: the vsi being configured
2774  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2775  *
2776  * This is a helper function for adding a new MAC/VLAN filter with the
2777  * specified VLAN for each existing MAC address already in the hash table.
2778  * This function does *not* perform any accounting to update filters based on
2779  * VLAN mode.
2780  *
2781  * NOTE: this function expects to be called while under the
2782  * mac_filter_hash_lock
2783  **/
2784 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2785 {
2786         struct i40e_mac_filter *f, *add_f;
2787         struct hlist_node *h;
2788         int bkt;
2789
2790         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2791                 if (f->state == I40E_FILTER_REMOVE)
2792                         continue;
2793                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2794                 if (!add_f) {
2795                         dev_info(&vsi->back->pdev->dev,
2796                                  "Could not add vlan filter %d for %pM\n",
2797                                  vid, f->macaddr);
2798                         return -ENOMEM;
2799                 }
2800         }
2801
2802         return 0;
2803 }
2804
2805 /**
2806  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2807  * @vsi: the VSI being configured
2808  * @vid: VLAN id to be added
2809  **/
2810 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2811 {
2812         int err;
2813
2814         if (vsi->info.pvid)
2815                 return -EINVAL;
2816
2817         /* The network stack will attempt to add VID=0, with the intention to
2818          * receive priority tagged packets with a VLAN of 0. Our HW receives
2819          * these packets by default when configured to receive untagged
2820          * packets, so we don't need to add a filter for this case.
2821          * Additionally, HW interprets adding a VID=0 filter as meaning to
2822          * receive *only* tagged traffic and stops receiving untagged traffic.
2823          * Thus, we do not want to actually add a filter for VID=0
2824          */
2825         if (!vid)
2826                 return 0;
2827
2828         /* Locked once because all functions invoked below iterates list*/
2829         spin_lock_bh(&vsi->mac_filter_hash_lock);
2830         err = i40e_add_vlan_all_mac(vsi, vid);
2831         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2832         if (err)
2833                 return err;
2834
2835         /* schedule our worker thread which will take care of
2836          * applying the new filter changes
2837          */
2838         i40e_service_event_schedule(vsi->back);
2839         return 0;
2840 }
2841
2842 /**
2843  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2844  * @vsi: the vsi being configured
2845  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2846  *
2847  * This function should be used to remove all VLAN filters which match the
2848  * given VID. It does not schedule the service event and does not take the
2849  * mac_filter_hash_lock so it may be combined with other operations under
2850  * a single invocation of the mac_filter_hash_lock.
2851  *
2852  * NOTE: this function expects to be called while under the
2853  * mac_filter_hash_lock
2854  */
2855 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2856 {
2857         struct i40e_mac_filter *f;
2858         struct hlist_node *h;
2859         int bkt;
2860
2861         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2862                 if (f->vlan == vid)
2863                         __i40e_del_filter(vsi, f);
2864         }
2865 }
2866
2867 /**
2868  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2869  * @vsi: the VSI being configured
2870  * @vid: VLAN id to be removed
2871  **/
2872 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2873 {
2874         if (!vid || vsi->info.pvid)
2875                 return;
2876
2877         spin_lock_bh(&vsi->mac_filter_hash_lock);
2878         i40e_rm_vlan_all_mac(vsi, vid);
2879         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2880
2881         /* schedule our worker thread which will take care of
2882          * applying the new filter changes
2883          */
2884         i40e_service_event_schedule(vsi->back);
2885 }
2886
2887 /**
2888  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2889  * @netdev: network interface to be adjusted
2890  * @proto: unused protocol value
2891  * @vid: vlan id to be added
2892  *
2893  * net_device_ops implementation for adding vlan ids
2894  **/
2895 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2896                                 __always_unused __be16 proto, u16 vid)
2897 {
2898         struct i40e_netdev_priv *np = netdev_priv(netdev);
2899         struct i40e_vsi *vsi = np->vsi;
2900         int ret = 0;
2901
2902         if (vid >= VLAN_N_VID)
2903                 return -EINVAL;
2904
2905         ret = i40e_vsi_add_vlan(vsi, vid);
2906         if (!ret)
2907                 set_bit(vid, vsi->active_vlans);
2908
2909         return ret;
2910 }
2911
2912 /**
2913  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
2914  * @netdev: network interface to be adjusted
2915  * @proto: unused protocol value
2916  * @vid: vlan id to be added
2917  **/
2918 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
2919                                     __always_unused __be16 proto, u16 vid)
2920 {
2921         struct i40e_netdev_priv *np = netdev_priv(netdev);
2922         struct i40e_vsi *vsi = np->vsi;
2923
2924         if (vid >= VLAN_N_VID)
2925                 return;
2926         set_bit(vid, vsi->active_vlans);
2927 }
2928
2929 /**
2930  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2931  * @netdev: network interface to be adjusted
2932  * @proto: unused protocol value
2933  * @vid: vlan id to be removed
2934  *
2935  * net_device_ops implementation for removing vlan ids
2936  **/
2937 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2938                                  __always_unused __be16 proto, u16 vid)
2939 {
2940         struct i40e_netdev_priv *np = netdev_priv(netdev);
2941         struct i40e_vsi *vsi = np->vsi;
2942
2943         /* return code is ignored as there is nothing a user
2944          * can do about failure to remove and a log message was
2945          * already printed from the other function
2946          */
2947         i40e_vsi_kill_vlan(vsi, vid);
2948
2949         clear_bit(vid, vsi->active_vlans);
2950
2951         return 0;
2952 }
2953
2954 /**
2955  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2956  * @vsi: the vsi being brought back up
2957  **/
2958 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2959 {
2960         u16 vid;
2961
2962         if (!vsi->netdev)
2963                 return;
2964
2965         if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2966                 i40e_vlan_stripping_enable(vsi);
2967         else
2968                 i40e_vlan_stripping_disable(vsi);
2969
2970         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2971                 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
2972                                         vid);
2973 }
2974
2975 /**
2976  * i40e_vsi_add_pvid - Add pvid for the VSI
2977  * @vsi: the vsi being adjusted
2978  * @vid: the vlan id to set as a PVID
2979  **/
2980 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2981 {
2982         struct i40e_vsi_context ctxt;
2983         i40e_status ret;
2984
2985         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2986         vsi->info.pvid = cpu_to_le16(vid);
2987         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2988                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2989                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2990
2991         ctxt.seid = vsi->seid;
2992         ctxt.info = vsi->info;
2993         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2994         if (ret) {
2995                 dev_info(&vsi->back->pdev->dev,
2996                          "add pvid failed, err %s aq_err %s\n",
2997                          i40e_stat_str(&vsi->back->hw, ret),
2998                          i40e_aq_str(&vsi->back->hw,
2999                                      vsi->back->hw.aq.asq_last_status));
3000                 return -ENOENT;
3001         }
3002
3003         return 0;
3004 }
3005
3006 /**
3007  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3008  * @vsi: the vsi being adjusted
3009  *
3010  * Just use the vlan_rx_register() service to put it back to normal
3011  **/
3012 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3013 {
3014         vsi->info.pvid = 0;
3015
3016         i40e_vlan_stripping_disable(vsi);
3017 }
3018
3019 /**
3020  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3021  * @vsi: ptr to the VSI
3022  *
3023  * If this function returns with an error, then it's possible one or
3024  * more of the rings is populated (while the rest are not).  It is the
3025  * callers duty to clean those orphaned rings.
3026  *
3027  * Return 0 on success, negative on failure
3028  **/
3029 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3030 {
3031         int i, err = 0;
3032
3033         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3034                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3035
3036         if (!i40e_enabled_xdp_vsi(vsi))
3037                 return err;
3038
3039         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3040                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3041
3042         return err;
3043 }
3044
3045 /**
3046  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3047  * @vsi: ptr to the VSI
3048  *
3049  * Free VSI's transmit software resources
3050  **/
3051 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3052 {
3053         int i;
3054
3055         if (vsi->tx_rings) {
3056                 for (i = 0; i < vsi->num_queue_pairs; i++)
3057                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3058                                 i40e_free_tx_resources(vsi->tx_rings[i]);
3059         }
3060
3061         if (vsi->xdp_rings) {
3062                 for (i = 0; i < vsi->num_queue_pairs; i++)
3063                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3064                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
3065         }
3066 }
3067
3068 /**
3069  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3070  * @vsi: ptr to the VSI
3071  *
3072  * If this function returns with an error, then it's possible one or
3073  * more of the rings is populated (while the rest are not).  It is the
3074  * callers duty to clean those orphaned rings.
3075  *
3076  * Return 0 on success, negative on failure
3077  **/
3078 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3079 {
3080         int i, err = 0;
3081
3082         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3083                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3084         return err;
3085 }
3086
3087 /**
3088  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3089  * @vsi: ptr to the VSI
3090  *
3091  * Free all receive software resources
3092  **/
3093 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3094 {
3095         int i;
3096
3097         if (!vsi->rx_rings)
3098                 return;
3099
3100         for (i = 0; i < vsi->num_queue_pairs; i++)
3101                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3102                         i40e_free_rx_resources(vsi->rx_rings[i]);
3103 }
3104
3105 /**
3106  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3107  * @ring: The Tx ring to configure
3108  *
3109  * This enables/disables XPS for a given Tx descriptor ring
3110  * based on the TCs enabled for the VSI that ring belongs to.
3111  **/
3112 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3113 {
3114         int cpu;
3115
3116         if (!ring->q_vector || !ring->netdev || ring->ch)
3117                 return;
3118
3119         /* We only initialize XPS once, so as not to overwrite user settings */
3120         if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3121                 return;
3122
3123         cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3124         netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3125                             ring->queue_index);
3126 }
3127
3128 /**
3129  * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3130  * @ring: The Tx or Rx ring
3131  *
3132  * Returns the AF_XDP buffer pool or NULL.
3133  **/
3134 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3135 {
3136         bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3137         int qid = ring->queue_index;
3138
3139         if (ring_is_xdp(ring))
3140                 qid -= ring->vsi->alloc_queue_pairs;
3141
3142         if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3143                 return NULL;
3144
3145         return xsk_get_pool_from_qid(ring->vsi->netdev, qid);
3146 }
3147
3148 /**
3149  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3150  * @ring: The Tx ring to configure
3151  *
3152  * Configure the Tx descriptor ring in the HMC context.
3153  **/
3154 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3155 {
3156         struct i40e_vsi *vsi = ring->vsi;
3157         u16 pf_q = vsi->base_queue + ring->queue_index;
3158         struct i40e_hw *hw = &vsi->back->hw;
3159         struct i40e_hmc_obj_txq tx_ctx;
3160         i40e_status err = 0;
3161         u32 qtx_ctl = 0;
3162
3163         if (ring_is_xdp(ring))
3164                 ring->xsk_pool = i40e_xsk_pool(ring);
3165
3166         /* some ATR related tx ring init */
3167         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
3168                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
3169                 ring->atr_count = 0;
3170         } else {
3171                 ring->atr_sample_rate = 0;
3172         }
3173
3174         /* configure XPS */
3175         i40e_config_xps_tx_ring(ring);
3176
3177         /* clear the context structure first */
3178         memset(&tx_ctx, 0, sizeof(tx_ctx));
3179
3180         tx_ctx.new_context = 1;
3181         tx_ctx.base = (ring->dma / 128);
3182         tx_ctx.qlen = ring->count;
3183         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
3184                                                I40E_FLAG_FD_ATR_ENABLED));
3185         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
3186         /* FDIR VSI tx ring can still use RS bit and writebacks */
3187         if (vsi->type != I40E_VSI_FDIR)
3188                 tx_ctx.head_wb_ena = 1;
3189         tx_ctx.head_wb_addr = ring->dma +
3190                               (ring->count * sizeof(struct i40e_tx_desc));
3191
3192         /* As part of VSI creation/update, FW allocates certain
3193          * Tx arbitration queue sets for each TC enabled for
3194          * the VSI. The FW returns the handles to these queue
3195          * sets as part of the response buffer to Add VSI,
3196          * Update VSI, etc. AQ commands. It is expected that
3197          * these queue set handles be associated with the Tx
3198          * queues by the driver as part of the TX queue context
3199          * initialization. This has to be done regardless of
3200          * DCB as by default everything is mapped to TC0.
3201          */
3202
3203         if (ring->ch)
3204                 tx_ctx.rdylist =
3205                         le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3206
3207         else
3208                 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3209
3210         tx_ctx.rdylist_act = 0;
3211
3212         /* clear the context in the HMC */
3213         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3214         if (err) {
3215                 dev_info(&vsi->back->pdev->dev,
3216                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3217                          ring->queue_index, pf_q, err);
3218                 return -ENOMEM;
3219         }
3220
3221         /* set the context in the HMC */
3222         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3223         if (err) {
3224                 dev_info(&vsi->back->pdev->dev,
3225                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3226                          ring->queue_index, pf_q, err);
3227                 return -ENOMEM;
3228         }
3229
3230         /* Now associate this queue with this PCI function */
3231         if (ring->ch) {
3232                 if (ring->ch->type == I40E_VSI_VMDQ2)
3233                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3234                 else
3235                         return -EINVAL;
3236
3237                 qtx_ctl |= (ring->ch->vsi_number <<
3238                             I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3239                             I40E_QTX_CTL_VFVM_INDX_MASK;
3240         } else {
3241                 if (vsi->type == I40E_VSI_VMDQ2) {
3242                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3243                         qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3244                                     I40E_QTX_CTL_VFVM_INDX_MASK;
3245                 } else {
3246                         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3247                 }
3248         }
3249
3250         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3251                     I40E_QTX_CTL_PF_INDX_MASK);
3252         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3253         i40e_flush(hw);
3254
3255         /* cache tail off for easier writes later */
3256         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3257
3258         return 0;
3259 }
3260
3261 /**
3262  * i40e_configure_rx_ring - Configure a receive ring context
3263  * @ring: The Rx ring to configure
3264  *
3265  * Configure the Rx descriptor ring in the HMC context.
3266  **/
3267 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3268 {
3269         struct i40e_vsi *vsi = ring->vsi;
3270         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3271         u16 pf_q = vsi->base_queue + ring->queue_index;
3272         struct i40e_hw *hw = &vsi->back->hw;
3273         struct i40e_hmc_obj_rxq rx_ctx;
3274         i40e_status err = 0;
3275         bool ok;
3276         int ret;
3277
3278         bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3279
3280         /* clear the context structure first */
3281         memset(&rx_ctx, 0, sizeof(rx_ctx));
3282
3283         if (ring->vsi->type == I40E_VSI_MAIN)
3284                 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
3285
3286         kfree(ring->rx_bi);
3287         ring->xsk_pool = i40e_xsk_pool(ring);
3288         if (ring->xsk_pool) {
3289                 ret = i40e_alloc_rx_bi_zc(ring);
3290                 if (ret)
3291                         return ret;
3292                 ring->rx_buf_len =
3293                   xsk_pool_get_rx_frame_size(ring->xsk_pool);
3294                 /* For AF_XDP ZC, we disallow packets to span on
3295                  * multiple buffers, thus letting us skip that
3296                  * handling in the fast-path.
3297                  */
3298                 chain_len = 1;
3299                 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3300                                                  MEM_TYPE_XSK_BUFF_POOL,
3301                                                  NULL);
3302                 if (ret)
3303                         return ret;
3304                 dev_info(&vsi->back->pdev->dev,
3305                          "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3306                          ring->queue_index);
3307
3308         } else {
3309                 ret = i40e_alloc_rx_bi(ring);
3310                 if (ret)
3311                         return ret;
3312                 ring->rx_buf_len = vsi->rx_buf_len;
3313                 if (ring->vsi->type == I40E_VSI_MAIN) {
3314                         ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3315                                                          MEM_TYPE_PAGE_SHARED,
3316                                                          NULL);
3317                         if (ret)
3318                                 return ret;
3319                 }
3320         }
3321
3322         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3323                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3324
3325         rx_ctx.base = (ring->dma / 128);
3326         rx_ctx.qlen = ring->count;
3327
3328         /* use 16 byte descriptors */
3329         rx_ctx.dsize = 0;
3330
3331         /* descriptor type is always zero
3332          * rx_ctx.dtype = 0;
3333          */
3334         rx_ctx.hsplit_0 = 0;
3335
3336         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3337         if (hw->revision_id == 0)
3338                 rx_ctx.lrxqthresh = 0;
3339         else
3340                 rx_ctx.lrxqthresh = 1;
3341         rx_ctx.crcstrip = 1;
3342         rx_ctx.l2tsel = 1;
3343         /* this controls whether VLAN is stripped from inner headers */
3344         rx_ctx.showiv = 0;
3345         /* set the prefena field to 1 because the manual says to */
3346         rx_ctx.prefena = 1;
3347
3348         /* clear the context in the HMC */
3349         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3350         if (err) {
3351                 dev_info(&vsi->back->pdev->dev,
3352                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3353                          ring->queue_index, pf_q, err);
3354                 return -ENOMEM;
3355         }
3356
3357         /* set the context in the HMC */
3358         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3359         if (err) {
3360                 dev_info(&vsi->back->pdev->dev,
3361                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3362                          ring->queue_index, pf_q, err);
3363                 return -ENOMEM;
3364         }
3365
3366         /* configure Rx buffer alignment */
3367         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3368                 clear_ring_build_skb_enabled(ring);
3369         else
3370                 set_ring_build_skb_enabled(ring);
3371
3372         /* cache tail for quicker writes, and clear the reg before use */
3373         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3374         writel(0, ring->tail);
3375
3376         if (ring->xsk_pool) {
3377                 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
3378                 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
3379         } else {
3380                 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3381         }
3382         if (!ok) {
3383                 /* Log this in case the user has forgotten to give the kernel
3384                  * any buffers, even later in the application.
3385                  */
3386                 dev_info(&vsi->back->pdev->dev,
3387                          "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3388                          ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3389                          ring->queue_index, pf_q);
3390         }
3391
3392         return 0;
3393 }
3394
3395 /**
3396  * i40e_vsi_configure_tx - Configure the VSI for Tx
3397  * @vsi: VSI structure describing this set of rings and resources
3398  *
3399  * Configure the Tx VSI for operation.
3400  **/
3401 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3402 {
3403         int err = 0;
3404         u16 i;
3405
3406         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3407                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3408
3409         if (err || !i40e_enabled_xdp_vsi(vsi))
3410                 return err;
3411
3412         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3413                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3414
3415         return err;
3416 }
3417
3418 /**
3419  * i40e_vsi_configure_rx - Configure the VSI for Rx
3420  * @vsi: the VSI being configured
3421  *
3422  * Configure the Rx VSI for operation.
3423  **/
3424 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3425 {
3426         int err = 0;
3427         u16 i;
3428
3429         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3430                 vsi->max_frame = I40E_MAX_RXBUFFER;
3431                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3432 #if (PAGE_SIZE < 8192)
3433         } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3434                    (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3435                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3436                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3437 #endif
3438         } else {
3439                 vsi->max_frame = I40E_MAX_RXBUFFER;
3440                 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3441                                                        I40E_RXBUFFER_2048;
3442         }
3443
3444         /* set up individual rings */
3445         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3446                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3447
3448         return err;
3449 }
3450
3451 /**
3452  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3453  * @vsi: ptr to the VSI
3454  **/
3455 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3456 {
3457         struct i40e_ring *tx_ring, *rx_ring;
3458         u16 qoffset, qcount;
3459         int i, n;
3460
3461         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3462                 /* Reset the TC information */
3463                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3464                         rx_ring = vsi->rx_rings[i];
3465                         tx_ring = vsi->tx_rings[i];
3466                         rx_ring->dcb_tc = 0;
3467                         tx_ring->dcb_tc = 0;
3468                 }
3469                 return;
3470         }
3471
3472         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3473                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3474                         continue;
3475
3476                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3477                 qcount = vsi->tc_config.tc_info[n].qcount;
3478                 for (i = qoffset; i < (qoffset + qcount); i++) {
3479                         rx_ring = vsi->rx_rings[i];
3480                         tx_ring = vsi->tx_rings[i];
3481                         rx_ring->dcb_tc = n;
3482                         tx_ring->dcb_tc = n;
3483                 }
3484         }
3485 }
3486
3487 /**
3488  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3489  * @vsi: ptr to the VSI
3490  **/
3491 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3492 {
3493         if (vsi->netdev)
3494                 i40e_set_rx_mode(vsi->netdev);
3495 }
3496
3497 /**
3498  * i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3499  * @pf: Pointer to the targeted PF
3500  *
3501  * Set all flow director counters to 0.
3502  */
3503 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)
3504 {
3505         pf->fd_tcp4_filter_cnt = 0;
3506         pf->fd_udp4_filter_cnt = 0;
3507         pf->fd_sctp4_filter_cnt = 0;
3508         pf->fd_ip4_filter_cnt = 0;
3509         pf->fd_tcp6_filter_cnt = 0;
3510         pf->fd_udp6_filter_cnt = 0;
3511         pf->fd_sctp6_filter_cnt = 0;
3512         pf->fd_ip6_filter_cnt = 0;
3513 }
3514
3515 /**
3516  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3517  * @vsi: Pointer to the targeted VSI
3518  *
3519  * This function replays the hlist on the hw where all the SB Flow Director
3520  * filters were saved.
3521  **/
3522 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3523 {
3524         struct i40e_fdir_filter *filter;
3525         struct i40e_pf *pf = vsi->back;
3526         struct hlist_node *node;
3527
3528         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3529                 return;
3530
3531         /* Reset FDir counters as we're replaying all existing filters */
3532         i40e_reset_fdir_filter_cnt(pf);
3533
3534         hlist_for_each_entry_safe(filter, node,
3535                                   &pf->fdir_filter_list, fdir_node) {
3536                 i40e_add_del_fdir(vsi, filter, true);
3537         }
3538 }
3539
3540 /**
3541  * i40e_vsi_configure - Set up the VSI for action
3542  * @vsi: the VSI being configured
3543  **/
3544 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3545 {
3546         int err;
3547
3548         i40e_set_vsi_rx_mode(vsi);
3549         i40e_restore_vlan(vsi);
3550         i40e_vsi_config_dcb_rings(vsi);
3551         err = i40e_vsi_configure_tx(vsi);
3552         if (!err)
3553                 err = i40e_vsi_configure_rx(vsi);
3554
3555         return err;
3556 }
3557
3558 /**
3559  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3560  * @vsi: the VSI being configured
3561  **/
3562 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3563 {
3564         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3565         struct i40e_pf *pf = vsi->back;
3566         struct i40e_hw *hw = &pf->hw;
3567         u16 vector;
3568         int i, q;
3569         u32 qp;
3570
3571         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3572          * and PFINT_LNKLSTn registers, e.g.:
3573          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3574          */
3575         qp = vsi->base_queue;
3576         vector = vsi->base_vector;
3577         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3578                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3579
3580                 q_vector->rx.next_update = jiffies + 1;
3581                 q_vector->rx.target_itr =
3582                         ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3583                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3584                      q_vector->rx.target_itr >> 1);
3585                 q_vector->rx.current_itr = q_vector->rx.target_itr;
3586
3587                 q_vector->tx.next_update = jiffies + 1;
3588                 q_vector->tx.target_itr =
3589                         ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3590                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3591                      q_vector->tx.target_itr >> 1);
3592                 q_vector->tx.current_itr = q_vector->tx.target_itr;
3593
3594                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3595                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3596
3597                 /* Linked list for the queuepairs assigned to this vector */
3598                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3599                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3600                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3601                         u32 val;
3602
3603                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3604                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3605                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3606                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3607                               (I40E_QUEUE_TYPE_TX <<
3608                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3609
3610                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3611
3612                         if (has_xdp) {
3613                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3614                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3615                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3616                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3617                                       (I40E_QUEUE_TYPE_TX <<
3618                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3619
3620                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3621                         }
3622
3623                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3624                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3625                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3626                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3627                               (I40E_QUEUE_TYPE_RX <<
3628                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3629
3630                         /* Terminate the linked list */
3631                         if (q == (q_vector->num_ringpairs - 1))
3632                                 val |= (I40E_QUEUE_END_OF_LIST <<
3633                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3634
3635                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3636                         qp++;
3637                 }
3638         }
3639
3640         i40e_flush(hw);
3641 }
3642
3643 /**
3644  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3645  * @pf: pointer to private device data structure
3646  **/
3647 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3648 {
3649         struct i40e_hw *hw = &pf->hw;
3650         u32 val;
3651
3652         /* clear things first */
3653         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3654         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3655
3656         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3657               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3658               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3659               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3660               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3661               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3662               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3663               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3664
3665         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3666                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3667
3668         if (pf->flags & I40E_FLAG_PTP)
3669                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3670
3671         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3672
3673         /* SW_ITR_IDX = 0, but don't change INTENA */
3674         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3675                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3676
3677         /* OTHER_ITR_IDX = 0 */
3678         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3679 }
3680
3681 /**
3682  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3683  * @vsi: the VSI being configured
3684  **/
3685 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3686 {
3687         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3688         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3689         struct i40e_pf *pf = vsi->back;
3690         struct i40e_hw *hw = &pf->hw;
3691         u32 val;
3692
3693         /* set the ITR configuration */
3694         q_vector->rx.next_update = jiffies + 1;
3695         q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
3696         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
3697         q_vector->rx.current_itr = q_vector->rx.target_itr;
3698         q_vector->tx.next_update = jiffies + 1;
3699         q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
3700         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
3701         q_vector->tx.current_itr = q_vector->tx.target_itr;
3702
3703         i40e_enable_misc_int_causes(pf);
3704
3705         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3706         wr32(hw, I40E_PFINT_LNKLST0, 0);
3707
3708         /* Associate the queue pair to the vector and enable the queue int */
3709         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                   |
3710               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3711               (nextqp      << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3712               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3713
3714         wr32(hw, I40E_QINT_RQCTL(0), val);
3715
3716         if (i40e_enabled_xdp_vsi(vsi)) {
3717                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                 |
3718                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3719                       (I40E_QUEUE_TYPE_TX
3720                        << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3721
3722                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3723         }
3724
3725         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3726               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3727               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3728
3729         wr32(hw, I40E_QINT_TQCTL(0), val);
3730         i40e_flush(hw);
3731 }
3732
3733 /**
3734  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3735  * @pf: board private structure
3736  **/
3737 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3738 {
3739         struct i40e_hw *hw = &pf->hw;
3740
3741         wr32(hw, I40E_PFINT_DYN_CTL0,
3742              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3743         i40e_flush(hw);
3744 }
3745
3746 /**
3747  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3748  * @pf: board private structure
3749  **/
3750 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
3751 {
3752         struct i40e_hw *hw = &pf->hw;
3753         u32 val;
3754
3755         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3756               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3757               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3758
3759         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3760         i40e_flush(hw);
3761 }
3762
3763 /**
3764  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3765  * @irq: interrupt number
3766  * @data: pointer to a q_vector
3767  **/
3768 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3769 {
3770         struct i40e_q_vector *q_vector = data;
3771
3772         if (!q_vector->tx.ring && !q_vector->rx.ring)
3773                 return IRQ_HANDLED;
3774
3775         napi_schedule_irqoff(&q_vector->napi);
3776
3777         return IRQ_HANDLED;
3778 }
3779
3780 /**
3781  * i40e_irq_affinity_notify - Callback for affinity changes
3782  * @notify: context as to what irq was changed
3783  * @mask: the new affinity mask
3784  *
3785  * This is a callback function used by the irq_set_affinity_notifier function
3786  * so that we may register to receive changes to the irq affinity masks.
3787  **/
3788 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3789                                      const cpumask_t *mask)
3790 {
3791         struct i40e_q_vector *q_vector =
3792                 container_of(notify, struct i40e_q_vector, affinity_notify);
3793
3794         cpumask_copy(&q_vector->affinity_mask, mask);
3795 }
3796
3797 /**
3798  * i40e_irq_affinity_release - Callback for affinity notifier release
3799  * @ref: internal core kernel usage
3800  *
3801  * This is a callback function used by the irq_set_affinity_notifier function
3802  * to inform the current notification subscriber that they will no longer
3803  * receive notifications.
3804  **/
3805 static void i40e_irq_affinity_release(struct kref *ref) {}
3806
3807 /**
3808  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3809  * @vsi: the VSI being configured
3810  * @basename: name for the vector
3811  *
3812  * Allocates MSI-X vectors and requests interrupts from the kernel.
3813  **/
3814 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3815 {
3816         int q_vectors = vsi->num_q_vectors;
3817         struct i40e_pf *pf = vsi->back;
3818         int base = vsi->base_vector;
3819         int rx_int_idx = 0;
3820         int tx_int_idx = 0;
3821         int vector, err;
3822         int irq_num;
3823         int cpu;
3824
3825         for (vector = 0; vector < q_vectors; vector++) {
3826                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3827
3828                 irq_num = pf->msix_entries[base + vector].vector;
3829
3830                 if (q_vector->tx.ring && q_vector->rx.ring) {
3831                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3832                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3833                         tx_int_idx++;
3834                 } else if (q_vector->rx.ring) {
3835                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3836                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3837                 } else if (q_vector->tx.ring) {
3838                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3839                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3840                 } else {
3841                         /* skip this unused q_vector */
3842                         continue;
3843                 }
3844                 err = request_irq(irq_num,
3845                                   vsi->irq_handler,
3846                                   0,
3847                                   q_vector->name,
3848                                   q_vector);
3849                 if (err) {
3850                         dev_info(&pf->pdev->dev,
3851                                  "MSIX request_irq failed, error: %d\n", err);
3852                         goto free_queue_irqs;
3853                 }
3854
3855                 /* register for affinity change notifications */
3856                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3857                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3858                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3859                 /* Spread affinity hints out across online CPUs.
3860                  *
3861                  * get_cpu_mask returns a static constant mask with
3862                  * a permanent lifetime so it's ok to pass to
3863                  * irq_set_affinity_hint without making a copy.
3864                  */
3865                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
3866                 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
3867         }
3868
3869         vsi->irqs_ready = true;
3870         return 0;
3871
3872 free_queue_irqs:
3873         while (vector) {
3874                 vector--;
3875                 irq_num = pf->msix_entries[base + vector].vector;
3876                 irq_set_affinity_notifier(irq_num, NULL);
3877                 irq_set_affinity_hint(irq_num, NULL);
3878                 free_irq(irq_num, &vsi->q_vectors[vector]);
3879         }
3880         return err;
3881 }
3882
3883 /**
3884  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3885  * @vsi: the VSI being un-configured
3886  **/
3887 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3888 {
3889         struct i40e_pf *pf = vsi->back;
3890         struct i40e_hw *hw = &pf->hw;
3891         int base = vsi->base_vector;
3892         int i;
3893
3894         /* disable interrupt causation from each queue */
3895         for (i = 0; i < vsi->num_queue_pairs; i++) {
3896                 u32 val;
3897
3898                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
3899                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3900                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
3901
3902                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
3903                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3904                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
3905
3906                 if (!i40e_enabled_xdp_vsi(vsi))
3907                         continue;
3908                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3909         }
3910
3911         /* disable each interrupt */
3912         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3913                 for (i = vsi->base_vector;
3914                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3915                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3916
3917                 i40e_flush(hw);
3918                 for (i = 0; i < vsi->num_q_vectors; i++)
3919                         synchronize_irq(pf->msix_entries[i + base].vector);
3920         } else {
3921                 /* Legacy and MSI mode - this stops all interrupt handling */
3922                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3923                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3924                 i40e_flush(hw);
3925                 synchronize_irq(pf->pdev->irq);
3926         }
3927 }
3928
3929 /**
3930  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3931  * @vsi: the VSI being configured
3932  **/
3933 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3934 {
3935         struct i40e_pf *pf = vsi->back;
3936         int i;
3937
3938         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3939                 for (i = 0; i < vsi->num_q_vectors; i++)
3940                         i40e_irq_dynamic_enable(vsi, i);
3941         } else {
3942                 i40e_irq_dynamic_enable_icr0(pf);
3943         }
3944
3945         i40e_flush(&pf->hw);
3946         return 0;
3947 }
3948
3949 /**
3950  * i40e_free_misc_vector - Free the vector that handles non-queue events
3951  * @pf: board private structure
3952  **/
3953 static void i40e_free_misc_vector(struct i40e_pf *pf)
3954 {
3955         /* Disable ICR 0 */
3956         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3957         i40e_flush(&pf->hw);
3958
3959         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
3960                 synchronize_irq(pf->msix_entries[0].vector);
3961                 free_irq(pf->msix_entries[0].vector, pf);
3962                 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
3963         }
3964 }
3965
3966 /**
3967  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3968  * @irq: interrupt number
3969  * @data: pointer to a q_vector
3970  *
3971  * This is the handler used for all MSI/Legacy interrupts, and deals
3972  * with both queue and non-queue interrupts.  This is also used in
3973  * MSIX mode to handle the non-queue interrupts.
3974  **/
3975 static irqreturn_t i40e_intr(int irq, void *data)
3976 {
3977         struct i40e_pf *pf = (struct i40e_pf *)data;
3978         struct i40e_hw *hw = &pf->hw;
3979         irqreturn_t ret = IRQ_NONE;
3980         u32 icr0, icr0_remaining;
3981         u32 val, ena_mask;
3982
3983         icr0 = rd32(hw, I40E_PFINT_ICR0);
3984         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3985
3986         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3987         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3988                 goto enable_intr;
3989
3990         /* if interrupt but no bits showing, must be SWINT */
3991         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3992             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3993                 pf->sw_int_count++;
3994
3995         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3996             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3997                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3998                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3999                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
4000         }
4001
4002         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4003         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4004                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
4005                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4006
4007                 /* We do not have a way to disarm Queue causes while leaving
4008                  * interrupt enabled for all other causes, ideally
4009                  * interrupt should be disabled while we are in NAPI but
4010                  * this is not a performance path and napi_schedule()
4011                  * can deal with rescheduling.
4012                  */
4013                 if (!test_bit(__I40E_DOWN, pf->state))
4014                         napi_schedule_irqoff(&q_vector->napi);
4015         }
4016
4017         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4018                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4019                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
4020                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4021         }
4022
4023         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4024                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4025                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
4026         }
4027
4028         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4029                 /* disable any further VFLR event notifications */
4030                 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4031                         u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4032
4033                         reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4034                         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4035                 } else {
4036                         ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4037                         set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4038                 }
4039         }
4040
4041         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4042                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4043                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4044                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4045                 val = rd32(hw, I40E_GLGEN_RSTAT);
4046                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
4047                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
4048                 if (val == I40E_RESET_CORER) {
4049                         pf->corer_count++;
4050                 } else if (val == I40E_RESET_GLOBR) {
4051                         pf->globr_count++;
4052                 } else if (val == I40E_RESET_EMPR) {
4053                         pf->empr_count++;
4054                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4055                 }
4056         }
4057
4058         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4059                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4060                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4061                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4062                          rd32(hw, I40E_PFHMC_ERRORINFO),
4063                          rd32(hw, I40E_PFHMC_ERRORDATA));
4064         }
4065
4066         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4067                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4068
4069                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
4070                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4071                         i40e_ptp_tx_hwtstamp(pf);
4072                 }
4073         }
4074
4075         /* If a critical error is pending we have no choice but to reset the
4076          * device.
4077          * Report and mask out any remaining unexpected interrupts.
4078          */
4079         icr0_remaining = icr0 & ena_mask;
4080         if (icr0_remaining) {
4081                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4082                          icr0_remaining);
4083                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4084                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4085                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4086                         dev_info(&pf->pdev->dev, "device will be reset\n");
4087                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4088                         i40e_service_event_schedule(pf);
4089                 }
4090                 ena_mask &= ~icr0_remaining;
4091         }
4092         ret = IRQ_HANDLED;
4093
4094 enable_intr:
4095         /* re-enable interrupt causes */
4096         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4097         if (!test_bit(__I40E_DOWN, pf->state) ||
4098             test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4099                 i40e_service_event_schedule(pf);
4100                 i40e_irq_dynamic_enable_icr0(pf);
4101         }
4102
4103         return ret;
4104 }
4105
4106 /**
4107  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4108  * @tx_ring:  tx ring to clean
4109  * @budget:   how many cleans we're allowed
4110  *
4111  * Returns true if there's any budget left (e.g. the clean is finished)
4112  **/
4113 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4114 {
4115         struct i40e_vsi *vsi = tx_ring->vsi;
4116         u16 i = tx_ring->next_to_clean;
4117         struct i40e_tx_buffer *tx_buf;
4118         struct i40e_tx_desc *tx_desc;
4119
4120         tx_buf = &tx_ring->tx_bi[i];
4121         tx_desc = I40E_TX_DESC(tx_ring, i);
4122         i -= tx_ring->count;
4123
4124         do {
4125                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4126
4127                 /* if next_to_watch is not set then there is no work pending */
4128                 if (!eop_desc)
4129                         break;
4130
4131                 /* prevent any other reads prior to eop_desc */
4132                 smp_rmb();
4133
4134                 /* if the descriptor isn't done, no work yet to do */
4135                 if (!(eop_desc->cmd_type_offset_bsz &
4136                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4137                         break;
4138
4139                 /* clear next_to_watch to prevent false hangs */
4140                 tx_buf->next_to_watch = NULL;
4141
4142                 tx_desc->buffer_addr = 0;
4143                 tx_desc->cmd_type_offset_bsz = 0;
4144                 /* move past filter desc */
4145                 tx_buf++;
4146                 tx_desc++;
4147                 i++;
4148                 if (unlikely(!i)) {
4149                         i -= tx_ring->count;
4150                         tx_buf = tx_ring->tx_bi;
4151                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4152                 }
4153                 /* unmap skb header data */
4154                 dma_unmap_single(tx_ring->dev,
4155                                  dma_unmap_addr(tx_buf, dma),
4156                                  dma_unmap_len(tx_buf, len),
4157                                  DMA_TO_DEVICE);
4158                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4159                         kfree(tx_buf->raw_buf);
4160
4161                 tx_buf->raw_buf = NULL;
4162                 tx_buf->tx_flags = 0;
4163                 tx_buf->next_to_watch = NULL;
4164                 dma_unmap_len_set(tx_buf, len, 0);
4165                 tx_desc->buffer_addr = 0;
4166                 tx_desc->cmd_type_offset_bsz = 0;
4167
4168                 /* move us past the eop_desc for start of next FD desc */
4169                 tx_buf++;
4170                 tx_desc++;
4171                 i++;
4172                 if (unlikely(!i)) {
4173                         i -= tx_ring->count;
4174                         tx_buf = tx_ring->tx_bi;
4175                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4176                 }
4177
4178                 /* update budget accounting */
4179                 budget--;
4180         } while (likely(budget));
4181
4182         i += tx_ring->count;
4183         tx_ring->next_to_clean = i;
4184
4185         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
4186                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4187
4188         return budget > 0;
4189 }
4190
4191 /**
4192  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4193  * @irq: interrupt number
4194  * @data: pointer to a q_vector
4195  **/
4196 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4197 {
4198         struct i40e_q_vector *q_vector = data;
4199         struct i40e_vsi *vsi;
4200
4201         if (!q_vector->tx.ring)
4202                 return IRQ_HANDLED;
4203
4204         vsi = q_vector->tx.ring->vsi;
4205         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4206
4207         return IRQ_HANDLED;
4208 }
4209
4210 /**
4211  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4212  * @vsi: the VSI being configured
4213  * @v_idx: vector index
4214  * @qp_idx: queue pair index
4215  **/
4216 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4217 {
4218         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4219         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4220         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4221
4222         tx_ring->q_vector = q_vector;
4223         tx_ring->next = q_vector->tx.ring;
4224         q_vector->tx.ring = tx_ring;
4225         q_vector->tx.count++;
4226
4227         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4228         if (i40e_enabled_xdp_vsi(vsi)) {
4229                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4230
4231                 xdp_ring->q_vector = q_vector;
4232                 xdp_ring->next = q_vector->tx.ring;
4233                 q_vector->tx.ring = xdp_ring;
4234                 q_vector->tx.count++;
4235         }
4236
4237         rx_ring->q_vector = q_vector;
4238         rx_ring->next = q_vector->rx.ring;
4239         q_vector->rx.ring = rx_ring;
4240         q_vector->rx.count++;
4241 }
4242
4243 /**
4244  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4245  * @vsi: the VSI being configured
4246  *
4247  * This function maps descriptor rings to the queue-specific vectors
4248  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4249  * one vector per queue pair, but on a constrained vector budget, we
4250  * group the queue pairs as "efficiently" as possible.
4251  **/
4252 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4253 {
4254         int qp_remaining = vsi->num_queue_pairs;
4255         int q_vectors = vsi->num_q_vectors;
4256         int num_ringpairs;
4257         int v_start = 0;
4258         int qp_idx = 0;
4259
4260         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4261          * group them so there are multiple queues per vector.
4262          * It is also important to go through all the vectors available to be
4263          * sure that if we don't use all the vectors, that the remaining vectors
4264          * are cleared. This is especially important when decreasing the
4265          * number of queues in use.
4266          */
4267         for (; v_start < q_vectors; v_start++) {
4268                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4269
4270                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4271
4272                 q_vector->num_ringpairs = num_ringpairs;
4273                 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4274
4275                 q_vector->rx.count = 0;
4276                 q_vector->tx.count = 0;
4277                 q_vector->rx.ring = NULL;
4278                 q_vector->tx.ring = NULL;
4279
4280                 while (num_ringpairs--) {
4281                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4282                         qp_idx++;
4283                         qp_remaining--;
4284                 }
4285         }
4286 }
4287
4288 /**
4289  * i40e_vsi_request_irq - Request IRQ from the OS
4290  * @vsi: the VSI being configured
4291  * @basename: name for the vector
4292  **/
4293 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4294 {
4295         struct i40e_pf *pf = vsi->back;
4296         int err;
4297
4298         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4299                 err = i40e_vsi_request_irq_msix(vsi, basename);
4300         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
4301                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4302                                   pf->int_name, pf);
4303         else
4304                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4305                                   pf->int_name, pf);
4306
4307         if (err)
4308                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4309
4310         return err;
4311 }
4312
4313 #ifdef CONFIG_NET_POLL_CONTROLLER
4314 /**
4315  * i40e_netpoll - A Polling 'interrupt' handler
4316  * @netdev: network interface device structure
4317  *
4318  * This is used by netconsole to send skbs without having to re-enable
4319  * interrupts.  It's not called while the normal interrupt routine is executing.
4320  **/
4321 static void i40e_netpoll(struct net_device *netdev)
4322 {
4323         struct i40e_netdev_priv *np = netdev_priv(netdev);
4324         struct i40e_vsi *vsi = np->vsi;
4325         struct i40e_pf *pf = vsi->back;
4326         int i;
4327
4328         /* if interface is down do nothing */
4329         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4330                 return;
4331
4332         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4333                 for (i = 0; i < vsi->num_q_vectors; i++)
4334                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4335         } else {
4336                 i40e_intr(pf->pdev->irq, netdev);
4337         }
4338 }
4339 #endif
4340
4341 #define I40E_QTX_ENA_WAIT_COUNT 50
4342
4343 /**
4344  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4345  * @pf: the PF being configured
4346  * @pf_q: the PF queue
4347  * @enable: enable or disable state of the queue
4348  *
4349  * This routine will wait for the given Tx queue of the PF to reach the
4350  * enabled or disabled state.
4351  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4352  * multiple retries; else will return 0 in case of success.
4353  **/
4354 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4355 {
4356         int i;
4357         u32 tx_reg;
4358
4359         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4360                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4361                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4362                         break;
4363
4364                 usleep_range(10, 20);
4365         }
4366         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4367                 return -ETIMEDOUT;
4368
4369         return 0;
4370 }
4371
4372 /**
4373  * i40e_control_tx_q - Start or stop a particular Tx queue
4374  * @pf: the PF structure
4375  * @pf_q: the PF queue to configure
4376  * @enable: start or stop the queue
4377  *
4378  * This function enables or disables a single queue. Note that any delay
4379  * required after the operation is expected to be handled by the caller of
4380  * this function.
4381  **/
4382 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4383 {
4384         struct i40e_hw *hw = &pf->hw;
4385         u32 tx_reg;
4386         int i;
4387
4388         /* warn the TX unit of coming changes */
4389         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4390         if (!enable)
4391                 usleep_range(10, 20);
4392
4393         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4394                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4395                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4396                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4397                         break;
4398                 usleep_range(1000, 2000);
4399         }
4400
4401         /* Skip if the queue is already in the requested state */
4402         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4403                 return;
4404
4405         /* turn on/off the queue */
4406         if (enable) {
4407                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4408                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4409         } else {
4410                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4411         }
4412
4413         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4414 }
4415
4416 /**
4417  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4418  * @seid: VSI SEID
4419  * @pf: the PF structure
4420  * @pf_q: the PF queue to configure
4421  * @is_xdp: true if the queue is used for XDP
4422  * @enable: start or stop the queue
4423  **/
4424 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4425                            bool is_xdp, bool enable)
4426 {
4427         int ret;
4428
4429         i40e_control_tx_q(pf, pf_q, enable);
4430
4431         /* wait for the change to finish */
4432         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4433         if (ret) {
4434                 dev_info(&pf->pdev->dev,
4435                          "VSI seid %d %sTx ring %d %sable timeout\n",
4436                          seid, (is_xdp ? "XDP " : ""), pf_q,
4437                          (enable ? "en" : "dis"));
4438         }
4439
4440         return ret;
4441 }
4442
4443 /**
4444  * i40e_vsi_control_tx - Start or stop a VSI's rings
4445  * @vsi: the VSI being configured
4446  * @enable: start or stop the rings
4447  **/
4448 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
4449 {
4450         struct i40e_pf *pf = vsi->back;
4451         int i, pf_q, ret = 0;
4452
4453         pf_q = vsi->base_queue;
4454         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4455                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4456                                              pf_q,
4457                                              false /*is xdp*/, enable);
4458                 if (ret)
4459                         break;
4460
4461                 if (!i40e_enabled_xdp_vsi(vsi))
4462                         continue;
4463
4464                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4465                                              pf_q + vsi->alloc_queue_pairs,
4466                                              true /*is xdp*/, enable);
4467                 if (ret)
4468                         break;
4469         }
4470         return ret;
4471 }
4472
4473 /**
4474  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4475  * @pf: the PF being configured
4476  * @pf_q: the PF queue
4477  * @enable: enable or disable state of the queue
4478  *
4479  * This routine will wait for the given Rx queue of the PF to reach the
4480  * enabled or disabled state.
4481  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4482  * multiple retries; else will return 0 in case of success.
4483  **/
4484 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4485 {
4486         int i;
4487         u32 rx_reg;
4488
4489         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4490                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4491                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4492                         break;
4493
4494                 usleep_range(10, 20);
4495         }
4496         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4497                 return -ETIMEDOUT;
4498
4499         return 0;
4500 }
4501
4502 /**
4503  * i40e_control_rx_q - Start or stop a particular Rx queue
4504  * @pf: the PF structure
4505  * @pf_q: the PF queue to configure
4506  * @enable: start or stop the queue
4507  *
4508  * This function enables or disables a single queue. Note that
4509  * any delay required after the operation is expected to be
4510  * handled by the caller of this function.
4511  **/
4512 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4513 {
4514         struct i40e_hw *hw = &pf->hw;
4515         u32 rx_reg;
4516         int i;
4517
4518         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4519                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4520                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4521                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4522                         break;
4523                 usleep_range(1000, 2000);
4524         }
4525
4526         /* Skip if the queue is already in the requested state */
4527         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4528                 return;
4529
4530         /* turn on/off the queue */
4531         if (enable)
4532                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4533         else
4534                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4535
4536         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4537 }
4538
4539 /**
4540  * i40e_control_wait_rx_q
4541  * @pf: the PF structure
4542  * @pf_q: queue being configured
4543  * @enable: start or stop the rings
4544  *
4545  * This function enables or disables a single queue along with waiting
4546  * for the change to finish. The caller of this function should handle
4547  * the delays needed in the case of disabling queues.
4548  **/
4549 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4550 {
4551         int ret = 0;
4552
4553         i40e_control_rx_q(pf, pf_q, enable);
4554
4555         /* wait for the change to finish */
4556         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4557         if (ret)
4558                 return ret;
4559
4560         return ret;
4561 }
4562
4563 /**
4564  * i40e_vsi_control_rx - Start or stop a VSI's rings
4565  * @vsi: the VSI being configured
4566  * @enable: start or stop the rings
4567  **/
4568 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
4569 {
4570         struct i40e_pf *pf = vsi->back;
4571         int i, pf_q, ret = 0;
4572
4573         pf_q = vsi->base_queue;
4574         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4575                 ret = i40e_control_wait_rx_q(pf, pf_q, enable);
4576                 if (ret) {
4577                         dev_info(&pf->pdev->dev,
4578                                  "VSI seid %d Rx ring %d %sable timeout\n",
4579                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
4580                         break;
4581                 }
4582         }
4583
4584         /* Due to HW errata, on Rx disable only, the register can indicate done
4585          * before it really is. Needs 50ms to be sure
4586          */
4587         if (!enable)
4588                 mdelay(50);
4589
4590         return ret;
4591 }
4592
4593 /**
4594  * i40e_vsi_start_rings - Start a VSI's rings
4595  * @vsi: the VSI being configured
4596  **/
4597 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4598 {
4599         int ret = 0;
4600
4601         /* do rx first for enable and last for disable */
4602         ret = i40e_vsi_control_rx(vsi, true);
4603         if (ret)
4604                 return ret;
4605         ret = i40e_vsi_control_tx(vsi, true);
4606
4607         return ret;
4608 }
4609
4610 /**
4611  * i40e_vsi_stop_rings - Stop a VSI's rings
4612  * @vsi: the VSI being configured
4613  **/
4614 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4615 {
4616         /* When port TX is suspended, don't wait */
4617         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4618                 return i40e_vsi_stop_rings_no_wait(vsi);
4619
4620         /* do rx first for enable and last for disable
4621          * Ignore return value, we need to shutdown whatever we can
4622          */
4623         i40e_vsi_control_tx(vsi, false);
4624         i40e_vsi_control_rx(vsi, false);
4625 }
4626
4627 /**
4628  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4629  * @vsi: the VSI being shutdown
4630  *
4631  * This function stops all the rings for a VSI but does not delay to verify
4632  * that rings have been disabled. It is expected that the caller is shutting
4633  * down multiple VSIs at once and will delay together for all the VSIs after
4634  * initiating the shutdown. This is particularly useful for shutting down lots
4635  * of VFs together. Otherwise, a large delay can be incurred while configuring
4636  * each VSI in serial.
4637  **/
4638 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4639 {
4640         struct i40e_pf *pf = vsi->back;
4641         int i, pf_q;
4642
4643         pf_q = vsi->base_queue;
4644         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4645                 i40e_control_tx_q(pf, pf_q, false);
4646                 i40e_control_rx_q(pf, pf_q, false);
4647         }
4648 }
4649
4650 /**
4651  * i40e_vsi_free_irq - Free the irq association with the OS
4652  * @vsi: the VSI being configured
4653  **/
4654 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4655 {
4656         struct i40e_pf *pf = vsi->back;
4657         struct i40e_hw *hw = &pf->hw;
4658         int base = vsi->base_vector;
4659         u32 val, qp;
4660         int i;
4661
4662         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4663                 if (!vsi->q_vectors)
4664                         return;
4665
4666                 if (!vsi->irqs_ready)
4667                         return;
4668
4669                 vsi->irqs_ready = false;
4670                 for (i = 0; i < vsi->num_q_vectors; i++) {
4671                         int irq_num;
4672                         u16 vector;
4673
4674                         vector = i + base;
4675                         irq_num = pf->msix_entries[vector].vector;
4676
4677                         /* free only the irqs that were actually requested */
4678                         if (!vsi->q_vectors[i] ||
4679                             !vsi->q_vectors[i]->num_ringpairs)
4680                                 continue;
4681
4682                         /* clear the affinity notifier in the IRQ descriptor */
4683                         irq_set_affinity_notifier(irq_num, NULL);
4684                         /* remove our suggested affinity mask for this IRQ */
4685                         irq_set_affinity_hint(irq_num, NULL);
4686                         synchronize_irq(irq_num);
4687                         free_irq(irq_num, vsi->q_vectors[i]);
4688
4689                         /* Tear down the interrupt queue link list
4690                          *
4691                          * We know that they come in pairs and always
4692                          * the Rx first, then the Tx.  To clear the
4693                          * link list, stick the EOL value into the
4694                          * next_q field of the registers.
4695                          */
4696                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4697                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4698                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4699                         val |= I40E_QUEUE_END_OF_LIST
4700                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4701                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4702
4703                         while (qp != I40E_QUEUE_END_OF_LIST) {
4704                                 u32 next;
4705
4706                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4707
4708                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4709                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4710                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4711                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4712
4713                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4714                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4715
4716                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4717
4718                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4719
4720                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4721                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4722
4723                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4724                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4725                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4726                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4727
4728                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4729                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4730
4731                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4732                                 qp = next;
4733                         }
4734                 }
4735         } else {
4736                 free_irq(pf->pdev->irq, pf);
4737
4738                 val = rd32(hw, I40E_PFINT_LNKLST0);
4739                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4740                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4741                 val |= I40E_QUEUE_END_OF_LIST
4742                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4743                 wr32(hw, I40E_PFINT_LNKLST0, val);
4744
4745                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4746                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4747                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4748                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4749                          I40E_QINT_RQCTL_INTEVENT_MASK);
4750
4751                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4752                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4753
4754                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4755
4756                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4757
4758                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4759                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4760                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4761                          I40E_QINT_TQCTL_INTEVENT_MASK);
4762
4763                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4764                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4765
4766                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4767         }
4768 }
4769
4770 /**
4771  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4772  * @vsi: the VSI being configured
4773  * @v_idx: Index of vector to be freed
4774  *
4775  * This function frees the memory allocated to the q_vector.  In addition if
4776  * NAPI is enabled it will delete any references to the NAPI struct prior
4777  * to freeing the q_vector.
4778  **/
4779 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4780 {
4781         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4782         struct i40e_ring *ring;
4783
4784         if (!q_vector)
4785                 return;
4786
4787         /* disassociate q_vector from rings */
4788         i40e_for_each_ring(ring, q_vector->tx)
4789                 ring->q_vector = NULL;
4790
4791         i40e_for_each_ring(ring, q_vector->rx)
4792                 ring->q_vector = NULL;
4793
4794         /* only VSI w/ an associated netdev is set up w/ NAPI */
4795         if (vsi->netdev)
4796                 netif_napi_del(&q_vector->napi);
4797
4798         vsi->q_vectors[v_idx] = NULL;
4799
4800         kfree_rcu(q_vector, rcu);
4801 }
4802
4803 /**
4804  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4805  * @vsi: the VSI being un-configured
4806  *
4807  * This frees the memory allocated to the q_vectors and
4808  * deletes references to the NAPI struct.
4809  **/
4810 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4811 {
4812         int v_idx;
4813
4814         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4815                 i40e_free_q_vector(vsi, v_idx);
4816 }
4817
4818 /**
4819  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4820  * @pf: board private structure
4821  **/
4822 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4823 {
4824         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4825         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4826                 pci_disable_msix(pf->pdev);
4827                 kfree(pf->msix_entries);
4828                 pf->msix_entries = NULL;
4829                 kfree(pf->irq_pile);
4830                 pf->irq_pile = NULL;
4831         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4832                 pci_disable_msi(pf->pdev);
4833         }
4834         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4835 }
4836
4837 /**
4838  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4839  * @pf: board private structure
4840  *
4841  * We go through and clear interrupt specific resources and reset the structure
4842  * to pre-load conditions
4843  **/
4844 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4845 {
4846         int i;
4847
4848         i40e_free_misc_vector(pf);
4849
4850         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4851                       I40E_IWARP_IRQ_PILE_ID);
4852
4853         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4854         for (i = 0; i < pf->num_alloc_vsi; i++)
4855                 if (pf->vsi[i])
4856                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4857         i40e_reset_interrupt_capability(pf);
4858 }
4859
4860 /**
4861  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4862  * @vsi: the VSI being configured
4863  **/
4864 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4865 {
4866         int q_idx;
4867
4868         if (!vsi->netdev)
4869                 return;
4870
4871         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4872                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4873
4874                 if (q_vector->rx.ring || q_vector->tx.ring)
4875                         napi_enable(&q_vector->napi);
4876         }
4877 }
4878
4879 /**
4880  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4881  * @vsi: the VSI being configured
4882  **/
4883 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4884 {
4885         int q_idx;
4886
4887         if (!vsi->netdev)
4888                 return;
4889
4890         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4891                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4892
4893                 if (q_vector->rx.ring || q_vector->tx.ring)
4894                         napi_disable(&q_vector->napi);
4895         }
4896 }
4897
4898 /**
4899  * i40e_vsi_close - Shut down a VSI
4900  * @vsi: the vsi to be quelled
4901  **/
4902 static void i40e_vsi_close(struct i40e_vsi *vsi)
4903 {
4904         struct i40e_pf *pf = vsi->back;
4905         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4906                 i40e_down(vsi);
4907         i40e_vsi_free_irq(vsi);
4908         i40e_vsi_free_tx_resources(vsi);
4909         i40e_vsi_free_rx_resources(vsi);
4910         vsi->current_netdev_flags = 0;
4911         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
4912         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4913                 set_bit(__I40E_CLIENT_RESET, pf->state);
4914 }
4915
4916 /**
4917  * i40e_quiesce_vsi - Pause a given VSI
4918  * @vsi: the VSI being paused
4919  **/
4920 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4921 {
4922         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4923                 return;
4924
4925         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4926         if (vsi->netdev && netif_running(vsi->netdev))
4927                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4928         else
4929                 i40e_vsi_close(vsi);
4930 }
4931
4932 /**
4933  * i40e_unquiesce_vsi - Resume a given VSI
4934  * @vsi: the VSI being resumed
4935  **/
4936 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4937 {
4938         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
4939                 return;
4940
4941         if (vsi->netdev && netif_running(vsi->netdev))
4942                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4943         else
4944                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4945 }
4946
4947 /**
4948  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4949  * @pf: the PF
4950  **/
4951 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4952 {
4953         int v;
4954
4955         for (v = 0; v < pf->num_alloc_vsi; v++) {
4956                 if (pf->vsi[v])
4957                         i40e_quiesce_vsi(pf->vsi[v]);
4958         }
4959 }
4960
4961 /**
4962  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4963  * @pf: the PF
4964  **/
4965 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4966 {
4967         int v;
4968
4969         for (v = 0; v < pf->num_alloc_vsi; v++) {
4970                 if (pf->vsi[v])
4971                         i40e_unquiesce_vsi(pf->vsi[v]);
4972         }
4973 }
4974
4975 /**
4976  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4977  * @vsi: the VSI being configured
4978  *
4979  * Wait until all queues on a given VSI have been disabled.
4980  **/
4981 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4982 {
4983         struct i40e_pf *pf = vsi->back;
4984         int i, pf_q, ret;
4985
4986         pf_q = vsi->base_queue;
4987         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4988                 /* Check and wait for the Tx queue */
4989                 ret = i40e_pf_txq_wait(pf, pf_q, false);
4990                 if (ret) {
4991                         dev_info(&pf->pdev->dev,
4992                                  "VSI seid %d Tx ring %d disable timeout\n",
4993                                  vsi->seid, pf_q);
4994                         return ret;
4995                 }
4996
4997                 if (!i40e_enabled_xdp_vsi(vsi))
4998                         goto wait_rx;
4999
5000                 /* Check and wait for the XDP Tx queue */
5001                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
5002                                        false);
5003                 if (ret) {
5004                         dev_info(&pf->pdev->dev,
5005                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
5006                                  vsi->seid, pf_q);
5007                         return ret;
5008                 }
5009 wait_rx:
5010                 /* Check and wait for the Rx queue */
5011                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
5012                 if (ret) {
5013                         dev_info(&pf->pdev->dev,
5014                                  "VSI seid %d Rx ring %d disable timeout\n",
5015                                  vsi->seid, pf_q);
5016                         return ret;
5017                 }
5018         }
5019
5020         return 0;
5021 }
5022
5023 #ifdef CONFIG_I40E_DCB
5024 /**
5025  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5026  * @pf: the PF
5027  *
5028  * This function waits for the queues to be in disabled state for all the
5029  * VSIs that are managed by this PF.
5030  **/
5031 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5032 {
5033         int v, ret = 0;
5034
5035         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
5036                 if (pf->vsi[v]) {
5037                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
5038                         if (ret)
5039                                 break;
5040                 }
5041         }
5042
5043         return ret;
5044 }
5045
5046 #endif
5047
5048 /**
5049  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5050  * @pf: pointer to PF
5051  *
5052  * Get TC map for ISCSI PF type that will include iSCSI TC
5053  * and LAN TC.
5054  **/
5055 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5056 {
5057         struct i40e_dcb_app_priority_table app;
5058         struct i40e_hw *hw = &pf->hw;
5059         u8 enabled_tc = 1; /* TC0 is always enabled */
5060         u8 tc, i;
5061         /* Get the iSCSI APP TLV */
5062         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5063
5064         for (i = 0; i < dcbcfg->numapps; i++) {
5065                 app = dcbcfg->app[i];
5066                 if (app.selector == I40E_APP_SEL_TCPIP &&
5067                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
5068                         tc = dcbcfg->etscfg.prioritytable[app.priority];
5069                         enabled_tc |= BIT(tc);
5070                         break;
5071                 }
5072         }
5073
5074         return enabled_tc;
5075 }
5076
5077 /**
5078  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5079  * @dcbcfg: the corresponding DCBx configuration structure
5080  *
5081  * Return the number of TCs from given DCBx configuration
5082  **/
5083 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5084 {
5085         int i, tc_unused = 0;
5086         u8 num_tc = 0;
5087         u8 ret = 0;
5088
5089         /* Scan the ETS Config Priority Table to find
5090          * traffic class enabled for a given priority
5091          * and create a bitmask of enabled TCs
5092          */
5093         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5094                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5095
5096         /* Now scan the bitmask to check for
5097          * contiguous TCs starting with TC0
5098          */
5099         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5100                 if (num_tc & BIT(i)) {
5101                         if (!tc_unused) {
5102                                 ret++;
5103                         } else {
5104                                 pr_err("Non-contiguous TC - Disabling DCB\n");
5105                                 return 1;
5106                         }
5107                 } else {
5108                         tc_unused = 1;
5109                 }
5110         }
5111
5112         /* There is always at least TC0 */
5113         if (!ret)
5114                 ret = 1;
5115
5116         return ret;
5117 }
5118
5119 /**
5120  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5121  * @dcbcfg: the corresponding DCBx configuration structure
5122  *
5123  * Query the current DCB configuration and return the number of
5124  * traffic classes enabled from the given DCBX config
5125  **/
5126 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5127 {
5128         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5129         u8 enabled_tc = 1;
5130         u8 i;
5131
5132         for (i = 0; i < num_tc; i++)
5133                 enabled_tc |= BIT(i);
5134
5135         return enabled_tc;
5136 }
5137
5138 /**
5139  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5140  * @pf: PF being queried
5141  *
5142  * Query the current MQPRIO configuration and return the number of
5143  * traffic classes enabled.
5144  **/
5145 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5146 {
5147         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5148         u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5149         u8 enabled_tc = 1, i;
5150
5151         for (i = 1; i < num_tc; i++)
5152                 enabled_tc |= BIT(i);
5153         return enabled_tc;
5154 }
5155
5156 /**
5157  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5158  * @pf: PF being queried
5159  *
5160  * Return number of traffic classes enabled for the given PF
5161  **/
5162 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5163 {
5164         struct i40e_hw *hw = &pf->hw;
5165         u8 i, enabled_tc = 1;
5166         u8 num_tc = 0;
5167         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5168
5169         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5170                 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5171
5172         /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5173         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5174                 return 1;
5175
5176         /* SFP mode will be enabled for all TCs on port */
5177         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5178                 return i40e_dcb_get_num_tc(dcbcfg);
5179
5180         /* MFP mode return count of enabled TCs for this PF */
5181         if (pf->hw.func_caps.iscsi)
5182                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
5183         else
5184                 return 1; /* Only TC0 */
5185
5186         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5187                 if (enabled_tc & BIT(i))
5188                         num_tc++;
5189         }
5190         return num_tc;
5191 }
5192
5193 /**
5194  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
5195  * @pf: PF being queried
5196  *
5197  * Return a bitmap for enabled traffic classes for this PF.
5198  **/
5199 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5200 {
5201         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5202                 return i40e_mqprio_get_enabled_tc(pf);
5203
5204         /* If neither MQPRIO nor DCB is enabled for this PF then just return
5205          * default TC
5206          */
5207         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5208                 return I40E_DEFAULT_TRAFFIC_CLASS;
5209
5210         /* SFP mode we want PF to be enabled for all TCs */
5211         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5212                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5213
5214         /* MFP enabled and iSCSI PF type */
5215         if (pf->hw.func_caps.iscsi)
5216                 return i40e_get_iscsi_tc_map(pf);
5217         else
5218                 return I40E_DEFAULT_TRAFFIC_CLASS;
5219 }
5220
5221 /**
5222  * i40e_vsi_get_bw_info - Query VSI BW Information
5223  * @vsi: the VSI being queried
5224  *
5225  * Returns 0 on success, negative value on failure
5226  **/
5227 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5228 {
5229         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5230         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5231         struct i40e_pf *pf = vsi->back;
5232         struct i40e_hw *hw = &pf->hw;
5233         i40e_status ret;
5234         u32 tc_bw_max;
5235         int i;
5236
5237         /* Get the VSI level BW configuration */
5238         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5239         if (ret) {
5240                 dev_info(&pf->pdev->dev,
5241                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
5242                          i40e_stat_str(&pf->hw, ret),
5243                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5244                 return -EINVAL;
5245         }
5246
5247         /* Get the VSI level BW configuration per TC */
5248         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5249                                                NULL);
5250         if (ret) {
5251                 dev_info(&pf->pdev->dev,
5252                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
5253                          i40e_stat_str(&pf->hw, ret),
5254                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5255                 return -EINVAL;
5256         }
5257
5258         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5259                 dev_info(&pf->pdev->dev,
5260                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5261                          bw_config.tc_valid_bits,
5262                          bw_ets_config.tc_valid_bits);
5263                 /* Still continuing */
5264         }
5265
5266         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5267         vsi->bw_max_quanta = bw_config.max_bw;
5268         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5269                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5270         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5271                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5272                 vsi->bw_ets_limit_credits[i] =
5273                                         le16_to_cpu(bw_ets_config.credits[i]);
5274                 /* 3 bits out of 4 for each TC */
5275                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5276         }
5277
5278         return 0;
5279 }
5280
5281 /**
5282  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5283  * @vsi: the VSI being configured
5284  * @enabled_tc: TC bitmap
5285  * @bw_share: BW shared credits per TC
5286  *
5287  * Returns 0 on success, negative value on failure
5288  **/
5289 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5290                                        u8 *bw_share)
5291 {
5292         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5293         struct i40e_pf *pf = vsi->back;
5294         i40e_status ret;
5295         int i;
5296
5297         /* There is no need to reset BW when mqprio mode is on.  */
5298         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5299                 return 0;
5300         if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5301                 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5302                 if (ret)
5303                         dev_info(&pf->pdev->dev,
5304                                  "Failed to reset tx rate for vsi->seid %u\n",
5305                                  vsi->seid);
5306                 return ret;
5307         }
5308         memset(&bw_data, 0, sizeof(bw_data));
5309         bw_data.tc_valid_bits = enabled_tc;
5310         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5311                 bw_data.tc_bw_credits[i] = bw_share[i];
5312
5313         ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5314         if (ret) {
5315                 dev_info(&pf->pdev->dev,
5316                          "AQ command Config VSI BW allocation per TC failed = %d\n",
5317                          pf->hw.aq.asq_last_status);
5318                 return -EINVAL;
5319         }
5320
5321         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5322                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5323
5324         return 0;
5325 }
5326
5327 /**
5328  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5329  * @vsi: the VSI being configured
5330  * @enabled_tc: TC map to be enabled
5331  *
5332  **/
5333 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5334 {
5335         struct net_device *netdev = vsi->netdev;
5336         struct i40e_pf *pf = vsi->back;
5337         struct i40e_hw *hw = &pf->hw;
5338         u8 netdev_tc = 0;
5339         int i;
5340         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5341
5342         if (!netdev)
5343                 return;
5344
5345         if (!enabled_tc) {
5346                 netdev_reset_tc(netdev);
5347                 return;
5348         }
5349
5350         /* Set up actual enabled TCs on the VSI */
5351         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5352                 return;
5353
5354         /* set per TC queues for the VSI */
5355         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5356                 /* Only set TC queues for enabled tcs
5357                  *
5358                  * e.g. For a VSI that has TC0 and TC3 enabled the
5359                  * enabled_tc bitmap would be 0x00001001; the driver
5360                  * will set the numtc for netdev as 2 that will be
5361                  * referenced by the netdev layer as TC 0 and 1.
5362                  */
5363                 if (vsi->tc_config.enabled_tc & BIT(i))
5364                         netdev_set_tc_queue(netdev,
5365                                         vsi->tc_config.tc_info[i].netdev_tc,
5366                                         vsi->tc_config.tc_info[i].qcount,
5367                                         vsi->tc_config.tc_info[i].qoffset);
5368         }
5369
5370         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5371                 return;
5372
5373         /* Assign UP2TC map for the VSI */
5374         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5375                 /* Get the actual TC# for the UP */
5376                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5377                 /* Get the mapped netdev TC# for the UP */
5378                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5379                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5380         }
5381 }
5382
5383 /**
5384  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5385  * @vsi: the VSI being configured
5386  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5387  **/
5388 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5389                                       struct i40e_vsi_context *ctxt)
5390 {
5391         /* copy just the sections touched not the entire info
5392          * since not all sections are valid as returned by
5393          * update vsi params
5394          */
5395         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5396         memcpy(&vsi->info.queue_mapping,
5397                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5398         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5399                sizeof(vsi->info.tc_mapping));
5400 }
5401
5402 /**
5403  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5404  * @vsi: VSI to be configured
5405  * @enabled_tc: TC bitmap
5406  *
5407  * This configures a particular VSI for TCs that are mapped to the
5408  * given TC bitmap. It uses default bandwidth share for TCs across
5409  * VSIs to configure TC for a particular VSI.
5410  *
5411  * NOTE:
5412  * It is expected that the VSI queues have been quisced before calling
5413  * this function.
5414  **/
5415 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5416 {
5417         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5418         struct i40e_pf *pf = vsi->back;
5419         struct i40e_hw *hw = &pf->hw;
5420         struct i40e_vsi_context ctxt;
5421         int ret = 0;
5422         int i;
5423
5424         /* Check if enabled_tc is same as existing or new TCs */
5425         if (vsi->tc_config.enabled_tc == enabled_tc &&
5426             vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5427                 return ret;
5428
5429         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5430         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5431                 if (enabled_tc & BIT(i))
5432                         bw_share[i] = 1;
5433         }
5434
5435         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5436         if (ret) {
5437                 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5438
5439                 dev_info(&pf->pdev->dev,
5440                          "Failed configuring TC map %d for VSI %d\n",
5441                          enabled_tc, vsi->seid);
5442                 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5443                                                   &bw_config, NULL);
5444                 if (ret) {
5445                         dev_info(&pf->pdev->dev,
5446                                  "Failed querying vsi bw info, err %s aq_err %s\n",
5447                                  i40e_stat_str(hw, ret),
5448                                  i40e_aq_str(hw, hw->aq.asq_last_status));
5449                         goto out;
5450                 }
5451                 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5452                         u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5453
5454                         if (!valid_tc)
5455                                 valid_tc = bw_config.tc_valid_bits;
5456                         /* Always enable TC0, no matter what */
5457                         valid_tc |= 1;
5458                         dev_info(&pf->pdev->dev,
5459                                  "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5460                                  enabled_tc, bw_config.tc_valid_bits, valid_tc);
5461                         enabled_tc = valid_tc;
5462                 }
5463
5464                 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5465                 if (ret) {
5466                         dev_err(&pf->pdev->dev,
5467                                 "Unable to  configure TC map %d for VSI %d\n",
5468                                 enabled_tc, vsi->seid);
5469                         goto out;
5470                 }
5471         }
5472
5473         /* Update Queue Pairs Mapping for currently enabled UPs */
5474         ctxt.seid = vsi->seid;
5475         ctxt.pf_num = vsi->back->hw.pf_id;
5476         ctxt.vf_num = 0;
5477         ctxt.uplink_seid = vsi->uplink_seid;
5478         ctxt.info = vsi->info;
5479         if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) {
5480                 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5481                 if (ret)
5482                         goto out;
5483         } else {
5484                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5485         }
5486
5487         /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5488          * queues changed.
5489          */
5490         if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5491                 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5492                                       vsi->num_queue_pairs);
5493                 ret = i40e_vsi_config_rss(vsi);
5494                 if (ret) {
5495                         dev_info(&vsi->back->pdev->dev,
5496                                  "Failed to reconfig rss for num_queues\n");
5497                         return ret;
5498                 }
5499                 vsi->reconfig_rss = false;
5500         }
5501         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5502                 ctxt.info.valid_sections |=
5503                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5504                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5505         }
5506
5507         /* Update the VSI after updating the VSI queue-mapping
5508          * information
5509          */
5510         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5511         if (ret) {
5512                 dev_info(&pf->pdev->dev,
5513                          "Update vsi tc config failed, err %s aq_err %s\n",
5514                          i40e_stat_str(hw, ret),
5515                          i40e_aq_str(hw, hw->aq.asq_last_status));
5516                 goto out;
5517         }
5518         /* update the local VSI info with updated queue map */
5519         i40e_vsi_update_queue_map(vsi, &ctxt);
5520         vsi->info.valid_sections = 0;
5521
5522         /* Update current VSI BW information */
5523         ret = i40e_vsi_get_bw_info(vsi);
5524         if (ret) {
5525                 dev_info(&pf->pdev->dev,
5526                          "Failed updating vsi bw info, err %s aq_err %s\n",
5527                          i40e_stat_str(hw, ret),
5528                          i40e_aq_str(hw, hw->aq.asq_last_status));
5529                 goto out;
5530         }
5531
5532         /* Update the netdev TC setup */
5533         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5534 out:
5535         return ret;
5536 }
5537
5538 /**
5539  * i40e_get_link_speed - Returns link speed for the interface
5540  * @vsi: VSI to be configured
5541  *
5542  **/
5543 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5544 {
5545         struct i40e_pf *pf = vsi->back;
5546
5547         switch (pf->hw.phy.link_info.link_speed) {
5548         case I40E_LINK_SPEED_40GB:
5549                 return 40000;
5550         case I40E_LINK_SPEED_25GB:
5551                 return 25000;
5552         case I40E_LINK_SPEED_20GB:
5553                 return 20000;
5554         case I40E_LINK_SPEED_10GB:
5555                 return 10000;
5556         case I40E_LINK_SPEED_1GB:
5557                 return 1000;
5558         default:
5559                 return -EINVAL;
5560         }
5561 }
5562
5563 /**
5564  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5565  * @vsi: VSI to be configured
5566  * @seid: seid of the channel/VSI
5567  * @max_tx_rate: max TX rate to be configured as BW limit
5568  *
5569  * Helper function to set BW limit for a given VSI
5570  **/
5571 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5572 {
5573         struct i40e_pf *pf = vsi->back;
5574         u64 credits = 0;
5575         int speed = 0;
5576         int ret = 0;
5577
5578         speed = i40e_get_link_speed(vsi);
5579         if (max_tx_rate > speed) {
5580                 dev_err(&pf->pdev->dev,
5581                         "Invalid max tx rate %llu specified for VSI seid %d.",
5582                         max_tx_rate, seid);
5583                 return -EINVAL;
5584         }
5585         if (max_tx_rate && max_tx_rate < 50) {
5586                 dev_warn(&pf->pdev->dev,
5587                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5588                 max_tx_rate = 50;
5589         }
5590
5591         /* Tx rate credits are in values of 50Mbps, 0 is disabled */
5592         credits = max_tx_rate;
5593         do_div(credits, I40E_BW_CREDIT_DIVISOR);
5594         ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5595                                           I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5596         if (ret)
5597                 dev_err(&pf->pdev->dev,
5598                         "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n",
5599                         max_tx_rate, seid, i40e_stat_str(&pf->hw, ret),
5600                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5601         return ret;
5602 }
5603
5604 /**
5605  * i40e_remove_queue_channels - Remove queue channels for the TCs
5606  * @vsi: VSI to be configured
5607  *
5608  * Remove queue channels for the TCs
5609  **/
5610 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
5611 {
5612         enum i40e_admin_queue_err last_aq_status;
5613         struct i40e_cloud_filter *cfilter;
5614         struct i40e_channel *ch, *ch_tmp;
5615         struct i40e_pf *pf = vsi->back;
5616         struct hlist_node *node;
5617         int ret, i;
5618
5619         /* Reset rss size that was stored when reconfiguring rss for
5620          * channel VSIs with non-power-of-2 queue count.
5621          */
5622         vsi->current_rss_size = 0;
5623
5624         /* perform cleanup for channels if they exist */
5625         if (list_empty(&vsi->ch_list))
5626                 return;
5627
5628         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5629                 struct i40e_vsi *p_vsi;
5630
5631                 list_del(&ch->list);
5632                 p_vsi = ch->parent_vsi;
5633                 if (!p_vsi || !ch->initialized) {
5634                         kfree(ch);
5635                         continue;
5636                 }
5637                 /* Reset queue contexts */
5638                 for (i = 0; i < ch->num_queue_pairs; i++) {
5639                         struct i40e_ring *tx_ring, *rx_ring;
5640                         u16 pf_q;
5641
5642                         pf_q = ch->base_queue + i;
5643                         tx_ring = vsi->tx_rings[pf_q];
5644                         tx_ring->ch = NULL;
5645
5646                         rx_ring = vsi->rx_rings[pf_q];
5647                         rx_ring->ch = NULL;
5648                 }
5649
5650                 /* Reset BW configured for this VSI via mqprio */
5651                 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
5652                 if (ret)
5653                         dev_info(&vsi->back->pdev->dev,
5654                                  "Failed to reset tx rate for ch->seid %u\n",
5655                                  ch->seid);
5656
5657                 /* delete cloud filters associated with this channel */
5658                 hlist_for_each_entry_safe(cfilter, node,
5659                                           &pf->cloud_filter_list, cloud_node) {
5660                         if (cfilter->seid != ch->seid)
5661                                 continue;
5662
5663                         hash_del(&cfilter->cloud_node);
5664                         if (cfilter->dst_port)
5665                                 ret = i40e_add_del_cloud_filter_big_buf(vsi,
5666                                                                         cfilter,
5667                                                                         false);
5668                         else
5669                                 ret = i40e_add_del_cloud_filter(vsi, cfilter,
5670                                                                 false);
5671                         last_aq_status = pf->hw.aq.asq_last_status;
5672                         if (ret)
5673                                 dev_info(&pf->pdev->dev,
5674                                          "Failed to delete cloud filter, err %s aq_err %s\n",
5675                                          i40e_stat_str(&pf->hw, ret),
5676                                          i40e_aq_str(&pf->hw, last_aq_status));
5677                         kfree(cfilter);
5678                 }
5679
5680                 /* delete VSI from FW */
5681                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
5682                                              NULL);
5683                 if (ret)
5684                         dev_err(&vsi->back->pdev->dev,
5685                                 "unable to remove channel (%d) for parent VSI(%d)\n",
5686                                 ch->seid, p_vsi->seid);
5687                 kfree(ch);
5688         }
5689         INIT_LIST_HEAD(&vsi->ch_list);
5690 }
5691
5692 /**
5693  * i40e_is_any_channel - channel exist or not
5694  * @vsi: ptr to VSI to which channels are associated with
5695  *
5696  * Returns true or false if channel(s) exist for associated VSI or not
5697  **/
5698 static bool i40e_is_any_channel(struct i40e_vsi *vsi)
5699 {
5700         struct i40e_channel *ch, *ch_tmp;
5701
5702         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5703                 if (ch->initialized)
5704                         return true;
5705         }
5706
5707         return false;
5708 }
5709
5710 /**
5711  * i40e_get_max_queues_for_channel
5712  * @vsi: ptr to VSI to which channels are associated with
5713  *
5714  * Helper function which returns max value among the queue counts set on the
5715  * channels/TCs created.
5716  **/
5717 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
5718 {
5719         struct i40e_channel *ch, *ch_tmp;
5720         int max = 0;
5721
5722         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5723                 if (!ch->initialized)
5724                         continue;
5725                 if (ch->num_queue_pairs > max)
5726                         max = ch->num_queue_pairs;
5727         }
5728
5729         return max;
5730 }
5731
5732 /**
5733  * i40e_validate_num_queues - validate num_queues w.r.t channel
5734  * @pf: ptr to PF device
5735  * @num_queues: number of queues
5736  * @vsi: the parent VSI
5737  * @reconfig_rss: indicates should the RSS be reconfigured or not
5738  *
5739  * This function validates number of queues in the context of new channel
5740  * which is being established and determines if RSS should be reconfigured
5741  * or not for parent VSI.
5742  **/
5743 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
5744                                     struct i40e_vsi *vsi, bool *reconfig_rss)
5745 {
5746         int max_ch_queues;
5747
5748         if (!reconfig_rss)
5749                 return -EINVAL;
5750
5751         *reconfig_rss = false;
5752         if (vsi->current_rss_size) {
5753                 if (num_queues > vsi->current_rss_size) {
5754                         dev_dbg(&pf->pdev->dev,
5755                                 "Error: num_queues (%d) > vsi's current_size(%d)\n",
5756                                 num_queues, vsi->current_rss_size);
5757                         return -EINVAL;
5758                 } else if ((num_queues < vsi->current_rss_size) &&
5759                            (!is_power_of_2(num_queues))) {
5760                         dev_dbg(&pf->pdev->dev,
5761                                 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
5762                                 num_queues, vsi->current_rss_size);
5763                         return -EINVAL;
5764                 }
5765         }
5766
5767         if (!is_power_of_2(num_queues)) {
5768                 /* Find the max num_queues configured for channel if channel
5769                  * exist.
5770                  * if channel exist, then enforce 'num_queues' to be more than
5771                  * max ever queues configured for channel.
5772                  */
5773                 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
5774                 if (num_queues < max_ch_queues) {
5775                         dev_dbg(&pf->pdev->dev,
5776                                 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
5777                                 num_queues, max_ch_queues);
5778                         return -EINVAL;
5779                 }
5780                 *reconfig_rss = true;
5781         }
5782
5783         return 0;
5784 }
5785
5786 /**
5787  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
5788  * @vsi: the VSI being setup
5789  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
5790  *
5791  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
5792  **/
5793 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
5794 {
5795         struct i40e_pf *pf = vsi->back;
5796         u8 seed[I40E_HKEY_ARRAY_SIZE];
5797         struct i40e_hw *hw = &pf->hw;
5798         int local_rss_size;
5799         u8 *lut;
5800         int ret;
5801
5802         if (!vsi->rss_size)
5803                 return -EINVAL;
5804
5805         if (rss_size > vsi->rss_size)
5806                 return -EINVAL;
5807
5808         local_rss_size = min_t(int, vsi->rss_size, rss_size);
5809         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
5810         if (!lut)
5811                 return -ENOMEM;
5812
5813         /* Ignoring user configured lut if there is one */
5814         i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
5815
5816         /* Use user configured hash key if there is one, otherwise
5817          * use default.
5818          */
5819         if (vsi->rss_hkey_user)
5820                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
5821         else
5822                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
5823
5824         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
5825         if (ret) {
5826                 dev_info(&pf->pdev->dev,
5827                          "Cannot set RSS lut, err %s aq_err %s\n",
5828                          i40e_stat_str(hw, ret),
5829                          i40e_aq_str(hw, hw->aq.asq_last_status));
5830                 kfree(lut);
5831                 return ret;
5832         }
5833         kfree(lut);
5834
5835         /* Do the update w.r.t. storing rss_size */
5836         if (!vsi->orig_rss_size)
5837                 vsi->orig_rss_size = vsi->rss_size;
5838         vsi->current_rss_size = local_rss_size;
5839
5840         return ret;
5841 }
5842
5843 /**
5844  * i40e_channel_setup_queue_map - Setup a channel queue map
5845  * @pf: ptr to PF device
5846  * @ctxt: VSI context structure
5847  * @ch: ptr to channel structure
5848  *
5849  * Setup queue map for a specific channel
5850  **/
5851 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
5852                                          struct i40e_vsi_context *ctxt,
5853                                          struct i40e_channel *ch)
5854 {
5855         u16 qcount, qmap, sections = 0;
5856         u8 offset = 0;
5857         int pow;
5858
5859         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
5860         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
5861
5862         qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
5863         ch->num_queue_pairs = qcount;
5864
5865         /* find the next higher power-of-2 of num queue pairs */
5866         pow = ilog2(qcount);
5867         if (!is_power_of_2(qcount))
5868                 pow++;
5869
5870         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5871                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
5872
5873         /* Setup queue TC[0].qmap for given VSI context */
5874         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
5875
5876         ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
5877         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
5878         ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
5879         ctxt->info.valid_sections |= cpu_to_le16(sections);
5880 }
5881
5882 /**
5883  * i40e_add_channel - add a channel by adding VSI
5884  * @pf: ptr to PF device
5885  * @uplink_seid: underlying HW switching element (VEB) ID
5886  * @ch: ptr to channel structure
5887  *
5888  * Add a channel (VSI) using add_vsi and queue_map
5889  **/
5890 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
5891                             struct i40e_channel *ch)
5892 {
5893         struct i40e_hw *hw = &pf->hw;
5894         struct i40e_vsi_context ctxt;
5895         u8 enabled_tc = 0x1; /* TC0 enabled */
5896         int ret;
5897
5898         if (ch->type != I40E_VSI_VMDQ2) {
5899                 dev_info(&pf->pdev->dev,
5900                          "add new vsi failed, ch->type %d\n", ch->type);
5901                 return -EINVAL;
5902         }
5903
5904         memset(&ctxt, 0, sizeof(ctxt));
5905         ctxt.pf_num = hw->pf_id;
5906         ctxt.vf_num = 0;
5907         ctxt.uplink_seid = uplink_seid;
5908         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5909         if (ch->type == I40E_VSI_VMDQ2)
5910                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
5911
5912         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
5913                 ctxt.info.valid_sections |=
5914                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5915                 ctxt.info.switch_id =
5916                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
5917         }
5918
5919         /* Set queue map for a given VSI context */
5920         i40e_channel_setup_queue_map(pf, &ctxt, ch);
5921
5922         /* Now time to create VSI */
5923         ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
5924         if (ret) {
5925                 dev_info(&pf->pdev->dev,
5926                          "add new vsi failed, err %s aq_err %s\n",
5927                          i40e_stat_str(&pf->hw, ret),
5928                          i40e_aq_str(&pf->hw,
5929                                      pf->hw.aq.asq_last_status));
5930                 return -ENOENT;
5931         }
5932
5933         /* Success, update channel, set enabled_tc only if the channel
5934          * is not a macvlan
5935          */
5936         ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
5937         ch->seid = ctxt.seid;
5938         ch->vsi_number = ctxt.vsi_number;
5939         ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
5940
5941         /* copy just the sections touched not the entire info
5942          * since not all sections are valid as returned by
5943          * update vsi params
5944          */
5945         ch->info.mapping_flags = ctxt.info.mapping_flags;
5946         memcpy(&ch->info.queue_mapping,
5947                &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
5948         memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
5949                sizeof(ctxt.info.tc_mapping));
5950
5951         return 0;
5952 }
5953
5954 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
5955                                   u8 *bw_share)
5956 {
5957         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5958         i40e_status ret;
5959         int i;
5960
5961         memset(&bw_data, 0, sizeof(bw_data));
5962         bw_data.tc_valid_bits = ch->enabled_tc;
5963         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5964                 bw_data.tc_bw_credits[i] = bw_share[i];
5965
5966         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
5967                                        &bw_data, NULL);
5968         if (ret) {
5969                 dev_info(&vsi->back->pdev->dev,
5970                          "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
5971                          vsi->back->hw.aq.asq_last_status, ch->seid);
5972                 return -EINVAL;
5973         }
5974
5975         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5976                 ch->info.qs_handle[i] = bw_data.qs_handles[i];
5977
5978         return 0;
5979 }
5980
5981 /**
5982  * i40e_channel_config_tx_ring - config TX ring associated with new channel
5983  * @pf: ptr to PF device
5984  * @vsi: the VSI being setup
5985  * @ch: ptr to channel structure
5986  *
5987  * Configure TX rings associated with channel (VSI) since queues are being
5988  * from parent VSI.
5989  **/
5990 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
5991                                        struct i40e_vsi *vsi,
5992                                        struct i40e_channel *ch)
5993 {
5994         i40e_status ret;
5995         int i;
5996         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5997
5998         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5999         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6000                 if (ch->enabled_tc & BIT(i))
6001                         bw_share[i] = 1;
6002         }
6003
6004         /* configure BW for new VSI */
6005         ret = i40e_channel_config_bw(vsi, ch, bw_share);
6006         if (ret) {
6007                 dev_info(&vsi->back->pdev->dev,
6008                          "Failed configuring TC map %d for channel (seid %u)\n",
6009                          ch->enabled_tc, ch->seid);
6010                 return ret;
6011         }
6012
6013         for (i = 0; i < ch->num_queue_pairs; i++) {
6014                 struct i40e_ring *tx_ring, *rx_ring;
6015                 u16 pf_q;
6016
6017                 pf_q = ch->base_queue + i;
6018
6019                 /* Get to TX ring ptr of main VSI, for re-setup TX queue
6020                  * context
6021                  */
6022                 tx_ring = vsi->tx_rings[pf_q];
6023                 tx_ring->ch = ch;
6024
6025                 /* Get the RX ring ptr */
6026                 rx_ring = vsi->rx_rings[pf_q];
6027                 rx_ring->ch = ch;
6028         }
6029
6030         return 0;
6031 }
6032
6033 /**
6034  * i40e_setup_hw_channel - setup new channel
6035  * @pf: ptr to PF device
6036  * @vsi: the VSI being setup
6037  * @ch: ptr to channel structure
6038  * @uplink_seid: underlying HW switching element (VEB) ID
6039  * @type: type of channel to be created (VMDq2/VF)
6040  *
6041  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6042  * and configures TX rings accordingly
6043  **/
6044 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6045                                         struct i40e_vsi *vsi,
6046                                         struct i40e_channel *ch,
6047                                         u16 uplink_seid, u8 type)
6048 {
6049         int ret;
6050
6051         ch->initialized = false;
6052         ch->base_queue = vsi->next_base_queue;
6053         ch->type = type;
6054
6055         /* Proceed with creation of channel (VMDq2) VSI */
6056         ret = i40e_add_channel(pf, uplink_seid, ch);
6057         if (ret) {
6058                 dev_info(&pf->pdev->dev,
6059                          "failed to add_channel using uplink_seid %u\n",
6060                          uplink_seid);
6061                 return ret;
6062         }
6063
6064         /* Mark the successful creation of channel */
6065         ch->initialized = true;
6066
6067         /* Reconfigure TX queues using QTX_CTL register */
6068         ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6069         if (ret) {
6070                 dev_info(&pf->pdev->dev,
6071                          "failed to configure TX rings for channel %u\n",
6072                          ch->seid);
6073                 return ret;
6074         }
6075
6076         /* update 'next_base_queue' */
6077         vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6078         dev_dbg(&pf->pdev->dev,
6079                 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6080                 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6081                 ch->num_queue_pairs,
6082                 vsi->next_base_queue);
6083         return ret;
6084 }
6085
6086 /**
6087  * i40e_setup_channel - setup new channel using uplink element
6088  * @pf: ptr to PF device
6089  * @vsi: pointer to the VSI to set up the channel within
6090  * @ch: ptr to channel structure
6091  *
6092  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6093  * and uplink switching element (uplink_seid)
6094  **/
6095 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6096                                struct i40e_channel *ch)
6097 {
6098         u8 vsi_type;
6099         u16 seid;
6100         int ret;
6101
6102         if (vsi->type == I40E_VSI_MAIN) {
6103                 vsi_type = I40E_VSI_VMDQ2;
6104         } else {
6105                 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6106                         vsi->type);
6107                 return false;
6108         }
6109
6110         /* underlying switching element */
6111         seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6112
6113         /* create channel (VSI), configure TX rings */
6114         ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6115         if (ret) {
6116                 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6117                 return false;
6118         }
6119
6120         return ch->initialized ? true : false;
6121 }
6122
6123 /**
6124  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6125  * @vsi: ptr to VSI which has PF backing
6126  *
6127  * Sets up switch mode correctly if it needs to be changed and perform
6128  * what are allowed modes.
6129  **/
6130 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6131 {
6132         u8 mode;
6133         struct i40e_pf *pf = vsi->back;
6134         struct i40e_hw *hw = &pf->hw;
6135         int ret;
6136
6137         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6138         if (ret)
6139                 return -EINVAL;
6140
6141         if (hw->dev_caps.switch_mode) {
6142                 /* if switch mode is set, support mode2 (non-tunneled for
6143                  * cloud filter) for now
6144                  */
6145                 u32 switch_mode = hw->dev_caps.switch_mode &
6146                                   I40E_SWITCH_MODE_MASK;
6147                 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6148                         if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6149                                 return 0;
6150                         dev_err(&pf->pdev->dev,
6151                                 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6152                                 hw->dev_caps.switch_mode);
6153                         return -EINVAL;
6154                 }
6155         }
6156
6157         /* Set Bit 7 to be valid */
6158         mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6159
6160         /* Set L4type for TCP support */
6161         mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6162
6163         /* Set cloud filter mode */
6164         mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6165
6166         /* Prep mode field for set_switch_config */
6167         ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6168                                         pf->last_sw_conf_valid_flags,
6169                                         mode, NULL);
6170         if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6171                 dev_err(&pf->pdev->dev,
6172                         "couldn't set switch config bits, err %s aq_err %s\n",
6173                         i40e_stat_str(hw, ret),
6174                         i40e_aq_str(hw,
6175                                     hw->aq.asq_last_status));
6176
6177         return ret;
6178 }
6179
6180 /**
6181  * i40e_create_queue_channel - function to create channel
6182  * @vsi: VSI to be configured
6183  * @ch: ptr to channel (it contains channel specific params)
6184  *
6185  * This function creates channel (VSI) using num_queues specified by user,
6186  * reconfigs RSS if needed.
6187  **/
6188 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6189                               struct i40e_channel *ch)
6190 {
6191         struct i40e_pf *pf = vsi->back;
6192         bool reconfig_rss;
6193         int err;
6194
6195         if (!ch)
6196                 return -EINVAL;
6197
6198         if (!ch->num_queue_pairs) {
6199                 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6200                         ch->num_queue_pairs);
6201                 return -EINVAL;
6202         }
6203
6204         /* validate user requested num_queues for channel */
6205         err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6206                                        &reconfig_rss);
6207         if (err) {
6208                 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6209                          ch->num_queue_pairs);
6210                 return -EINVAL;
6211         }
6212
6213         /* By default we are in VEPA mode, if this is the first VF/VMDq
6214          * VSI to be added switch to VEB mode.
6215          */
6216         if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) ||
6217             (!i40e_is_any_channel(vsi))) {
6218                 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) {
6219                         dev_dbg(&pf->pdev->dev,
6220                                 "Failed to create channel. Override queues (%u) not power of 2\n",
6221                                 vsi->tc_config.tc_info[0].qcount);
6222                         return -EINVAL;
6223                 }
6224
6225                 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
6226                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
6227
6228                         if (vsi->type == I40E_VSI_MAIN) {
6229                                 if (pf->flags & I40E_FLAG_TC_MQPRIO)
6230                                         i40e_do_reset(pf, I40E_PF_RESET_FLAG,
6231                                                       true);
6232                                 else
6233                                         i40e_do_reset_safe(pf,
6234                                                            I40E_PF_RESET_FLAG);
6235                         }
6236                 }
6237                 /* now onwards for main VSI, number of queues will be value
6238                  * of TC0's queue count
6239                  */
6240         }
6241
6242         /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6243          * it should be more than num_queues
6244          */
6245         if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6246                 dev_dbg(&pf->pdev->dev,
6247                         "Error: cnt_q_avail (%u) less than num_queues %d\n",
6248                         vsi->cnt_q_avail, ch->num_queue_pairs);
6249                 return -EINVAL;
6250         }
6251
6252         /* reconfig_rss only if vsi type is MAIN_VSI */
6253         if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6254                 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6255                 if (err) {
6256                         dev_info(&pf->pdev->dev,
6257                                  "Error: unable to reconfig rss for num_queues (%u)\n",
6258                                  ch->num_queue_pairs);
6259                         return -EINVAL;
6260                 }
6261         }
6262
6263         if (!i40e_setup_channel(pf, vsi, ch)) {
6264                 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6265                 return -EINVAL;
6266         }
6267
6268         dev_info(&pf->pdev->dev,
6269                  "Setup channel (id:%u) utilizing num_queues %d\n",
6270                  ch->seid, ch->num_queue_pairs);
6271
6272         /* configure VSI for BW limit */
6273         if (ch->max_tx_rate) {
6274                 u64 credits = ch->max_tx_rate;
6275
6276                 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6277                         return -EINVAL;
6278
6279                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6280                 dev_dbg(&pf->pdev->dev,
6281                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6282                         ch->max_tx_rate,
6283                         credits,
6284                         ch->seid);
6285         }
6286
6287         /* in case of VF, this will be main SRIOV VSI */
6288         ch->parent_vsi = vsi;
6289
6290         /* and update main_vsi's count for queue_available to use */
6291         vsi->cnt_q_avail -= ch->num_queue_pairs;
6292
6293         return 0;
6294 }
6295
6296 /**
6297  * i40e_configure_queue_channels - Add queue channel for the given TCs
6298  * @vsi: VSI to be configured
6299  *
6300  * Configures queue channel mapping to the given TCs
6301  **/
6302 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6303 {
6304         struct i40e_channel *ch;
6305         u64 max_rate = 0;
6306         int ret = 0, i;
6307
6308         /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6309         vsi->tc_seid_map[0] = vsi->seid;
6310         for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6311                 if (vsi->tc_config.enabled_tc & BIT(i)) {
6312                         ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6313                         if (!ch) {
6314                                 ret = -ENOMEM;
6315                                 goto err_free;
6316                         }
6317
6318                         INIT_LIST_HEAD(&ch->list);
6319                         ch->num_queue_pairs =
6320                                 vsi->tc_config.tc_info[i].qcount;
6321                         ch->base_queue =
6322                                 vsi->tc_config.tc_info[i].qoffset;
6323
6324                         /* Bandwidth limit through tc interface is in bytes/s,
6325                          * change to Mbit/s
6326                          */
6327                         max_rate = vsi->mqprio_qopt.max_rate[i];
6328                         do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6329                         ch->max_tx_rate = max_rate;
6330
6331                         list_add_tail(&ch->list, &vsi->ch_list);
6332
6333                         ret = i40e_create_queue_channel(vsi, ch);
6334                         if (ret) {
6335                                 dev_err(&vsi->back->pdev->dev,
6336                                         "Failed creating queue channel with TC%d: queues %d\n",
6337                                         i, ch->num_queue_pairs);
6338                                 goto err_free;
6339                         }
6340                         vsi->tc_seid_map[i] = ch->seid;
6341                 }
6342         }
6343         return ret;
6344
6345 err_free:
6346         i40e_remove_queue_channels(vsi);
6347         return ret;
6348 }
6349
6350 /**
6351  * i40e_veb_config_tc - Configure TCs for given VEB
6352  * @veb: given VEB
6353  * @enabled_tc: TC bitmap
6354  *
6355  * Configures given TC bitmap for VEB (switching) element
6356  **/
6357 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6358 {
6359         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6360         struct i40e_pf *pf = veb->pf;
6361         int ret = 0;
6362         int i;
6363
6364         /* No TCs or already enabled TCs just return */
6365         if (!enabled_tc || veb->enabled_tc == enabled_tc)
6366                 return ret;
6367
6368         bw_data.tc_valid_bits = enabled_tc;
6369         /* bw_data.absolute_credits is not set (relative) */
6370
6371         /* Enable ETS TCs with equal BW Share for now */
6372         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6373                 if (enabled_tc & BIT(i))
6374                         bw_data.tc_bw_share_credits[i] = 1;
6375         }
6376
6377         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6378                                                    &bw_data, NULL);
6379         if (ret) {
6380                 dev_info(&pf->pdev->dev,
6381                          "VEB bw config failed, err %s aq_err %s\n",
6382                          i40e_stat_str(&pf->hw, ret),
6383                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6384                 goto out;
6385         }
6386
6387         /* Update the BW information */
6388         ret = i40e_veb_get_bw_info(veb);
6389         if (ret) {
6390                 dev_info(&pf->pdev->dev,
6391                          "Failed getting veb bw config, err %s aq_err %s\n",
6392                          i40e_stat_str(&pf->hw, ret),
6393                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6394         }
6395
6396 out:
6397         return ret;
6398 }
6399
6400 #ifdef CONFIG_I40E_DCB
6401 /**
6402  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6403  * @pf: PF struct
6404  *
6405  * Reconfigure VEB/VSIs on a given PF; it is assumed that
6406  * the caller would've quiesce all the VSIs before calling
6407  * this function
6408  **/
6409 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6410 {
6411         u8 tc_map = 0;
6412         int ret;
6413         u8 v;
6414
6415         /* Enable the TCs available on PF to all VEBs */
6416         tc_map = i40e_pf_get_tc_map(pf);
6417         if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)
6418                 return;
6419
6420         for (v = 0; v < I40E_MAX_VEB; v++) {
6421                 if (!pf->veb[v])
6422                         continue;
6423                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
6424                 if (ret) {
6425                         dev_info(&pf->pdev->dev,
6426                                  "Failed configuring TC for VEB seid=%d\n",
6427                                  pf->veb[v]->seid);
6428                         /* Will try to configure as many components */
6429                 }
6430         }
6431
6432         /* Update each VSI */
6433         for (v = 0; v < pf->num_alloc_vsi; v++) {
6434                 if (!pf->vsi[v])
6435                         continue;
6436
6437                 /* - Enable all TCs for the LAN VSI
6438                  * - For all others keep them at TC0 for now
6439                  */
6440                 if (v == pf->lan_vsi)
6441                         tc_map = i40e_pf_get_tc_map(pf);
6442                 else
6443                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6444
6445                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
6446                 if (ret) {
6447                         dev_info(&pf->pdev->dev,
6448                                  "Failed configuring TC for VSI seid=%d\n",
6449                                  pf->vsi[v]->seid);
6450                         /* Will try to configure as many components */
6451                 } else {
6452                         /* Re-configure VSI vectors based on updated TC map */
6453                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
6454                         if (pf->vsi[v]->netdev)
6455                                 i40e_dcbnl_set_all(pf->vsi[v]);
6456                 }
6457         }
6458 }
6459
6460 /**
6461  * i40e_resume_port_tx - Resume port Tx
6462  * @pf: PF struct
6463  *
6464  * Resume a port's Tx and issue a PF reset in case of failure to
6465  * resume.
6466  **/
6467 static int i40e_resume_port_tx(struct i40e_pf *pf)
6468 {
6469         struct i40e_hw *hw = &pf->hw;
6470         int ret;
6471
6472         ret = i40e_aq_resume_port_tx(hw, NULL);
6473         if (ret) {
6474                 dev_info(&pf->pdev->dev,
6475                          "Resume Port Tx failed, err %s aq_err %s\n",
6476                           i40e_stat_str(&pf->hw, ret),
6477                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6478                 /* Schedule PF reset to recover */
6479                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6480                 i40e_service_event_schedule(pf);
6481         }
6482
6483         return ret;
6484 }
6485
6486 /**
6487  * i40e_suspend_port_tx - Suspend port Tx
6488  * @pf: PF struct
6489  *
6490  * Suspend a port's Tx and issue a PF reset in case of failure.
6491  **/
6492 static int i40e_suspend_port_tx(struct i40e_pf *pf)
6493 {
6494         struct i40e_hw *hw = &pf->hw;
6495         int ret;
6496
6497         ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);
6498         if (ret) {
6499                 dev_info(&pf->pdev->dev,
6500                          "Suspend Port Tx failed, err %s aq_err %s\n",
6501                          i40e_stat_str(&pf->hw, ret),
6502                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6503                 /* Schedule PF reset to recover */
6504                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6505                 i40e_service_event_schedule(pf);
6506         }
6507
6508         return ret;
6509 }
6510
6511 /**
6512  * i40e_hw_set_dcb_config - Program new DCBX settings into HW
6513  * @pf: PF being configured
6514  * @new_cfg: New DCBX configuration
6515  *
6516  * Program DCB settings into HW and reconfigure VEB/VSIs on
6517  * given PF. Uses "Set LLDP MIB" AQC to program the hardware.
6518  **/
6519 static int i40e_hw_set_dcb_config(struct i40e_pf *pf,
6520                                   struct i40e_dcbx_config *new_cfg)
6521 {
6522         struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;
6523         int ret;
6524
6525         /* Check if need reconfiguration */
6526         if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) {
6527                 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");
6528                 return 0;
6529         }
6530
6531         /* Config change disable all VSIs */
6532         i40e_pf_quiesce_all_vsi(pf);
6533
6534         /* Copy the new config to the current config */
6535         *old_cfg = *new_cfg;
6536         old_cfg->etsrec = old_cfg->etscfg;
6537         ret = i40e_set_dcb_config(&pf->hw);
6538         if (ret) {
6539                 dev_info(&pf->pdev->dev,
6540                          "Set DCB Config failed, err %s aq_err %s\n",
6541                          i40e_stat_str(&pf->hw, ret),
6542                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6543                 goto out;
6544         }
6545
6546         /* Changes in configuration update VEB/VSI */
6547         i40e_dcb_reconfigure(pf);
6548 out:
6549         /* In case of reset do not try to resume anything */
6550         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
6551                 /* Re-start the VSIs if disabled */
6552                 ret = i40e_resume_port_tx(pf);
6553                 /* In case of error no point in resuming VSIs */
6554                 if (ret)
6555                         goto err;
6556                 i40e_pf_unquiesce_all_vsi(pf);
6557         }
6558 err:
6559         return ret;
6560 }
6561
6562 /**
6563  * i40e_hw_dcb_config - Program new DCBX settings into HW
6564  * @pf: PF being configured
6565  * @new_cfg: New DCBX configuration
6566  *
6567  * Program DCB settings into HW and reconfigure VEB/VSIs on
6568  * given PF
6569  **/
6570 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
6571 {
6572         struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6573         u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};
6574         u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];
6575         struct i40e_dcbx_config *old_cfg;
6576         u8 mode[I40E_MAX_TRAFFIC_CLASS];
6577         struct i40e_rx_pb_config pb_cfg;
6578         struct i40e_hw *hw = &pf->hw;
6579         u8 num_ports = hw->num_ports;
6580         bool need_reconfig;
6581         int ret = -EINVAL;
6582         u8 lltc_map = 0;
6583         u8 tc_map = 0;
6584         u8 new_numtc;
6585         u8 i;
6586
6587         dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");
6588         /* Un-pack information to Program ETS HW via shared API
6589          * numtc, tcmap
6590          * LLTC map
6591          * ETS/NON-ETS arbiter mode
6592          * max exponent (credit refills)
6593          * Total number of ports
6594          * PFC priority bit-map
6595          * Priority Table
6596          * BW % per TC
6597          * Arbiter mode between UPs sharing same TC
6598          * TSA table (ETS or non-ETS)
6599          * EEE enabled or not
6600          * MFS TC table
6601          */
6602
6603         new_numtc = i40e_dcb_get_num_tc(new_cfg);
6604
6605         memset(&ets_data, 0, sizeof(ets_data));
6606         for (i = 0; i < new_numtc; i++) {
6607                 tc_map |= BIT(i);
6608                 switch (new_cfg->etscfg.tsatable[i]) {
6609                 case I40E_IEEE_TSA_ETS:
6610                         prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;
6611                         ets_data.tc_bw_share_credits[i] =
6612                                         new_cfg->etscfg.tcbwtable[i];
6613                         break;
6614                 case I40E_IEEE_TSA_STRICT:
6615                         prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;
6616                         lltc_map |= BIT(i);
6617                         ets_data.tc_bw_share_credits[i] =
6618                                         I40E_DCB_STRICT_PRIO_CREDITS;
6619                         break;
6620                 default:
6621                         /* Invalid TSA type */
6622                         need_reconfig = false;
6623                         goto out;
6624                 }
6625         }
6626
6627         old_cfg = &hw->local_dcbx_config;
6628         /* Check if need reconfiguration */
6629         need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);
6630
6631         /* If needed, enable/disable frame tagging, disable all VSIs
6632          * and suspend port tx
6633          */
6634         if (need_reconfig) {
6635                 /* Enable DCB tagging only when more than one TC */
6636                 if (new_numtc > 1)
6637                         pf->flags |= I40E_FLAG_DCB_ENABLED;
6638                 else
6639                         pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6640
6641                 set_bit(__I40E_PORT_SUSPENDED, pf->state);
6642                 /* Reconfiguration needed quiesce all VSIs */
6643                 i40e_pf_quiesce_all_vsi(pf);
6644                 ret = i40e_suspend_port_tx(pf);
6645                 if (ret)
6646                         goto err;
6647         }
6648
6649         /* Configure Port ETS Tx Scheduler */
6650         ets_data.tc_valid_bits = tc_map;
6651         ets_data.tc_strict_priority_flags = lltc_map;
6652         ret = i40e_aq_config_switch_comp_ets
6653                 (hw, pf->mac_seid, &ets_data,
6654                  i40e_aqc_opc_modify_switching_comp_ets, NULL);
6655         if (ret) {
6656                 dev_info(&pf->pdev->dev,
6657                          "Modify Port ETS failed, err %s aq_err %s\n",
6658                          i40e_stat_str(&pf->hw, ret),
6659                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6660                 goto out;
6661         }
6662
6663         /* Configure Rx ETS HW */
6664         memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));
6665         i40e_dcb_hw_set_num_tc(hw, new_numtc);
6666         i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN,
6667                                    I40E_DCB_ARB_MODE_STRICT_PRIORITY,
6668                                    I40E_DCB_DEFAULT_MAX_EXPONENT,
6669                                    lltc_map);
6670         i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports);
6671         i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,
6672                                      prio_type);
6673         i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,
6674                                new_cfg->etscfg.prioritytable);
6675         i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);
6676
6677         /* Configure Rx Packet Buffers in HW */
6678         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6679                 mfs_tc[i] = pf->vsi[pf->lan_vsi]->netdev->mtu;
6680                 mfs_tc[i] += I40E_PACKET_HDR_PAD;
6681         }
6682
6683         i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,
6684                                          false, new_cfg->pfc.pfcenable,
6685                                          mfs_tc, &pb_cfg);
6686         i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);
6687
6688         /* Update the local Rx Packet buffer config */
6689         pf->pb_cfg = pb_cfg;
6690
6691         /* Inform the FW about changes to DCB configuration */
6692         ret = i40e_aq_dcb_updated(&pf->hw, NULL);
6693         if (ret) {
6694                 dev_info(&pf->pdev->dev,
6695                          "DCB Updated failed, err %s aq_err %s\n",
6696                          i40e_stat_str(&pf->hw, ret),
6697                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6698                 goto out;
6699         }
6700
6701         /* Update the port DCBx configuration */
6702         *old_cfg = *new_cfg;
6703
6704         /* Changes in configuration update VEB/VSI */
6705         i40e_dcb_reconfigure(pf);
6706 out:
6707         /* Re-start the VSIs if disabled */
6708         if (need_reconfig) {
6709                 ret = i40e_resume_port_tx(pf);
6710
6711                 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
6712                 /* In case of error no point in resuming VSIs */
6713                 if (ret)
6714                         goto err;
6715
6716                 /* Wait for the PF's queues to be disabled */
6717                 ret = i40e_pf_wait_queues_disabled(pf);
6718                 if (ret) {
6719                         /* Schedule PF reset to recover */
6720                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6721                         i40e_service_event_schedule(pf);
6722                         goto err;
6723                 } else {
6724                         i40e_pf_unquiesce_all_vsi(pf);
6725                         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
6726                         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
6727                 }
6728         /* registers are set, lets apply */
6729         if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB)
6730                 ret = i40e_hw_set_dcb_config(pf, new_cfg);
6731         }
6732
6733 err:
6734         return ret;
6735 }
6736
6737 /**
6738  * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
6739  * @pf: PF being queried
6740  *
6741  * Set default DCB configuration in case DCB is to be done in SW.
6742  **/
6743 int i40e_dcb_sw_default_config(struct i40e_pf *pf)
6744 {
6745         struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;
6746         struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6747         struct i40e_hw *hw = &pf->hw;
6748         int err;
6749
6750         if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB) {
6751                 /* Update the local cached instance with TC0 ETS */
6752                 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
6753                 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
6754                 pf->tmp_cfg.etscfg.maxtcs = 0;
6755                 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
6756                 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
6757                 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;
6758                 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
6759                 /* FW needs one App to configure HW */
6760                 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;
6761                 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;
6762                 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;
6763                 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;
6764
6765                 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);
6766         }
6767
6768         memset(&ets_data, 0, sizeof(ets_data));
6769         ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */
6770         ets_data.tc_strict_priority_flags = 0; /* ETS */
6771         ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */
6772
6773         /* Enable ETS on the Physical port */
6774         err = i40e_aq_config_switch_comp_ets
6775                 (hw, pf->mac_seid, &ets_data,
6776                  i40e_aqc_opc_enable_switching_comp_ets, NULL);
6777         if (err) {
6778                 dev_info(&pf->pdev->dev,
6779                          "Enable Port ETS failed, err %s aq_err %s\n",
6780                          i40e_stat_str(&pf->hw, err),
6781                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6782                 err = -ENOENT;
6783                 goto out;
6784         }
6785
6786         /* Update the local cached instance with TC0 ETS */
6787         dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
6788         dcb_cfg->etscfg.cbs = 0;
6789         dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;
6790         dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
6791
6792 out:
6793         return err;
6794 }
6795
6796 /**
6797  * i40e_init_pf_dcb - Initialize DCB configuration
6798  * @pf: PF being configured
6799  *
6800  * Query the current DCB configuration and cache it
6801  * in the hardware structure
6802  **/
6803 static int i40e_init_pf_dcb(struct i40e_pf *pf)
6804 {
6805         struct i40e_hw *hw = &pf->hw;
6806         int err;
6807
6808         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable
6809          * Also do not enable DCBx if FW LLDP agent is disabled
6810          */
6811         if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT) {
6812                 dev_info(&pf->pdev->dev, "DCB is not supported.\n");
6813                 err = I40E_NOT_SUPPORTED;
6814                 goto out;
6815         }
6816         if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) {
6817                 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
6818                 err = i40e_dcb_sw_default_config(pf);
6819                 if (err) {
6820                         dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");
6821                         goto out;
6822                 }
6823                 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");
6824                 pf->dcbx_cap = DCB_CAP_DCBX_HOST |
6825                                DCB_CAP_DCBX_VER_IEEE;
6826                 /* at init capable but disabled */
6827                 pf->flags |= I40E_FLAG_DCB_CAPABLE;
6828                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6829                 goto out;
6830         }
6831         err = i40e_init_dcb(hw, true);
6832         if (!err) {
6833                 /* Device/Function is not DCBX capable */
6834                 if ((!hw->func_caps.dcb) ||
6835                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
6836                         dev_info(&pf->pdev->dev,
6837                                  "DCBX offload is not supported or is disabled for this PF.\n");
6838                 } else {
6839                         /* When status is not DISABLED then DCBX in FW */
6840                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
6841                                        DCB_CAP_DCBX_VER_IEEE;
6842
6843                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
6844                         /* Enable DCB tagging only when more than one TC
6845                          * or explicitly disable if only one TC
6846                          */
6847                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6848                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
6849                         else
6850                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6851                         dev_dbg(&pf->pdev->dev,
6852                                 "DCBX offload is supported for this PF.\n");
6853                 }
6854         } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
6855                 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
6856                 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
6857         } else {
6858                 dev_info(&pf->pdev->dev,
6859                          "Query for DCB configuration failed, err %s aq_err %s\n",
6860                          i40e_stat_str(&pf->hw, err),
6861                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6862         }
6863
6864 out:
6865         return err;
6866 }
6867 #endif /* CONFIG_I40E_DCB */
6868
6869 /**
6870  * i40e_set_lldp_forwarding - set forwarding of lldp frames
6871  * @pf: PF being configured
6872  * @enable: if forwarding to OS shall be enabled
6873  *
6874  * Toggle forwarding of lldp frames behavior,
6875  * When passing DCB control from firmware to software
6876  * lldp frames must be forwarded to the software based
6877  * lldp agent.
6878  */
6879 void i40e_set_lldp_forwarding(struct i40e_pf *pf, bool enable)
6880 {
6881         if (pf->lan_vsi == I40E_NO_VSI)
6882                 return;
6883
6884         if (!pf->vsi[pf->lan_vsi])
6885                 return;
6886
6887         /* No need to check the outcome, commands may fail
6888          * if desired value is already set
6889          */
6890         i40e_aq_add_rem_control_packet_filter(&pf->hw, NULL, ETH_P_LLDP,
6891                                               I40E_AQC_ADD_CONTROL_PACKET_FLAGS_TX |
6892                                               I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC,
6893                                               pf->vsi[pf->lan_vsi]->seid, 0,
6894                                               enable, NULL, NULL);
6895
6896         i40e_aq_add_rem_control_packet_filter(&pf->hw, NULL, ETH_P_LLDP,
6897                                               I40E_AQC_ADD_CONTROL_PACKET_FLAGS_RX |
6898                                               I40E_AQC_ADD_CONTROL_PACKET_FLAGS_IGNORE_MAC,
6899                                               pf->vsi[pf->lan_vsi]->seid, 0,
6900                                               enable, NULL, NULL);
6901 }
6902
6903 /**
6904  * i40e_print_link_message - print link up or down
6905  * @vsi: the VSI for which link needs a message
6906  * @isup: true of link is up, false otherwise
6907  */
6908 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
6909 {
6910         enum i40e_aq_link_speed new_speed;
6911         struct i40e_pf *pf = vsi->back;
6912         char *speed = "Unknown";
6913         char *fc = "Unknown";
6914         char *fec = "";
6915         char *req_fec = "";
6916         char *an = "";
6917
6918         if (isup)
6919                 new_speed = pf->hw.phy.link_info.link_speed;
6920         else
6921                 new_speed = I40E_LINK_SPEED_UNKNOWN;
6922
6923         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
6924                 return;
6925         vsi->current_isup = isup;
6926         vsi->current_speed = new_speed;
6927         if (!isup) {
6928                 netdev_info(vsi->netdev, "NIC Link is Down\n");
6929                 return;
6930         }
6931
6932         /* Warn user if link speed on NPAR enabled partition is not at
6933          * least 10GB
6934          */
6935         if (pf->hw.func_caps.npar_enable &&
6936             (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
6937              pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
6938                 netdev_warn(vsi->netdev,
6939                             "The partition detected link speed that is less than 10Gbps\n");
6940
6941         switch (pf->hw.phy.link_info.link_speed) {
6942         case I40E_LINK_SPEED_40GB:
6943                 speed = "40 G";
6944                 break;
6945         case I40E_LINK_SPEED_20GB:
6946                 speed = "20 G";
6947                 break;
6948         case I40E_LINK_SPEED_25GB:
6949                 speed = "25 G";
6950                 break;
6951         case I40E_LINK_SPEED_10GB:
6952                 speed = "10 G";
6953                 break;
6954         case I40E_LINK_SPEED_5GB:
6955                 speed = "5 G";
6956                 break;
6957         case I40E_LINK_SPEED_2_5GB:
6958                 speed = "2.5 G";
6959                 break;
6960         case I40E_LINK_SPEED_1GB:
6961                 speed = "1000 M";
6962                 break;
6963         case I40E_LINK_SPEED_100MB:
6964                 speed = "100 M";
6965                 break;
6966         default:
6967                 break;
6968         }
6969
6970         switch (pf->hw.fc.current_mode) {
6971         case I40E_FC_FULL:
6972                 fc = "RX/TX";
6973                 break;
6974         case I40E_FC_TX_PAUSE:
6975                 fc = "TX";
6976                 break;
6977         case I40E_FC_RX_PAUSE:
6978                 fc = "RX";
6979                 break;
6980         default:
6981                 fc = "None";
6982                 break;
6983         }
6984
6985         if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
6986                 req_fec = "None";
6987                 fec = "None";
6988                 an = "False";
6989
6990                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
6991                         an = "True";
6992
6993                 if (pf->hw.phy.link_info.fec_info &
6994                     I40E_AQ_CONFIG_FEC_KR_ENA)
6995                         fec = "CL74 FC-FEC/BASE-R";
6996                 else if (pf->hw.phy.link_info.fec_info &
6997                          I40E_AQ_CONFIG_FEC_RS_ENA)
6998                         fec = "CL108 RS-FEC";
6999
7000                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
7001                  * both RS and FC are requested
7002                  */
7003                 if (vsi->back->hw.phy.link_info.req_fec_info &
7004                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
7005                         if (vsi->back->hw.phy.link_info.req_fec_info &
7006                             I40E_AQ_REQUEST_FEC_RS)
7007                                 req_fec = "CL108 RS-FEC";
7008                         else
7009                                 req_fec = "CL74 FC-FEC/BASE-R";
7010                 }
7011                 netdev_info(vsi->netdev,
7012                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7013                             speed, req_fec, fec, an, fc);
7014         } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
7015                 req_fec = "None";
7016                 fec = "None";
7017                 an = "False";
7018
7019                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7020                         an = "True";
7021
7022                 if (pf->hw.phy.link_info.fec_info &
7023                     I40E_AQ_CONFIG_FEC_KR_ENA)
7024                         fec = "CL74 FC-FEC/BASE-R";
7025
7026                 if (pf->hw.phy.link_info.req_fec_info &
7027                     I40E_AQ_REQUEST_FEC_KR)
7028                         req_fec = "CL74 FC-FEC/BASE-R";
7029
7030                 netdev_info(vsi->netdev,
7031                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7032                             speed, req_fec, fec, an, fc);
7033         } else {
7034                 netdev_info(vsi->netdev,
7035                             "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
7036                             speed, fc);
7037         }
7038
7039 }
7040
7041 /**
7042  * i40e_up_complete - Finish the last steps of bringing up a connection
7043  * @vsi: the VSI being configured
7044  **/
7045 static int i40e_up_complete(struct i40e_vsi *vsi)
7046 {
7047         struct i40e_pf *pf = vsi->back;
7048         int err;
7049
7050         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7051                 i40e_vsi_configure_msix(vsi);
7052         else
7053                 i40e_configure_msi_and_legacy(vsi);
7054
7055         /* start rings */
7056         err = i40e_vsi_start_rings(vsi);
7057         if (err)
7058                 return err;
7059
7060         clear_bit(__I40E_VSI_DOWN, vsi->state);
7061         i40e_napi_enable_all(vsi);
7062         i40e_vsi_enable_irq(vsi);
7063
7064         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
7065             (vsi->netdev)) {
7066                 i40e_print_link_message(vsi, true);
7067                 netif_tx_start_all_queues(vsi->netdev);
7068                 netif_carrier_on(vsi->netdev);
7069         }
7070
7071         /* replay FDIR SB filters */
7072         if (vsi->type == I40E_VSI_FDIR) {
7073                 /* reset fd counters */
7074                 pf->fd_add_err = 0;
7075                 pf->fd_atr_cnt = 0;
7076                 i40e_fdir_filter_restore(vsi);
7077         }
7078
7079         /* On the next run of the service_task, notify any clients of the new
7080          * opened netdev
7081          */
7082         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7083         i40e_service_event_schedule(pf);
7084
7085         return 0;
7086 }
7087
7088 /**
7089  * i40e_vsi_reinit_locked - Reset the VSI
7090  * @vsi: the VSI being configured
7091  *
7092  * Rebuild the ring structs after some configuration
7093  * has changed, e.g. MTU size.
7094  **/
7095 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
7096 {
7097         struct i40e_pf *pf = vsi->back;
7098
7099         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
7100                 usleep_range(1000, 2000);
7101         i40e_down(vsi);
7102
7103         i40e_up(vsi);
7104         clear_bit(__I40E_CONFIG_BUSY, pf->state);
7105 }
7106
7107 /**
7108  * i40e_force_link_state - Force the link status
7109  * @pf: board private structure
7110  * @is_up: whether the link state should be forced up or down
7111  **/
7112 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up)
7113 {
7114         struct i40e_aq_get_phy_abilities_resp abilities;
7115         struct i40e_aq_set_phy_config config = {0};
7116         bool non_zero_phy_type = is_up;
7117         struct i40e_hw *hw = &pf->hw;
7118         i40e_status err;
7119         u64 mask;
7120         u8 speed;
7121
7122         /* Card might've been put in an unstable state by other drivers
7123          * and applications, which causes incorrect speed values being
7124          * set on startup. In order to clear speed registers, we call
7125          * get_phy_capabilities twice, once to get initial state of
7126          * available speeds, and once to get current PHY config.
7127          */
7128         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
7129                                            NULL);
7130         if (err) {
7131                 dev_err(&pf->pdev->dev,
7132                         "failed to get phy cap., ret =  %s last_status =  %s\n",
7133                         i40e_stat_str(hw, err),
7134                         i40e_aq_str(hw, hw->aq.asq_last_status));
7135                 return err;
7136         }
7137         speed = abilities.link_speed;
7138
7139         /* Get the current phy config */
7140         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
7141                                            NULL);
7142         if (err) {
7143                 dev_err(&pf->pdev->dev,
7144                         "failed to get phy cap., ret =  %s last_status =  %s\n",
7145                         i40e_stat_str(hw, err),
7146                         i40e_aq_str(hw, hw->aq.asq_last_status));
7147                 return err;
7148         }
7149
7150         /* If link needs to go up, but was not forced to go down,
7151          * and its speed values are OK, no need for a flap
7152          * if non_zero_phy_type was set, still need to force up
7153          */
7154         if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)
7155                 non_zero_phy_type = true;
7156         else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
7157                 return I40E_SUCCESS;
7158
7159         /* To force link we need to set bits for all supported PHY types,
7160          * but there are now more than 32, so we need to split the bitmap
7161          * across two fields.
7162          */
7163         mask = I40E_PHY_TYPES_BITMASK;
7164         config.phy_type =
7165                 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
7166         config.phy_type_ext =
7167                 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
7168         /* Copy the old settings, except of phy_type */
7169         config.abilities = abilities.abilities;
7170         if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) {
7171                 if (is_up)
7172                         config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
7173                 else
7174                         config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
7175         }
7176         if (abilities.link_speed != 0)
7177                 config.link_speed = abilities.link_speed;
7178         else
7179                 config.link_speed = speed;
7180         config.eee_capability = abilities.eee_capability;
7181         config.eeer = abilities.eeer_val;
7182         config.low_power_ctrl = abilities.d3_lpan;
7183         config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
7184                             I40E_AQ_PHY_FEC_CONFIG_MASK;
7185         err = i40e_aq_set_phy_config(hw, &config, NULL);
7186
7187         if (err) {
7188                 dev_err(&pf->pdev->dev,
7189                         "set phy config ret =  %s last_status =  %s\n",
7190                         i40e_stat_str(&pf->hw, err),
7191                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7192                 return err;
7193         }
7194
7195         /* Update the link info */
7196         err = i40e_update_link_info(hw);
7197         if (err) {
7198                 /* Wait a little bit (on 40G cards it sometimes takes a really
7199                  * long time for link to come back from the atomic reset)
7200                  * and try once more
7201                  */
7202                 msleep(1000);
7203                 i40e_update_link_info(hw);
7204         }
7205
7206         i40e_aq_set_link_restart_an(hw, is_up, NULL);
7207
7208         return I40E_SUCCESS;
7209 }
7210
7211 /**
7212  * i40e_up - Bring the connection back up after being down
7213  * @vsi: the VSI being configured
7214  **/
7215 int i40e_up(struct i40e_vsi *vsi)
7216 {
7217         int err;
7218
7219         if (vsi->type == I40E_VSI_MAIN &&
7220             (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7221              vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7222                 i40e_force_link_state(vsi->back, true);
7223
7224         err = i40e_vsi_configure(vsi);
7225         if (!err)
7226                 err = i40e_up_complete(vsi);
7227
7228         return err;
7229 }
7230
7231 /**
7232  * i40e_down - Shutdown the connection processing
7233  * @vsi: the VSI being stopped
7234  **/
7235 void i40e_down(struct i40e_vsi *vsi)
7236 {
7237         int i;
7238
7239         /* It is assumed that the caller of this function
7240          * sets the vsi->state __I40E_VSI_DOWN bit.
7241          */
7242         if (vsi->netdev) {
7243                 netif_carrier_off(vsi->netdev);
7244                 netif_tx_disable(vsi->netdev);
7245         }
7246         i40e_vsi_disable_irq(vsi);
7247         i40e_vsi_stop_rings(vsi);
7248         if (vsi->type == I40E_VSI_MAIN &&
7249            (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7250             vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7251                 i40e_force_link_state(vsi->back, false);
7252         i40e_napi_disable_all(vsi);
7253
7254         for (i = 0; i < vsi->num_queue_pairs; i++) {
7255                 i40e_clean_tx_ring(vsi->tx_rings[i]);
7256                 if (i40e_enabled_xdp_vsi(vsi)) {
7257                         /* Make sure that in-progress ndo_xdp_xmit and
7258                          * ndo_xsk_wakeup calls are completed.
7259                          */
7260                         synchronize_rcu();
7261                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
7262                 }
7263                 i40e_clean_rx_ring(vsi->rx_rings[i]);
7264         }
7265
7266 }
7267
7268 /**
7269  * i40e_validate_mqprio_qopt- validate queue mapping info
7270  * @vsi: the VSI being configured
7271  * @mqprio_qopt: queue parametrs
7272  **/
7273 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7274                                      struct tc_mqprio_qopt_offload *mqprio_qopt)
7275 {
7276         u64 sum_max_rate = 0;
7277         u64 max_rate = 0;
7278         int i;
7279
7280         if (mqprio_qopt->qopt.offset[0] != 0 ||
7281             mqprio_qopt->qopt.num_tc < 1 ||
7282             mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7283                 return -EINVAL;
7284         for (i = 0; ; i++) {
7285                 if (!mqprio_qopt->qopt.count[i])
7286                         return -EINVAL;
7287                 if (mqprio_qopt->min_rate[i]) {
7288                         dev_err(&vsi->back->pdev->dev,
7289                                 "Invalid min tx rate (greater than 0) specified\n");
7290                         return -EINVAL;
7291                 }
7292                 max_rate = mqprio_qopt->max_rate[i];
7293                 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7294                 sum_max_rate += max_rate;
7295
7296                 if (i >= mqprio_qopt->qopt.num_tc - 1)
7297                         break;
7298                 if (mqprio_qopt->qopt.offset[i + 1] !=
7299                     (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7300                         return -EINVAL;
7301         }
7302         if (vsi->num_queue_pairs <
7303             (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7304                 return -EINVAL;
7305         }
7306         if (sum_max_rate > i40e_get_link_speed(vsi)) {
7307                 dev_err(&vsi->back->pdev->dev,
7308                         "Invalid max tx rate specified\n");
7309                 return -EINVAL;
7310         }
7311         return 0;
7312 }
7313
7314 /**
7315  * i40e_vsi_set_default_tc_config - set default values for tc configuration
7316  * @vsi: the VSI being configured
7317  **/
7318 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7319 {
7320         u16 qcount;
7321         int i;
7322
7323         /* Only TC0 is enabled */
7324         vsi->tc_config.numtc = 1;
7325         vsi->tc_config.enabled_tc = 1;
7326         qcount = min_t(int, vsi->alloc_queue_pairs,
7327                        i40e_pf_get_max_q_per_tc(vsi->back));
7328         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7329                 /* For the TC that is not enabled set the offset to to default
7330                  * queue and allocate one queue for the given TC.
7331                  */
7332                 vsi->tc_config.tc_info[i].qoffset = 0;
7333                 if (i == 0)
7334                         vsi->tc_config.tc_info[i].qcount = qcount;
7335                 else
7336                         vsi->tc_config.tc_info[i].qcount = 1;
7337                 vsi->tc_config.tc_info[i].netdev_tc = 0;
7338         }
7339 }
7340
7341 /**
7342  * i40e_del_macvlan_filter
7343  * @hw: pointer to the HW structure
7344  * @seid: seid of the channel VSI
7345  * @macaddr: the mac address to apply as a filter
7346  * @aq_err: store the admin Q error
7347  *
7348  * This function deletes a mac filter on the channel VSI which serves as the
7349  * macvlan. Returns 0 on success.
7350  **/
7351 static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7352                                            const u8 *macaddr, int *aq_err)
7353 {
7354         struct i40e_aqc_remove_macvlan_element_data element;
7355         i40e_status status;
7356
7357         memset(&element, 0, sizeof(element));
7358         ether_addr_copy(element.mac_addr, macaddr);
7359         element.vlan_tag = 0;
7360         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7361         status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
7362         *aq_err = hw->aq.asq_last_status;
7363
7364         return status;
7365 }
7366
7367 /**
7368  * i40e_add_macvlan_filter
7369  * @hw: pointer to the HW structure
7370  * @seid: seid of the channel VSI
7371  * @macaddr: the mac address to apply as a filter
7372  * @aq_err: store the admin Q error
7373  *
7374  * This function adds a mac filter on the channel VSI which serves as the
7375  * macvlan. Returns 0 on success.
7376  **/
7377 static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7378                                            const u8 *macaddr, int *aq_err)
7379 {
7380         struct i40e_aqc_add_macvlan_element_data element;
7381         i40e_status status;
7382         u16 cmd_flags = 0;
7383
7384         ether_addr_copy(element.mac_addr, macaddr);
7385         element.vlan_tag = 0;
7386         element.queue_number = 0;
7387         element.match_method = I40E_AQC_MM_ERR_NO_RES;
7388         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7389         element.flags = cpu_to_le16(cmd_flags);
7390         status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
7391         *aq_err = hw->aq.asq_last_status;
7392
7393         return status;
7394 }
7395
7396 /**
7397  * i40e_reset_ch_rings - Reset the queue contexts in a channel
7398  * @vsi: the VSI we want to access
7399  * @ch: the channel we want to access
7400  */
7401 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7402 {
7403         struct i40e_ring *tx_ring, *rx_ring;
7404         u16 pf_q;
7405         int i;
7406
7407         for (i = 0; i < ch->num_queue_pairs; i++) {
7408                 pf_q = ch->base_queue + i;
7409                 tx_ring = vsi->tx_rings[pf_q];
7410                 tx_ring->ch = NULL;
7411                 rx_ring = vsi->rx_rings[pf_q];
7412                 rx_ring->ch = NULL;
7413         }
7414 }
7415
7416 /**
7417  * i40e_free_macvlan_channels
7418  * @vsi: the VSI we want to access
7419  *
7420  * This function frees the Qs of the channel VSI from
7421  * the stack and also deletes the channel VSIs which
7422  * serve as macvlans.
7423  */
7424 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7425 {
7426         struct i40e_channel *ch, *ch_tmp;
7427         int ret;
7428
7429         if (list_empty(&vsi->macvlan_list))
7430                 return;
7431
7432         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7433                 struct i40e_vsi *parent_vsi;
7434
7435                 if (i40e_is_channel_macvlan(ch)) {
7436                         i40e_reset_ch_rings(vsi, ch);
7437                         clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7438                         netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7439                         netdev_set_sb_channel(ch->fwd->netdev, 0);
7440                         kfree(ch->fwd);
7441                         ch->fwd = NULL;
7442                 }
7443
7444                 list_del(&ch->list);
7445                 parent_vsi = ch->parent_vsi;
7446                 if (!parent_vsi || !ch->initialized) {
7447                         kfree(ch);
7448                         continue;
7449                 }
7450
7451                 /* remove the VSI */
7452                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7453                                              NULL);
7454                 if (ret)
7455                         dev_err(&vsi->back->pdev->dev,
7456                                 "unable to remove channel (%d) for parent VSI(%d)\n",
7457                                 ch->seid, parent_vsi->seid);
7458                 kfree(ch);
7459         }
7460         vsi->macvlan_cnt = 0;
7461 }
7462
7463 /**
7464  * i40e_fwd_ring_up - bring the macvlan device up
7465  * @vsi: the VSI we want to access
7466  * @vdev: macvlan netdevice
7467  * @fwd: the private fwd structure
7468  */
7469 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7470                             struct i40e_fwd_adapter *fwd)
7471 {
7472         int ret = 0, num_tc = 1,  i, aq_err;
7473         struct i40e_channel *ch, *ch_tmp;
7474         struct i40e_pf *pf = vsi->back;
7475         struct i40e_hw *hw = &pf->hw;
7476
7477         if (list_empty(&vsi->macvlan_list))
7478                 return -EINVAL;
7479
7480         /* Go through the list and find an available channel */
7481         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7482                 if (!i40e_is_channel_macvlan(ch)) {
7483                         ch->fwd = fwd;
7484                         /* record configuration for macvlan interface in vdev */
7485                         for (i = 0; i < num_tc; i++)
7486                                 netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7487                                                              i,
7488                                                              ch->num_queue_pairs,
7489                                                              ch->base_queue);
7490                         for (i = 0; i < ch->num_queue_pairs; i++) {
7491                                 struct i40e_ring *tx_ring, *rx_ring;
7492                                 u16 pf_q;
7493
7494                                 pf_q = ch->base_queue + i;
7495
7496                                 /* Get to TX ring ptr */
7497                                 tx_ring = vsi->tx_rings[pf_q];
7498                                 tx_ring->ch = ch;
7499
7500                                 /* Get the RX ring ptr */
7501                                 rx_ring = vsi->rx_rings[pf_q];
7502                                 rx_ring->ch = ch;
7503                         }
7504                         break;
7505                 }
7506         }
7507
7508         /* Guarantee all rings are updated before we update the
7509          * MAC address filter.
7510          */
7511         wmb();
7512
7513         /* Add a mac filter */
7514         ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7515         if (ret) {
7516                 /* if we cannot add the MAC rule then disable the offload */
7517                 macvlan_release_l2fw_offload(vdev);
7518                 for (i = 0; i < ch->num_queue_pairs; i++) {
7519                         struct i40e_ring *rx_ring;
7520                         u16 pf_q;
7521
7522                         pf_q = ch->base_queue + i;
7523                         rx_ring = vsi->rx_rings[pf_q];
7524                         rx_ring->netdev = NULL;
7525                 }
7526                 dev_info(&pf->pdev->dev,
7527                          "Error adding mac filter on macvlan err %s, aq_err %s\n",
7528                           i40e_stat_str(hw, ret),
7529                           i40e_aq_str(hw, aq_err));
7530                 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7531         }
7532
7533         return ret;
7534 }
7535
7536 /**
7537  * i40e_setup_macvlans - create the channels which will be macvlans
7538  * @vsi: the VSI we want to access
7539  * @macvlan_cnt: no. of macvlans to be setup
7540  * @qcnt: no. of Qs per macvlan
7541  * @vdev: macvlan netdevice
7542  */
7543 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7544                                struct net_device *vdev)
7545 {
7546         struct i40e_pf *pf = vsi->back;
7547         struct i40e_hw *hw = &pf->hw;
7548         struct i40e_vsi_context ctxt;
7549         u16 sections, qmap, num_qps;
7550         struct i40e_channel *ch;
7551         int i, pow, ret = 0;
7552         u8 offset = 0;
7553
7554         if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7555                 return -EINVAL;
7556
7557         num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7558
7559         /* find the next higher power-of-2 of num queue pairs */
7560         pow = fls(roundup_pow_of_two(num_qps) - 1);
7561
7562         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7563                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7564
7565         /* Setup context bits for the main VSI */
7566         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7567         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7568         memset(&ctxt, 0, sizeof(ctxt));
7569         ctxt.seid = vsi->seid;
7570         ctxt.pf_num = vsi->back->hw.pf_id;
7571         ctxt.vf_num = 0;
7572         ctxt.uplink_seid = vsi->uplink_seid;
7573         ctxt.info = vsi->info;
7574         ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7575         ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7576         ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7577         ctxt.info.valid_sections |= cpu_to_le16(sections);
7578
7579         /* Reconfigure RSS for main VSI with new max queue count */
7580         vsi->rss_size = max_t(u16, num_qps, qcnt);
7581         ret = i40e_vsi_config_rss(vsi);
7582         if (ret) {
7583                 dev_info(&pf->pdev->dev,
7584                          "Failed to reconfig RSS for num_queues (%u)\n",
7585                          vsi->rss_size);
7586                 return ret;
7587         }
7588         vsi->reconfig_rss = true;
7589         dev_dbg(&vsi->back->pdev->dev,
7590                 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7591         vsi->next_base_queue = num_qps;
7592         vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7593
7594         /* Update the VSI after updating the VSI queue-mapping
7595          * information
7596          */
7597         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7598         if (ret) {
7599                 dev_info(&pf->pdev->dev,
7600                          "Update vsi tc config failed, err %s aq_err %s\n",
7601                          i40e_stat_str(hw, ret),
7602                          i40e_aq_str(hw, hw->aq.asq_last_status));
7603                 return ret;
7604         }
7605         /* update the local VSI info with updated queue map */
7606         i40e_vsi_update_queue_map(vsi, &ctxt);
7607         vsi->info.valid_sections = 0;
7608
7609         /* Create channels for macvlans */
7610         INIT_LIST_HEAD(&vsi->macvlan_list);
7611         for (i = 0; i < macvlan_cnt; i++) {
7612                 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7613                 if (!ch) {
7614                         ret = -ENOMEM;
7615                         goto err_free;
7616                 }
7617                 INIT_LIST_HEAD(&ch->list);
7618                 ch->num_queue_pairs = qcnt;
7619                 if (!i40e_setup_channel(pf, vsi, ch)) {
7620                         ret = -EINVAL;
7621                         kfree(ch);
7622                         goto err_free;
7623                 }
7624                 ch->parent_vsi = vsi;
7625                 vsi->cnt_q_avail -= ch->num_queue_pairs;
7626                 vsi->macvlan_cnt++;
7627                 list_add_tail(&ch->list, &vsi->macvlan_list);
7628         }
7629
7630         return ret;
7631
7632 err_free:
7633         dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
7634         i40e_free_macvlan_channels(vsi);
7635
7636         return ret;
7637 }
7638
7639 /**
7640  * i40e_fwd_add - configure macvlans
7641  * @netdev: net device to configure
7642  * @vdev: macvlan netdevice
7643  **/
7644 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
7645 {
7646         struct i40e_netdev_priv *np = netdev_priv(netdev);
7647         u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
7648         struct i40e_vsi *vsi = np->vsi;
7649         struct i40e_pf *pf = vsi->back;
7650         struct i40e_fwd_adapter *fwd;
7651         int avail_macvlan, ret;
7652
7653         if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
7654                 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
7655                 return ERR_PTR(-EINVAL);
7656         }
7657         if ((pf->flags & I40E_FLAG_TC_MQPRIO)) {
7658                 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
7659                 return ERR_PTR(-EINVAL);
7660         }
7661         if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
7662                 netdev_info(netdev, "Not enough vectors available to support macvlans\n");
7663                 return ERR_PTR(-EINVAL);
7664         }
7665
7666         /* The macvlan device has to be a single Q device so that the
7667          * tc_to_txq field can be reused to pick the tx queue.
7668          */
7669         if (netif_is_multiqueue(vdev))
7670                 return ERR_PTR(-ERANGE);
7671
7672         if (!vsi->macvlan_cnt) {
7673                 /* reserve bit 0 for the pf device */
7674                 set_bit(0, vsi->fwd_bitmask);
7675
7676                 /* Try to reserve as many queues as possible for macvlans. First
7677                  * reserve 3/4th of max vectors, then half, then quarter and
7678                  * calculate Qs per macvlan as you go
7679                  */
7680                 vectors = pf->num_lan_msix;
7681                 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
7682                         /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
7683                         q_per_macvlan = 4;
7684                         macvlan_cnt = (vectors - 32) / 4;
7685                 } else if (vectors <= 64 && vectors > 32) {
7686                         /* allocate 2 Qs per macvlan and 16 Qs to the PF*/
7687                         q_per_macvlan = 2;
7688                         macvlan_cnt = (vectors - 16) / 2;
7689                 } else if (vectors <= 32 && vectors > 16) {
7690                         /* allocate 1 Q per macvlan and 16 Qs to the PF*/
7691                         q_per_macvlan = 1;
7692                         macvlan_cnt = vectors - 16;
7693                 } else if (vectors <= 16 && vectors > 8) {
7694                         /* allocate 1 Q per macvlan and 8 Qs to the PF */
7695                         q_per_macvlan = 1;
7696                         macvlan_cnt = vectors - 8;
7697                 } else {
7698                         /* allocate 1 Q per macvlan and 1 Q to the PF */
7699                         q_per_macvlan = 1;
7700                         macvlan_cnt = vectors - 1;
7701                 }
7702
7703                 if (macvlan_cnt == 0)
7704                         return ERR_PTR(-EBUSY);
7705
7706                 /* Quiesce VSI queues */
7707                 i40e_quiesce_vsi(vsi);
7708
7709                 /* sets up the macvlans but does not "enable" them */
7710                 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
7711                                           vdev);
7712                 if (ret)
7713                         return ERR_PTR(ret);
7714
7715                 /* Unquiesce VSI */
7716                 i40e_unquiesce_vsi(vsi);
7717         }
7718         avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
7719                                             vsi->macvlan_cnt);
7720         if (avail_macvlan >= I40E_MAX_MACVLANS)
7721                 return ERR_PTR(-EBUSY);
7722
7723         /* create the fwd struct */
7724         fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
7725         if (!fwd)
7726                 return ERR_PTR(-ENOMEM);
7727
7728         set_bit(avail_macvlan, vsi->fwd_bitmask);
7729         fwd->bit_no = avail_macvlan;
7730         netdev_set_sb_channel(vdev, avail_macvlan);
7731         fwd->netdev = vdev;
7732
7733         if (!netif_running(netdev))
7734                 return fwd;
7735
7736         /* Set fwd ring up */
7737         ret = i40e_fwd_ring_up(vsi, vdev, fwd);
7738         if (ret) {
7739                 /* unbind the queues and drop the subordinate channel config */
7740                 netdev_unbind_sb_channel(netdev, vdev);
7741                 netdev_set_sb_channel(vdev, 0);
7742
7743                 kfree(fwd);
7744                 return ERR_PTR(-EINVAL);
7745         }
7746
7747         return fwd;
7748 }
7749
7750 /**
7751  * i40e_del_all_macvlans - Delete all the mac filters on the channels
7752  * @vsi: the VSI we want to access
7753  */
7754 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
7755 {
7756         struct i40e_channel *ch, *ch_tmp;
7757         struct i40e_pf *pf = vsi->back;
7758         struct i40e_hw *hw = &pf->hw;
7759         int aq_err, ret = 0;
7760
7761         if (list_empty(&vsi->macvlan_list))
7762                 return;
7763
7764         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7765                 if (i40e_is_channel_macvlan(ch)) {
7766                         ret = i40e_del_macvlan_filter(hw, ch->seid,
7767                                                       i40e_channel_mac(ch),
7768                                                       &aq_err);
7769                         if (!ret) {
7770                                 /* Reset queue contexts */
7771                                 i40e_reset_ch_rings(vsi, ch);
7772                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7773                                 netdev_unbind_sb_channel(vsi->netdev,
7774                                                          ch->fwd->netdev);
7775                                 netdev_set_sb_channel(ch->fwd->netdev, 0);
7776                                 kfree(ch->fwd);
7777                                 ch->fwd = NULL;
7778                         }
7779                 }
7780         }
7781 }
7782
7783 /**
7784  * i40e_fwd_del - delete macvlan interfaces
7785  * @netdev: net device to configure
7786  * @vdev: macvlan netdevice
7787  */
7788 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
7789 {
7790         struct i40e_netdev_priv *np = netdev_priv(netdev);
7791         struct i40e_fwd_adapter *fwd = vdev;
7792         struct i40e_channel *ch, *ch_tmp;
7793         struct i40e_vsi *vsi = np->vsi;
7794         struct i40e_pf *pf = vsi->back;
7795         struct i40e_hw *hw = &pf->hw;
7796         int aq_err, ret = 0;
7797
7798         /* Find the channel associated with the macvlan and del mac filter */
7799         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7800                 if (i40e_is_channel_macvlan(ch) &&
7801                     ether_addr_equal(i40e_channel_mac(ch),
7802                                      fwd->netdev->dev_addr)) {
7803                         ret = i40e_del_macvlan_filter(hw, ch->seid,
7804                                                       i40e_channel_mac(ch),
7805                                                       &aq_err);
7806                         if (!ret) {
7807                                 /* Reset queue contexts */
7808                                 i40e_reset_ch_rings(vsi, ch);
7809                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7810                                 netdev_unbind_sb_channel(netdev, fwd->netdev);
7811                                 netdev_set_sb_channel(fwd->netdev, 0);
7812                                 kfree(ch->fwd);
7813                                 ch->fwd = NULL;
7814                         } else {
7815                                 dev_info(&pf->pdev->dev,
7816                                          "Error deleting mac filter on macvlan err %s, aq_err %s\n",
7817                                           i40e_stat_str(hw, ret),
7818                                           i40e_aq_str(hw, aq_err));
7819                         }
7820                         break;
7821                 }
7822         }
7823 }
7824
7825 /**
7826  * i40e_setup_tc - configure multiple traffic classes
7827  * @netdev: net device to configure
7828  * @type_data: tc offload data
7829  **/
7830 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
7831 {
7832         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
7833         struct i40e_netdev_priv *np = netdev_priv(netdev);
7834         struct i40e_vsi *vsi = np->vsi;
7835         struct i40e_pf *pf = vsi->back;
7836         u8 enabled_tc = 0, num_tc, hw;
7837         bool need_reset = false;
7838         int old_queue_pairs;
7839         int ret = -EINVAL;
7840         u16 mode;
7841         int i;
7842
7843         old_queue_pairs = vsi->num_queue_pairs;
7844         num_tc = mqprio_qopt->qopt.num_tc;
7845         hw = mqprio_qopt->qopt.hw;
7846         mode = mqprio_qopt->mode;
7847         if (!hw) {
7848                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7849                 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
7850                 goto config_tc;
7851         }
7852
7853         /* Check if MFP enabled */
7854         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
7855                 netdev_info(netdev,
7856                             "Configuring TC not supported in MFP mode\n");
7857                 return ret;
7858         }
7859         switch (mode) {
7860         case TC_MQPRIO_MODE_DCB:
7861                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7862
7863                 /* Check if DCB enabled to continue */
7864                 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7865                         netdev_info(netdev,
7866                                     "DCB is not enabled for adapter\n");
7867                         return ret;
7868                 }
7869
7870                 /* Check whether tc count is within enabled limit */
7871                 if (num_tc > i40e_pf_get_num_tc(pf)) {
7872                         netdev_info(netdev,
7873                                     "TC count greater than enabled on link for adapter\n");
7874                         return ret;
7875                 }
7876                 break;
7877         case TC_MQPRIO_MODE_CHANNEL:
7878                 if (pf->flags & I40E_FLAG_DCB_ENABLED) {
7879                         netdev_info(netdev,
7880                                     "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
7881                         return ret;
7882                 }
7883                 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
7884                         return ret;
7885                 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
7886                 if (ret)
7887                         return ret;
7888                 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
7889                        sizeof(*mqprio_qopt));
7890                 pf->flags |= I40E_FLAG_TC_MQPRIO;
7891                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
7892                 break;
7893         default:
7894                 return -EINVAL;
7895         }
7896
7897 config_tc:
7898         /* Generate TC map for number of tc requested */
7899         for (i = 0; i < num_tc; i++)
7900                 enabled_tc |= BIT(i);
7901
7902         /* Requesting same TC configuration as already enabled */
7903         if (enabled_tc == vsi->tc_config.enabled_tc &&
7904             mode != TC_MQPRIO_MODE_CHANNEL)
7905                 return 0;
7906
7907         /* Quiesce VSI queues */
7908         i40e_quiesce_vsi(vsi);
7909
7910         if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO))
7911                 i40e_remove_queue_channels(vsi);
7912
7913         /* Configure VSI for enabled TCs */
7914         ret = i40e_vsi_config_tc(vsi, enabled_tc);
7915         if (ret) {
7916                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
7917                             vsi->seid);
7918                 need_reset = true;
7919                 goto exit;
7920         } else {
7921                 dev_info(&vsi->back->pdev->dev,
7922                          "Setup channel (id:%u) utilizing num_queues %d\n",
7923                          vsi->seid, vsi->tc_config.tc_info[0].qcount);
7924         }
7925
7926         if (pf->flags & I40E_FLAG_TC_MQPRIO) {
7927                 if (vsi->mqprio_qopt.max_rate[0]) {
7928                         u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
7929
7930                         do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
7931                         ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
7932                         if (!ret) {
7933                                 u64 credits = max_tx_rate;
7934
7935                                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
7936                                 dev_dbg(&vsi->back->pdev->dev,
7937                                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
7938                                         max_tx_rate,
7939                                         credits,
7940                                         vsi->seid);
7941                         } else {
7942                                 need_reset = true;
7943                                 goto exit;
7944                         }
7945                 }
7946                 ret = i40e_configure_queue_channels(vsi);
7947                 if (ret) {
7948                         vsi->num_queue_pairs = old_queue_pairs;
7949                         netdev_info(netdev,
7950                                     "Failed configuring queue channels\n");
7951                         need_reset = true;
7952                         goto exit;
7953                 }
7954         }
7955
7956 exit:
7957         /* Reset the configuration data to defaults, only TC0 is enabled */
7958         if (need_reset) {
7959                 i40e_vsi_set_default_tc_config(vsi);
7960                 need_reset = false;
7961         }
7962
7963         /* Unquiesce VSI */
7964         i40e_unquiesce_vsi(vsi);
7965         return ret;
7966 }
7967
7968 /**
7969  * i40e_set_cld_element - sets cloud filter element data
7970  * @filter: cloud filter rule
7971  * @cld: ptr to cloud filter element data
7972  *
7973  * This is helper function to copy data into cloud filter element
7974  **/
7975 static inline void
7976 i40e_set_cld_element(struct i40e_cloud_filter *filter,
7977                      struct i40e_aqc_cloud_filters_element_data *cld)
7978 {
7979         u32 ipa;
7980         int i;
7981
7982         memset(cld, 0, sizeof(*cld));
7983         ether_addr_copy(cld->outer_mac, filter->dst_mac);
7984         ether_addr_copy(cld->inner_mac, filter->src_mac);
7985
7986         if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
7987                 return;
7988
7989         if (filter->n_proto == ETH_P_IPV6) {
7990 #define IPV6_MAX_INDEX  (ARRAY_SIZE(filter->dst_ipv6) - 1)
7991                 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
7992                         ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
7993
7994                         *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
7995                 }
7996         } else {
7997                 ipa = be32_to_cpu(filter->dst_ipv4);
7998
7999                 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
8000         }
8001
8002         cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
8003
8004         /* tenant_id is not supported by FW now, once the support is enabled
8005          * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
8006          */
8007         if (filter->tenant_id)
8008                 return;
8009 }
8010
8011 /**
8012  * i40e_add_del_cloud_filter - Add/del cloud filter
8013  * @vsi: pointer to VSI
8014  * @filter: cloud filter rule
8015  * @add: if true, add, if false, delete
8016  *
8017  * Add or delete a cloud filter for a specific flow spec.
8018  * Returns 0 if the filter were successfully added.
8019  **/
8020 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
8021                               struct i40e_cloud_filter *filter, bool add)
8022 {
8023         struct i40e_aqc_cloud_filters_element_data cld_filter;
8024         struct i40e_pf *pf = vsi->back;
8025         int ret;
8026         static const u16 flag_table[128] = {
8027                 [I40E_CLOUD_FILTER_FLAGS_OMAC]  =
8028                         I40E_AQC_ADD_CLOUD_FILTER_OMAC,
8029                 [I40E_CLOUD_FILTER_FLAGS_IMAC]  =
8030                         I40E_AQC_ADD_CLOUD_FILTER_IMAC,
8031                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
8032                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
8033                 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
8034                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
8035                 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
8036                         I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
8037                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
8038                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
8039                 [I40E_CLOUD_FILTER_FLAGS_IIP] =
8040                         I40E_AQC_ADD_CLOUD_FILTER_IIP,
8041         };
8042
8043         if (filter->flags >= ARRAY_SIZE(flag_table))
8044                 return I40E_ERR_CONFIG;
8045
8046         memset(&cld_filter, 0, sizeof(cld_filter));
8047
8048         /* copy element needed to add cloud filter from filter */
8049         i40e_set_cld_element(filter, &cld_filter);
8050
8051         if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
8052                 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
8053                                              I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
8054
8055         if (filter->n_proto == ETH_P_IPV6)
8056                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8057                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8058         else
8059                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8060                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8061
8062         if (add)
8063                 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
8064                                                 &cld_filter, 1);
8065         else
8066                 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
8067                                                 &cld_filter, 1);
8068         if (ret)
8069                 dev_dbg(&pf->pdev->dev,
8070                         "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
8071                         add ? "add" : "delete", filter->dst_port, ret,
8072                         pf->hw.aq.asq_last_status);
8073         else
8074                 dev_info(&pf->pdev->dev,
8075                          "%s cloud filter for VSI: %d\n",
8076                          add ? "Added" : "Deleted", filter->seid);
8077         return ret;
8078 }
8079
8080 /**
8081  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8082  * @vsi: pointer to VSI
8083  * @filter: cloud filter rule
8084  * @add: if true, add, if false, delete
8085  *
8086  * Add or delete a cloud filter for a specific flow spec using big buffer.
8087  * Returns 0 if the filter were successfully added.
8088  **/
8089 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
8090                                       struct i40e_cloud_filter *filter,
8091                                       bool add)
8092 {
8093         struct i40e_aqc_cloud_filters_element_bb cld_filter;
8094         struct i40e_pf *pf = vsi->back;
8095         int ret;
8096
8097         /* Both (src/dst) valid mac_addr are not supported */
8098         if ((is_valid_ether_addr(filter->dst_mac) &&
8099              is_valid_ether_addr(filter->src_mac)) ||
8100             (is_multicast_ether_addr(filter->dst_mac) &&
8101              is_multicast_ether_addr(filter->src_mac)))
8102                 return -EOPNOTSUPP;
8103
8104         /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
8105          * ports are not supported via big buffer now.
8106          */
8107         if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
8108                 return -EOPNOTSUPP;
8109
8110         /* adding filter using src_port/src_ip is not supported at this stage */
8111         if (filter->src_port ||
8112             (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8113             !ipv6_addr_any(&filter->ip.v6.src_ip6))
8114                 return -EOPNOTSUPP;
8115
8116         memset(&cld_filter, 0, sizeof(cld_filter));
8117
8118         /* copy element needed to add cloud filter from filter */
8119         i40e_set_cld_element(filter, &cld_filter.element);
8120
8121         if (is_valid_ether_addr(filter->dst_mac) ||
8122             is_valid_ether_addr(filter->src_mac) ||
8123             is_multicast_ether_addr(filter->dst_mac) ||
8124             is_multicast_ether_addr(filter->src_mac)) {
8125                 /* MAC + IP : unsupported mode */
8126                 if (filter->dst_ipv4)
8127                         return -EOPNOTSUPP;
8128
8129                 /* since we validated that L4 port must be valid before
8130                  * we get here, start with respective "flags" value
8131                  * and update if vlan is present or not
8132                  */
8133                 cld_filter.element.flags =
8134                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
8135
8136                 if (filter->vlan_id) {
8137                         cld_filter.element.flags =
8138                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
8139                 }
8140
8141         } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8142                    !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
8143                 cld_filter.element.flags =
8144                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
8145                 if (filter->n_proto == ETH_P_IPV6)
8146                         cld_filter.element.flags |=
8147                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8148                 else
8149                         cld_filter.element.flags |=
8150                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8151         } else {
8152                 dev_err(&pf->pdev->dev,
8153                         "either mac or ip has to be valid for cloud filter\n");
8154                 return -EINVAL;
8155         }
8156
8157         /* Now copy L4 port in Byte 6..7 in general fields */
8158         cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
8159                                                 be16_to_cpu(filter->dst_port);
8160
8161         if (add) {
8162                 /* Validate current device switch mode, change if necessary */
8163                 ret = i40e_validate_and_set_switch_mode(vsi);
8164                 if (ret) {
8165                         dev_err(&pf->pdev->dev,
8166                                 "failed to set switch mode, ret %d\n",
8167                                 ret);
8168                         return ret;
8169                 }
8170
8171                 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
8172                                                    &cld_filter, 1);
8173         } else {
8174                 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
8175                                                    &cld_filter, 1);
8176         }
8177
8178         if (ret)
8179                 dev_dbg(&pf->pdev->dev,
8180                         "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
8181                         add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
8182         else
8183                 dev_info(&pf->pdev->dev,
8184                          "%s cloud filter for VSI: %d, L4 port: %d\n",
8185                          add ? "add" : "delete", filter->seid,
8186                          ntohs(filter->dst_port));
8187         return ret;
8188 }
8189
8190 /**
8191  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8192  * @vsi: Pointer to VSI
8193  * @f: Pointer to struct flow_cls_offload
8194  * @filter: Pointer to cloud filter structure
8195  *
8196  **/
8197 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8198                                  struct flow_cls_offload *f,
8199                                  struct i40e_cloud_filter *filter)
8200 {
8201         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
8202         struct flow_dissector *dissector = rule->match.dissector;
8203         u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8204         struct i40e_pf *pf = vsi->back;
8205         u8 field_flags = 0;
8206
8207         if (dissector->used_keys &
8208             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
8209               BIT(FLOW_DISSECTOR_KEY_BASIC) |
8210               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8211               BIT(FLOW_DISSECTOR_KEY_VLAN) |
8212               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8213               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8214               BIT(FLOW_DISSECTOR_KEY_PORTS) |
8215               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8216                 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
8217                         dissector->used_keys);
8218                 return -EOPNOTSUPP;
8219         }
8220
8221         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8222                 struct flow_match_enc_keyid match;
8223
8224                 flow_rule_match_enc_keyid(rule, &match);
8225                 if (match.mask->keyid != 0)
8226                         field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8227
8228                 filter->tenant_id = be32_to_cpu(match.key->keyid);
8229         }
8230
8231         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
8232                 struct flow_match_basic match;
8233
8234                 flow_rule_match_basic(rule, &match);
8235                 n_proto_key = ntohs(match.key->n_proto);
8236                 n_proto_mask = ntohs(match.mask->n_proto);
8237
8238                 if (n_proto_key == ETH_P_ALL) {
8239                         n_proto_key = 0;
8240                         n_proto_mask = 0;
8241                 }
8242                 filter->n_proto = n_proto_key & n_proto_mask;
8243                 filter->ip_proto = match.key->ip_proto;
8244         }
8245
8246         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8247                 struct flow_match_eth_addrs match;
8248
8249                 flow_rule_match_eth_addrs(rule, &match);
8250
8251                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
8252                 if (!is_zero_ether_addr(match.mask->dst)) {
8253                         if (is_broadcast_ether_addr(match.mask->dst)) {
8254                                 field_flags |= I40E_CLOUD_FIELD_OMAC;
8255                         } else {
8256                                 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8257                                         match.mask->dst);
8258                                 return I40E_ERR_CONFIG;
8259                         }
8260                 }
8261
8262                 if (!is_zero_ether_addr(match.mask->src)) {
8263                         if (is_broadcast_ether_addr(match.mask->src)) {
8264                                 field_flags |= I40E_CLOUD_FIELD_IMAC;
8265                         } else {
8266                                 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8267                                         match.mask->src);
8268                                 return I40E_ERR_CONFIG;
8269                         }
8270                 }
8271                 ether_addr_copy(filter->dst_mac, match.key->dst);
8272                 ether_addr_copy(filter->src_mac, match.key->src);
8273         }
8274
8275         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
8276                 struct flow_match_vlan match;
8277
8278                 flow_rule_match_vlan(rule, &match);
8279                 if (match.mask->vlan_id) {
8280                         if (match.mask->vlan_id == VLAN_VID_MASK) {
8281                                 field_flags |= I40E_CLOUD_FIELD_IVLAN;
8282
8283                         } else {
8284                                 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8285                                         match.mask->vlan_id);
8286                                 return I40E_ERR_CONFIG;
8287                         }
8288                 }
8289
8290                 filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8291         }
8292
8293         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
8294                 struct flow_match_control match;
8295
8296                 flow_rule_match_control(rule, &match);
8297                 addr_type = match.key->addr_type;
8298         }
8299
8300         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8301                 struct flow_match_ipv4_addrs match;
8302
8303                 flow_rule_match_ipv4_addrs(rule, &match);
8304                 if (match.mask->dst) {
8305                         if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8306                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8307                         } else {
8308                                 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8309                                         &match.mask->dst);
8310                                 return I40E_ERR_CONFIG;
8311                         }
8312                 }
8313
8314                 if (match.mask->src) {
8315                         if (match.mask->src == cpu_to_be32(0xffffffff)) {
8316                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8317                         } else {
8318                                 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8319                                         &match.mask->src);
8320                                 return I40E_ERR_CONFIG;
8321                         }
8322                 }
8323
8324                 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8325                         dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8326                         return I40E_ERR_CONFIG;
8327                 }
8328                 filter->dst_ipv4 = match.key->dst;
8329                 filter->src_ipv4 = match.key->src;
8330         }
8331
8332         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8333                 struct flow_match_ipv6_addrs match;
8334
8335                 flow_rule_match_ipv6_addrs(rule, &match);
8336
8337                 /* src and dest IPV6 address should not be LOOPBACK
8338                  * (0:0:0:0:0:0:0:1), which can be represented as ::1
8339                  */
8340                 if (ipv6_addr_loopback(&match.key->dst) ||
8341                     ipv6_addr_loopback(&match.key->src)) {
8342                         dev_err(&pf->pdev->dev,
8343                                 "Bad ipv6, addr is LOOPBACK\n");
8344                         return I40E_ERR_CONFIG;
8345                 }
8346                 if (!ipv6_addr_any(&match.mask->dst) ||
8347                     !ipv6_addr_any(&match.mask->src))
8348                         field_flags |= I40E_CLOUD_FIELD_IIP;
8349
8350                 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8351                        sizeof(filter->src_ipv6));
8352                 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8353                        sizeof(filter->dst_ipv6));
8354         }
8355
8356         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
8357                 struct flow_match_ports match;
8358
8359                 flow_rule_match_ports(rule, &match);
8360                 if (match.mask->src) {
8361                         if (match.mask->src == cpu_to_be16(0xffff)) {
8362                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8363                         } else {
8364                                 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8365                                         be16_to_cpu(match.mask->src));
8366                                 return I40E_ERR_CONFIG;
8367                         }
8368                 }
8369
8370                 if (match.mask->dst) {
8371                         if (match.mask->dst == cpu_to_be16(0xffff)) {
8372                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8373                         } else {
8374                                 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8375                                         be16_to_cpu(match.mask->dst));
8376                                 return I40E_ERR_CONFIG;
8377                         }
8378                 }
8379
8380                 filter->dst_port = match.key->dst;
8381                 filter->src_port = match.key->src;
8382
8383                 switch (filter->ip_proto) {
8384                 case IPPROTO_TCP:
8385                 case IPPROTO_UDP:
8386                         break;
8387                 default:
8388                         dev_err(&pf->pdev->dev,
8389                                 "Only UDP and TCP transport are supported\n");
8390                         return -EINVAL;
8391                 }
8392         }
8393         filter->flags = field_flags;
8394         return 0;
8395 }
8396
8397 /**
8398  * i40e_handle_tclass: Forward to a traffic class on the device
8399  * @vsi: Pointer to VSI
8400  * @tc: traffic class index on the device
8401  * @filter: Pointer to cloud filter structure
8402  *
8403  **/
8404 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8405                               struct i40e_cloud_filter *filter)
8406 {
8407         struct i40e_channel *ch, *ch_tmp;
8408
8409         /* direct to a traffic class on the same device */
8410         if (tc == 0) {
8411                 filter->seid = vsi->seid;
8412                 return 0;
8413         } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8414                 if (!filter->dst_port) {
8415                         dev_err(&vsi->back->pdev->dev,
8416                                 "Specify destination port to direct to traffic class that is not default\n");
8417                         return -EINVAL;
8418                 }
8419                 if (list_empty(&vsi->ch_list))
8420                         return -EINVAL;
8421                 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8422                                          list) {
8423                         if (ch->seid == vsi->tc_seid_map[tc])
8424                                 filter->seid = ch->seid;
8425                 }
8426                 return 0;
8427         }
8428         dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8429         return -EINVAL;
8430 }
8431
8432 /**
8433  * i40e_configure_clsflower - Configure tc flower filters
8434  * @vsi: Pointer to VSI
8435  * @cls_flower: Pointer to struct flow_cls_offload
8436  *
8437  **/
8438 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8439                                     struct flow_cls_offload *cls_flower)
8440 {
8441         int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8442         struct i40e_cloud_filter *filter = NULL;
8443         struct i40e_pf *pf = vsi->back;
8444         int err = 0;
8445
8446         if (tc < 0) {
8447                 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8448                 return -EOPNOTSUPP;
8449         }
8450
8451         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8452             test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8453                 return -EBUSY;
8454
8455         if (pf->fdir_pf_active_filters ||
8456             (!hlist_empty(&pf->fdir_filter_list))) {
8457                 dev_err(&vsi->back->pdev->dev,
8458                         "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8459                 return -EINVAL;
8460         }
8461
8462         if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
8463                 dev_err(&vsi->back->pdev->dev,
8464                         "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8465                 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8466                 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8467         }
8468
8469         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8470         if (!filter)
8471                 return -ENOMEM;
8472
8473         filter->cookie = cls_flower->cookie;
8474
8475         err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8476         if (err < 0)
8477                 goto err;
8478
8479         err = i40e_handle_tclass(vsi, tc, filter);
8480         if (err < 0)
8481                 goto err;
8482
8483         /* Add cloud filter */
8484         if (filter->dst_port)
8485                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8486         else
8487                 err = i40e_add_del_cloud_filter(vsi, filter, true);
8488
8489         if (err) {
8490                 dev_err(&pf->pdev->dev,
8491                         "Failed to add cloud filter, err %s\n",
8492                         i40e_stat_str(&pf->hw, err));
8493                 goto err;
8494         }
8495
8496         /* add filter to the ordered list */
8497         INIT_HLIST_NODE(&filter->cloud_node);
8498
8499         hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8500
8501         pf->num_cloud_filters++;
8502
8503         return err;
8504 err:
8505         kfree(filter);
8506         return err;
8507 }
8508
8509 /**
8510  * i40e_find_cloud_filter - Find the could filter in the list
8511  * @vsi: Pointer to VSI
8512  * @cookie: filter specific cookie
8513  *
8514  **/
8515 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8516                                                         unsigned long *cookie)
8517 {
8518         struct i40e_cloud_filter *filter = NULL;
8519         struct hlist_node *node2;
8520
8521         hlist_for_each_entry_safe(filter, node2,
8522                                   &vsi->back->cloud_filter_list, cloud_node)
8523                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8524                         return filter;
8525         return NULL;
8526 }
8527
8528 /**
8529  * i40e_delete_clsflower - Remove tc flower filters
8530  * @vsi: Pointer to VSI
8531  * @cls_flower: Pointer to struct flow_cls_offload
8532  *
8533  **/
8534 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8535                                  struct flow_cls_offload *cls_flower)
8536 {
8537         struct i40e_cloud_filter *filter = NULL;
8538         struct i40e_pf *pf = vsi->back;
8539         int err = 0;
8540
8541         filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8542
8543         if (!filter)
8544                 return -EINVAL;
8545
8546         hash_del(&filter->cloud_node);
8547
8548         if (filter->dst_port)
8549                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8550         else
8551                 err = i40e_add_del_cloud_filter(vsi, filter, false);
8552
8553         kfree(filter);
8554         if (err) {
8555                 dev_err(&pf->pdev->dev,
8556                         "Failed to delete cloud filter, err %s\n",
8557                         i40e_stat_str(&pf->hw, err));
8558                 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8559         }
8560
8561         pf->num_cloud_filters--;
8562         if (!pf->num_cloud_filters)
8563                 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8564                     !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8565                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8566                         pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8567                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8568                 }
8569         return 0;
8570 }
8571
8572 /**
8573  * i40e_setup_tc_cls_flower - flower classifier offloads
8574  * @np: net device to configure
8575  * @cls_flower: offload data
8576  **/
8577 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8578                                     struct flow_cls_offload *cls_flower)
8579 {
8580         struct i40e_vsi *vsi = np->vsi;
8581
8582         switch (cls_flower->command) {
8583         case FLOW_CLS_REPLACE:
8584                 return i40e_configure_clsflower(vsi, cls_flower);
8585         case FLOW_CLS_DESTROY:
8586                 return i40e_delete_clsflower(vsi, cls_flower);
8587         case FLOW_CLS_STATS:
8588                 return -EOPNOTSUPP;
8589         default:
8590                 return -EOPNOTSUPP;
8591         }
8592 }
8593
8594 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8595                                   void *cb_priv)
8596 {
8597         struct i40e_netdev_priv *np = cb_priv;
8598
8599         if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8600                 return -EOPNOTSUPP;
8601
8602         switch (type) {
8603         case TC_SETUP_CLSFLOWER:
8604                 return i40e_setup_tc_cls_flower(np, type_data);
8605
8606         default:
8607                 return -EOPNOTSUPP;
8608         }
8609 }
8610
8611 static LIST_HEAD(i40e_block_cb_list);
8612
8613 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8614                            void *type_data)
8615 {
8616         struct i40e_netdev_priv *np = netdev_priv(netdev);
8617
8618         switch (type) {
8619         case TC_SETUP_QDISC_MQPRIO:
8620                 return i40e_setup_tc(netdev, type_data);
8621         case TC_SETUP_BLOCK:
8622                 return flow_block_cb_setup_simple(type_data,
8623                                                   &i40e_block_cb_list,
8624                                                   i40e_setup_tc_block_cb,
8625                                                   np, np, true);
8626         default:
8627                 return -EOPNOTSUPP;
8628         }
8629 }
8630
8631 /**
8632  * i40e_open - Called when a network interface is made active
8633  * @netdev: network interface device structure
8634  *
8635  * The open entry point is called when a network interface is made
8636  * active by the system (IFF_UP).  At this point all resources needed
8637  * for transmit and receive operations are allocated, the interrupt
8638  * handler is registered with the OS, the netdev watchdog subtask is
8639  * enabled, and the stack is notified that the interface is ready.
8640  *
8641  * Returns 0 on success, negative value on failure
8642  **/
8643 int i40e_open(struct net_device *netdev)
8644 {
8645         struct i40e_netdev_priv *np = netdev_priv(netdev);
8646         struct i40e_vsi *vsi = np->vsi;
8647         struct i40e_pf *pf = vsi->back;
8648         int err;
8649
8650         /* disallow open during test or if eeprom is broken */
8651         if (test_bit(__I40E_TESTING, pf->state) ||
8652             test_bit(__I40E_BAD_EEPROM, pf->state))
8653                 return -EBUSY;
8654
8655         netif_carrier_off(netdev);
8656
8657         if (i40e_force_link_state(pf, true))
8658                 return -EAGAIN;
8659
8660         err = i40e_vsi_open(vsi);
8661         if (err)
8662                 return err;
8663
8664         /* configure global TSO hardware offload settings */
8665         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
8666                                                        TCP_FLAG_FIN) >> 16);
8667         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
8668                                                        TCP_FLAG_FIN |
8669                                                        TCP_FLAG_CWR) >> 16);
8670         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
8671         udp_tunnel_get_rx_info(netdev);
8672
8673         return 0;
8674 }
8675
8676 /**
8677  * i40e_vsi_open -
8678  * @vsi: the VSI to open
8679  *
8680  * Finish initialization of the VSI.
8681  *
8682  * Returns 0 on success, negative value on failure
8683  *
8684  * Note: expects to be called while under rtnl_lock()
8685  **/
8686 int i40e_vsi_open(struct i40e_vsi *vsi)
8687 {
8688         struct i40e_pf *pf = vsi->back;
8689         char int_name[I40E_INT_NAME_STR_LEN];
8690         int err;
8691
8692         /* allocate descriptors */
8693         err = i40e_vsi_setup_tx_resources(vsi);
8694         if (err)
8695                 goto err_setup_tx;
8696         err = i40e_vsi_setup_rx_resources(vsi);
8697         if (err)
8698                 goto err_setup_rx;
8699
8700         err = i40e_vsi_configure(vsi);
8701         if (err)
8702                 goto err_setup_rx;
8703
8704         if (vsi->netdev) {
8705                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
8706                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
8707                 err = i40e_vsi_request_irq(vsi, int_name);
8708                 if (err)
8709                         goto err_setup_rx;
8710
8711                 /* Notify the stack of the actual queue counts. */
8712                 err = netif_set_real_num_tx_queues(vsi->netdev,
8713                                                    vsi->num_queue_pairs);
8714                 if (err)
8715                         goto err_set_queues;
8716
8717                 err = netif_set_real_num_rx_queues(vsi->netdev,
8718                                                    vsi->num_queue_pairs);
8719                 if (err)
8720                         goto err_set_queues;
8721
8722         } else if (vsi->type == I40E_VSI_FDIR) {
8723                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
8724                          dev_driver_string(&pf->pdev->dev),
8725                          dev_name(&pf->pdev->dev));
8726                 err = i40e_vsi_request_irq(vsi, int_name);
8727
8728         } else {
8729                 err = -EINVAL;
8730                 goto err_setup_rx;
8731         }
8732
8733         err = i40e_up_complete(vsi);
8734         if (err)
8735                 goto err_up_complete;
8736
8737         return 0;
8738
8739 err_up_complete:
8740         i40e_down(vsi);
8741 err_set_queues:
8742         i40e_vsi_free_irq(vsi);
8743 err_setup_rx:
8744         i40e_vsi_free_rx_resources(vsi);
8745 err_setup_tx:
8746         i40e_vsi_free_tx_resources(vsi);
8747         if (vsi == pf->vsi[pf->lan_vsi])
8748                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
8749
8750         return err;
8751 }
8752
8753 /**
8754  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
8755  * @pf: Pointer to PF
8756  *
8757  * This function destroys the hlist where all the Flow Director
8758  * filters were saved.
8759  **/
8760 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
8761 {
8762         struct i40e_fdir_filter *filter;
8763         struct i40e_flex_pit *pit_entry, *tmp;
8764         struct hlist_node *node2;
8765
8766         hlist_for_each_entry_safe(filter, node2,
8767                                   &pf->fdir_filter_list, fdir_node) {
8768                 hlist_del(&filter->fdir_node);
8769                 kfree(filter);
8770         }
8771
8772         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
8773                 list_del(&pit_entry->list);
8774                 kfree(pit_entry);
8775         }
8776         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
8777
8778         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
8779                 list_del(&pit_entry->list);
8780                 kfree(pit_entry);
8781         }
8782         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
8783
8784         pf->fdir_pf_active_filters = 0;
8785         i40e_reset_fdir_filter_cnt(pf);
8786
8787         /* Reprogram the default input set for TCP/IPv4 */
8788         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8789                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8790                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8791
8792         /* Reprogram the default input set for TCP/IPv6 */
8793         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
8794                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
8795                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8796
8797         /* Reprogram the default input set for UDP/IPv4 */
8798         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
8799                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8800                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8801
8802         /* Reprogram the default input set for UDP/IPv6 */
8803         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
8804                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
8805                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8806
8807         /* Reprogram the default input set for SCTP/IPv4 */
8808         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
8809                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8810                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8811
8812         /* Reprogram the default input set for SCTP/IPv6 */
8813         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
8814                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
8815                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8816
8817         /* Reprogram the default input set for Other/IPv4 */
8818         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
8819                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8820
8821         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
8822                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8823
8824         /* Reprogram the default input set for Other/IPv6 */
8825         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
8826                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8827
8828         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6,
8829                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8830 }
8831
8832 /**
8833  * i40e_cloud_filter_exit - Cleans up the cloud filters
8834  * @pf: Pointer to PF
8835  *
8836  * This function destroys the hlist where all the cloud filters
8837  * were saved.
8838  **/
8839 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
8840 {
8841         struct i40e_cloud_filter *cfilter;
8842         struct hlist_node *node;
8843
8844         hlist_for_each_entry_safe(cfilter, node,
8845                                   &pf->cloud_filter_list, cloud_node) {
8846                 hlist_del(&cfilter->cloud_node);
8847                 kfree(cfilter);
8848         }
8849         pf->num_cloud_filters = 0;
8850
8851         if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8852             !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8853                 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8854                 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8855                 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8856         }
8857 }
8858
8859 /**
8860  * i40e_close - Disables a network interface
8861  * @netdev: network interface device structure
8862  *
8863  * The close entry point is called when an interface is de-activated
8864  * by the OS.  The hardware is still under the driver's control, but
8865  * this netdev interface is disabled.
8866  *
8867  * Returns 0, this is not allowed to fail
8868  **/
8869 int i40e_close(struct net_device *netdev)
8870 {
8871         struct i40e_netdev_priv *np = netdev_priv(netdev);
8872         struct i40e_vsi *vsi = np->vsi;
8873
8874         i40e_vsi_close(vsi);
8875
8876         return 0;
8877 }
8878
8879 /**
8880  * i40e_do_reset - Start a PF or Core Reset sequence
8881  * @pf: board private structure
8882  * @reset_flags: which reset is requested
8883  * @lock_acquired: indicates whether or not the lock has been acquired
8884  * before this function was called.
8885  *
8886  * The essential difference in resets is that the PF Reset
8887  * doesn't clear the packet buffers, doesn't reset the PE
8888  * firmware, and doesn't bother the other PFs on the chip.
8889  **/
8890 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
8891 {
8892         u32 val;
8893
8894         /* do the biggest reset indicated */
8895         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
8896
8897                 /* Request a Global Reset
8898                  *
8899                  * This will start the chip's countdown to the actual full
8900                  * chip reset event, and a warning interrupt to be sent
8901                  * to all PFs, including the requestor.  Our handler
8902                  * for the warning interrupt will deal with the shutdown
8903                  * and recovery of the switch setup.
8904                  */
8905                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
8906                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8907                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
8908                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8909
8910         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
8911
8912                 /* Request a Core Reset
8913                  *
8914                  * Same as Global Reset, except does *not* include the MAC/PHY
8915                  */
8916                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
8917                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8918                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
8919                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8920                 i40e_flush(&pf->hw);
8921
8922         } else if (reset_flags & I40E_PF_RESET_FLAG) {
8923
8924                 /* Request a PF Reset
8925                  *
8926                  * Resets only the PF-specific registers
8927                  *
8928                  * This goes directly to the tear-down and rebuild of
8929                  * the switch, since we need to do all the recovery as
8930                  * for the Core Reset.
8931                  */
8932                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
8933                 i40e_handle_reset_warning(pf, lock_acquired);
8934
8935         } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
8936                 /* Request a PF Reset
8937                  *
8938                  * Resets PF and reinitializes PFs VSI.
8939                  */
8940                 i40e_prep_for_reset(pf);
8941                 i40e_reset_and_rebuild(pf, true, lock_acquired);
8942                 dev_info(&pf->pdev->dev,
8943                          pf->flags & I40E_FLAG_DISABLE_FW_LLDP ?
8944                          "FW LLDP is disabled\n" :
8945                          "FW LLDP is enabled\n");
8946
8947         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
8948                 int v;
8949
8950                 /* Find the VSI(s) that requested a re-init */
8951                 dev_info(&pf->pdev->dev,
8952                          "VSI reinit requested\n");
8953                 for (v = 0; v < pf->num_alloc_vsi; v++) {
8954                         struct i40e_vsi *vsi = pf->vsi[v];
8955
8956                         if (vsi != NULL &&
8957                             test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
8958                                                vsi->state))
8959                                 i40e_vsi_reinit_locked(pf->vsi[v]);
8960                 }
8961         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
8962                 int v;
8963
8964                 /* Find the VSI(s) that needs to be brought down */
8965                 dev_info(&pf->pdev->dev, "VSI down requested\n");
8966                 for (v = 0; v < pf->num_alloc_vsi; v++) {
8967                         struct i40e_vsi *vsi = pf->vsi[v];
8968
8969                         if (vsi != NULL &&
8970                             test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
8971                                                vsi->state)) {
8972                                 set_bit(__I40E_VSI_DOWN, vsi->state);
8973                                 i40e_down(vsi);
8974                         }
8975                 }
8976         } else {
8977                 dev_info(&pf->pdev->dev,
8978                          "bad reset request 0x%08x\n", reset_flags);
8979         }
8980 }
8981
8982 #ifdef CONFIG_I40E_DCB
8983 /**
8984  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
8985  * @pf: board private structure
8986  * @old_cfg: current DCB config
8987  * @new_cfg: new DCB config
8988  **/
8989 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
8990                             struct i40e_dcbx_config *old_cfg,
8991                             struct i40e_dcbx_config *new_cfg)
8992 {
8993         bool need_reconfig = false;
8994
8995         /* Check if ETS configuration has changed */
8996         if (memcmp(&new_cfg->etscfg,
8997                    &old_cfg->etscfg,
8998                    sizeof(new_cfg->etscfg))) {
8999                 /* If Priority Table has changed reconfig is needed */
9000                 if (memcmp(&new_cfg->etscfg.prioritytable,
9001                            &old_cfg->etscfg.prioritytable,
9002                            sizeof(new_cfg->etscfg.prioritytable))) {
9003                         need_reconfig = true;
9004                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
9005                 }
9006
9007                 if (memcmp(&new_cfg->etscfg.tcbwtable,
9008                            &old_cfg->etscfg.tcbwtable,
9009                            sizeof(new_cfg->etscfg.tcbwtable)))
9010                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
9011
9012                 if (memcmp(&new_cfg->etscfg.tsatable,
9013                            &old_cfg->etscfg.tsatable,
9014                            sizeof(new_cfg->etscfg.tsatable)))
9015                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
9016         }
9017
9018         /* Check if PFC configuration has changed */
9019         if (memcmp(&new_cfg->pfc,
9020                    &old_cfg->pfc,
9021                    sizeof(new_cfg->pfc))) {
9022                 need_reconfig = true;
9023                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
9024         }
9025
9026         /* Check if APP Table has changed */
9027         if (memcmp(&new_cfg->app,
9028                    &old_cfg->app,
9029                    sizeof(new_cfg->app))) {
9030                 need_reconfig = true;
9031                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
9032         }
9033
9034         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
9035         return need_reconfig;
9036 }
9037
9038 /**
9039  * i40e_handle_lldp_event - Handle LLDP Change MIB event
9040  * @pf: board private structure
9041  * @e: event info posted on ARQ
9042  **/
9043 static int i40e_handle_lldp_event(struct i40e_pf *pf,
9044                                   struct i40e_arq_event_info *e)
9045 {
9046         struct i40e_aqc_lldp_get_mib *mib =
9047                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
9048         struct i40e_hw *hw = &pf->hw;
9049         struct i40e_dcbx_config tmp_dcbx_cfg;
9050         bool need_reconfig = false;
9051         int ret = 0;
9052         u8 type;
9053
9054         /* X710-T*L 2.5G and 5G speeds don't support DCB */
9055         if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9056             (hw->phy.link_info.link_speed &
9057              ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
9058              !(pf->flags & I40E_FLAG_DCB_CAPABLE))
9059                 /* let firmware decide if the DCB should be disabled */
9060                 pf->flags |= I40E_FLAG_DCB_CAPABLE;
9061
9062         /* Not DCB capable or capability disabled */
9063         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
9064                 return ret;
9065
9066         /* Ignore if event is not for Nearest Bridge */
9067         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
9068                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
9069         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
9070         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
9071                 return ret;
9072
9073         /* Check MIB Type and return if event for Remote MIB update */
9074         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
9075         dev_dbg(&pf->pdev->dev,
9076                 "LLDP event mib type %s\n", type ? "remote" : "local");
9077         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
9078                 /* Update the remote cached instance and return */
9079                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
9080                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
9081                                 &hw->remote_dcbx_config);
9082                 goto exit;
9083         }
9084
9085         /* Store the old configuration */
9086         tmp_dcbx_cfg = hw->local_dcbx_config;
9087
9088         /* Reset the old DCBx configuration data */
9089         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
9090         /* Get updated DCBX data from firmware */
9091         ret = i40e_get_dcb_config(&pf->hw);
9092         if (ret) {
9093                 /* X710-T*L 2.5G and 5G speeds don't support DCB */
9094                 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9095                     (hw->phy.link_info.link_speed &
9096                      (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
9097                         dev_warn(&pf->pdev->dev,
9098                                  "DCB is not supported for X710-T*L 2.5/5G speeds\n");
9099                         pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
9100                 } else {
9101                         dev_info(&pf->pdev->dev,
9102                                  "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
9103                                  i40e_stat_str(&pf->hw, ret),
9104                                  i40e_aq_str(&pf->hw,
9105                                              pf->hw.aq.asq_last_status));
9106                 }
9107                 goto exit;
9108         }
9109
9110         /* No change detected in DCBX configs */
9111         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
9112                     sizeof(tmp_dcbx_cfg))) {
9113                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
9114                 goto exit;
9115         }
9116
9117         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
9118                                                &hw->local_dcbx_config);
9119
9120         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
9121
9122         if (!need_reconfig)
9123                 goto exit;
9124
9125         /* Enable DCB tagging only when more than one TC */
9126         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
9127                 pf->flags |= I40E_FLAG_DCB_ENABLED;
9128         else
9129                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
9130
9131         set_bit(__I40E_PORT_SUSPENDED, pf->state);
9132         /* Reconfiguration needed quiesce all VSIs */
9133         i40e_pf_quiesce_all_vsi(pf);
9134
9135         /* Changes in configuration update VEB/VSI */
9136         i40e_dcb_reconfigure(pf);
9137
9138         ret = i40e_resume_port_tx(pf);
9139
9140         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
9141         /* In case of error no point in resuming VSIs */
9142         if (ret)
9143                 goto exit;
9144
9145         /* Wait for the PF's queues to be disabled */
9146         ret = i40e_pf_wait_queues_disabled(pf);
9147         if (ret) {
9148                 /* Schedule PF reset to recover */
9149                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9150                 i40e_service_event_schedule(pf);
9151         } else {
9152                 i40e_pf_unquiesce_all_vsi(pf);
9153                 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
9154                 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
9155         }
9156
9157 exit:
9158         return ret;
9159 }
9160 #endif /* CONFIG_I40E_DCB */
9161
9162 /**
9163  * i40e_do_reset_safe - Protected reset path for userland calls.
9164  * @pf: board private structure
9165  * @reset_flags: which reset is requested
9166  *
9167  **/
9168 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
9169 {
9170         rtnl_lock();
9171         i40e_do_reset(pf, reset_flags, true);
9172         rtnl_unlock();
9173 }
9174
9175 /**
9176  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9177  * @pf: board private structure
9178  * @e: event info posted on ARQ
9179  *
9180  * Handler for LAN Queue Overflow Event generated by the firmware for PF
9181  * and VF queues
9182  **/
9183 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
9184                                            struct i40e_arq_event_info *e)
9185 {
9186         struct i40e_aqc_lan_overflow *data =
9187                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
9188         u32 queue = le32_to_cpu(data->prtdcb_rupto);
9189         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
9190         struct i40e_hw *hw = &pf->hw;
9191         struct i40e_vf *vf;
9192         u16 vf_id;
9193
9194         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
9195                 queue, qtx_ctl);
9196
9197         /* Queue belongs to VF, find the VF and issue VF reset */
9198         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
9199             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
9200                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
9201                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
9202                 vf_id -= hw->func_caps.vf_base_id;
9203                 vf = &pf->vf[vf_id];
9204                 i40e_vc_notify_vf_reset(vf);
9205                 /* Allow VF to process pending reset notification */
9206                 msleep(20);
9207                 i40e_reset_vf(vf, false);
9208         }
9209 }
9210
9211 /**
9212  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9213  * @pf: board private structure
9214  **/
9215 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
9216 {
9217         u32 val, fcnt_prog;
9218
9219         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9220         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
9221         return fcnt_prog;
9222 }
9223
9224 /**
9225  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9226  * @pf: board private structure
9227  **/
9228 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9229 {
9230         u32 val, fcnt_prog;
9231
9232         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9233         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9234                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
9235                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
9236         return fcnt_prog;
9237 }
9238
9239 /**
9240  * i40e_get_global_fd_count - Get total FD filters programmed on device
9241  * @pf: board private structure
9242  **/
9243 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9244 {
9245         u32 val, fcnt_prog;
9246
9247         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9248         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9249                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
9250                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
9251         return fcnt_prog;
9252 }
9253
9254 /**
9255  * i40e_reenable_fdir_sb - Restore FDir SB capability
9256  * @pf: board private structure
9257  **/
9258 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9259 {
9260         if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
9261                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
9262                     (I40E_DEBUG_FD & pf->hw.debug_mask))
9263                         dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9264 }
9265
9266 /**
9267  * i40e_reenable_fdir_atr - Restore FDir ATR capability
9268  * @pf: board private structure
9269  **/
9270 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9271 {
9272         if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
9273                 /* ATR uses the same filtering logic as SB rules. It only
9274                  * functions properly if the input set mask is at the default
9275                  * settings. It is safe to restore the default input set
9276                  * because there are no active TCPv4 filter rules.
9277                  */
9278                 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9279                                         I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9280                                         I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9281
9282                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
9283                     (I40E_DEBUG_FD & pf->hw.debug_mask))
9284                         dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9285         }
9286 }
9287
9288 /**
9289  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9290  * @pf: board private structure
9291  * @filter: FDir filter to remove
9292  */
9293 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9294                                        struct i40e_fdir_filter *filter)
9295 {
9296         /* Update counters */
9297         pf->fdir_pf_active_filters--;
9298         pf->fd_inv = 0;
9299
9300         switch (filter->flow_type) {
9301         case TCP_V4_FLOW:
9302                 pf->fd_tcp4_filter_cnt--;
9303                 break;
9304         case UDP_V4_FLOW:
9305                 pf->fd_udp4_filter_cnt--;
9306                 break;
9307         case SCTP_V4_FLOW:
9308                 pf->fd_sctp4_filter_cnt--;
9309                 break;
9310         case TCP_V6_FLOW:
9311                 pf->fd_tcp6_filter_cnt--;
9312                 break;
9313         case UDP_V6_FLOW:
9314                 pf->fd_udp6_filter_cnt--;
9315                 break;
9316         case SCTP_V6_FLOW:
9317                 pf->fd_udp6_filter_cnt--;
9318                 break;
9319         case IP_USER_FLOW:
9320                 switch (filter->ipl4_proto) {
9321                 case IPPROTO_TCP:
9322                         pf->fd_tcp4_filter_cnt--;
9323                         break;
9324                 case IPPROTO_UDP:
9325                         pf->fd_udp4_filter_cnt--;
9326                         break;
9327                 case IPPROTO_SCTP:
9328                         pf->fd_sctp4_filter_cnt--;
9329                         break;
9330                 case IPPROTO_IP:
9331                         pf->fd_ip4_filter_cnt--;
9332                         break;
9333                 }
9334                 break;
9335         case IPV6_USER_FLOW:
9336                 switch (filter->ipl4_proto) {
9337                 case IPPROTO_TCP:
9338                         pf->fd_tcp6_filter_cnt--;
9339                         break;
9340                 case IPPROTO_UDP:
9341                         pf->fd_udp6_filter_cnt--;
9342                         break;
9343                 case IPPROTO_SCTP:
9344                         pf->fd_sctp6_filter_cnt--;
9345                         break;
9346                 case IPPROTO_IP:
9347                         pf->fd_ip6_filter_cnt--;
9348                         break;
9349                 }
9350                 break;
9351         }
9352
9353         /* Remove the filter from the list and free memory */
9354         hlist_del(&filter->fdir_node);
9355         kfree(filter);
9356 }
9357
9358 /**
9359  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9360  * @pf: board private structure
9361  **/
9362 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9363 {
9364         struct i40e_fdir_filter *filter;
9365         u32 fcnt_prog, fcnt_avail;
9366         struct hlist_node *node;
9367
9368         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9369                 return;
9370
9371         /* Check if we have enough room to re-enable FDir SB capability. */
9372         fcnt_prog = i40e_get_global_fd_count(pf);
9373         fcnt_avail = pf->fdir_pf_filter_count;
9374         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9375             (pf->fd_add_err == 0) ||
9376             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9377                 i40e_reenable_fdir_sb(pf);
9378
9379         /* We should wait for even more space before re-enabling ATR.
9380          * Additionally, we cannot enable ATR as long as we still have TCP SB
9381          * rules active.
9382          */
9383         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9384             pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)
9385                 i40e_reenable_fdir_atr(pf);
9386
9387         /* if hw had a problem adding a filter, delete it */
9388         if (pf->fd_inv > 0) {
9389                 hlist_for_each_entry_safe(filter, node,
9390                                           &pf->fdir_filter_list, fdir_node)
9391                         if (filter->fd_id == pf->fd_inv)
9392                                 i40e_delete_invalid_filter(pf, filter);
9393         }
9394 }
9395
9396 #define I40E_MIN_FD_FLUSH_INTERVAL 10
9397 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9398 /**
9399  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9400  * @pf: board private structure
9401  **/
9402 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9403 {
9404         unsigned long min_flush_time;
9405         int flush_wait_retry = 50;
9406         bool disable_atr = false;
9407         int fd_room;
9408         int reg;
9409
9410         if (!time_after(jiffies, pf->fd_flush_timestamp +
9411                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9412                 return;
9413
9414         /* If the flush is happening too quick and we have mostly SB rules we
9415          * should not re-enable ATR for some time.
9416          */
9417         min_flush_time = pf->fd_flush_timestamp +
9418                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9419         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9420
9421         if (!(time_after(jiffies, min_flush_time)) &&
9422             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9423                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9424                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9425                 disable_atr = true;
9426         }
9427
9428         pf->fd_flush_timestamp = jiffies;
9429         set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9430         /* flush all filters */
9431         wr32(&pf->hw, I40E_PFQF_CTL_1,
9432              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9433         i40e_flush(&pf->hw);
9434         pf->fd_flush_cnt++;
9435         pf->fd_add_err = 0;
9436         do {
9437                 /* Check FD flush status every 5-6msec */
9438                 usleep_range(5000, 6000);
9439                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9440                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9441                         break;
9442         } while (flush_wait_retry--);
9443         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9444                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9445         } else {
9446                 /* replay sideband filters */
9447                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
9448                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9449                         clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9450                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
9451                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9452                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9453         }
9454 }
9455
9456 /**
9457  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
9458  * @pf: board private structure
9459  **/
9460 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9461 {
9462         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9463 }
9464
9465 /**
9466  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9467  * @pf: board private structure
9468  **/
9469 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9470 {
9471
9472         /* if interface is down do nothing */
9473         if (test_bit(__I40E_DOWN, pf->state))
9474                 return;
9475
9476         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9477                 i40e_fdir_flush_and_replay(pf);
9478
9479         i40e_fdir_check_and_reenable(pf);
9480
9481 }
9482
9483 /**
9484  * i40e_vsi_link_event - notify VSI of a link event
9485  * @vsi: vsi to be notified
9486  * @link_up: link up or down
9487  **/
9488 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9489 {
9490         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9491                 return;
9492
9493         switch (vsi->type) {
9494         case I40E_VSI_MAIN:
9495                 if (!vsi->netdev || !vsi->netdev_registered)
9496                         break;
9497
9498                 if (link_up) {
9499                         netif_carrier_on(vsi->netdev);
9500                         netif_tx_wake_all_queues(vsi->netdev);
9501                 } else {
9502                         netif_carrier_off(vsi->netdev);
9503                         netif_tx_stop_all_queues(vsi->netdev);
9504                 }
9505                 break;
9506
9507         case I40E_VSI_SRIOV:
9508         case I40E_VSI_VMDQ2:
9509         case I40E_VSI_CTRL:
9510         case I40E_VSI_IWARP:
9511         case I40E_VSI_MIRROR:
9512         default:
9513                 /* there is no notification for other VSIs */
9514                 break;
9515         }
9516 }
9517
9518 /**
9519  * i40e_veb_link_event - notify elements on the veb of a link event
9520  * @veb: veb to be notified
9521  * @link_up: link up or down
9522  **/
9523 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9524 {
9525         struct i40e_pf *pf;
9526         int i;
9527
9528         if (!veb || !veb->pf)
9529                 return;
9530         pf = veb->pf;
9531
9532         /* depth first... */
9533         for (i = 0; i < I40E_MAX_VEB; i++)
9534                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
9535                         i40e_veb_link_event(pf->veb[i], link_up);
9536
9537         /* ... now the local VSIs */
9538         for (i = 0; i < pf->num_alloc_vsi; i++)
9539                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
9540                         i40e_vsi_link_event(pf->vsi[i], link_up);
9541 }
9542
9543 /**
9544  * i40e_link_event - Update netif_carrier status
9545  * @pf: board private structure
9546  **/
9547 static void i40e_link_event(struct i40e_pf *pf)
9548 {
9549         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9550         u8 new_link_speed, old_link_speed;
9551         i40e_status status;
9552         bool new_link, old_link;
9553 #ifdef CONFIG_I40E_DCB
9554         int err;
9555 #endif /* CONFIG_I40E_DCB */
9556
9557         /* set this to force the get_link_status call to refresh state */
9558         pf->hw.phy.get_link_info = true;
9559         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9560         status = i40e_get_link_status(&pf->hw, &new_link);
9561
9562         /* On success, disable temp link polling */
9563         if (status == I40E_SUCCESS) {
9564                 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9565         } else {
9566                 /* Enable link polling temporarily until i40e_get_link_status
9567                  * returns I40E_SUCCESS
9568                  */
9569                 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9570                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9571                         status);
9572                 return;
9573         }
9574
9575         old_link_speed = pf->hw.phy.link_info_old.link_speed;
9576         new_link_speed = pf->hw.phy.link_info.link_speed;
9577
9578         if (new_link == old_link &&
9579             new_link_speed == old_link_speed &&
9580             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9581              new_link == netif_carrier_ok(vsi->netdev)))
9582                 return;
9583
9584         i40e_print_link_message(vsi, new_link);
9585
9586         /* Notify the base of the switch tree connected to
9587          * the link.  Floating VEBs are not notified.
9588          */
9589         if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
9590                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
9591         else
9592                 i40e_vsi_link_event(vsi, new_link);
9593
9594         if (pf->vf)
9595                 i40e_vc_notify_link_state(pf);
9596
9597         if (pf->flags & I40E_FLAG_PTP)
9598                 i40e_ptp_set_increment(pf);
9599 #ifdef CONFIG_I40E_DCB
9600         if (new_link == old_link)
9601                 return;
9602         /* Not SW DCB so firmware will take care of default settings */
9603         if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
9604                 return;
9605
9606         /* We cover here only link down, as after link up in case of SW DCB
9607          * SW LLDP agent will take care of setting it up
9608          */
9609         if (!new_link) {
9610                 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");
9611                 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
9612                 err = i40e_dcb_sw_default_config(pf);
9613                 if (err) {
9614                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
9615                                        I40E_FLAG_DCB_ENABLED);
9616                 } else {
9617                         pf->dcbx_cap = DCB_CAP_DCBX_HOST |
9618                                        DCB_CAP_DCBX_VER_IEEE;
9619                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
9620                         pf->flags &= ~I40E_FLAG_DCB_ENABLED;
9621                 }
9622         }
9623 #endif /* CONFIG_I40E_DCB */
9624 }
9625
9626 /**
9627  * i40e_watchdog_subtask - periodic checks not using event driven response
9628  * @pf: board private structure
9629  **/
9630 static void i40e_watchdog_subtask(struct i40e_pf *pf)
9631 {
9632         int i;
9633
9634         /* if interface is down do nothing */
9635         if (test_bit(__I40E_DOWN, pf->state) ||
9636             test_bit(__I40E_CONFIG_BUSY, pf->state))
9637                 return;
9638
9639         /* make sure we don't do these things too often */
9640         if (time_before(jiffies, (pf->service_timer_previous +
9641                                   pf->service_timer_period)))
9642                 return;
9643         pf->service_timer_previous = jiffies;
9644
9645         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
9646             test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
9647                 i40e_link_event(pf);
9648
9649         /* Update the stats for active netdevs so the network stack
9650          * can look at updated numbers whenever it cares to
9651          */
9652         for (i = 0; i < pf->num_alloc_vsi; i++)
9653                 if (pf->vsi[i] && pf->vsi[i]->netdev)
9654                         i40e_update_stats(pf->vsi[i]);
9655
9656         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
9657                 /* Update the stats for the active switching components */
9658                 for (i = 0; i < I40E_MAX_VEB; i++)
9659                         if (pf->veb[i])
9660                                 i40e_update_veb_stats(pf->veb[i]);
9661         }
9662
9663         i40e_ptp_rx_hang(pf);
9664         i40e_ptp_tx_hang(pf);
9665 }
9666
9667 /**
9668  * i40e_reset_subtask - Set up for resetting the device and driver
9669  * @pf: board private structure
9670  **/
9671 static void i40e_reset_subtask(struct i40e_pf *pf)
9672 {
9673         u32 reset_flags = 0;
9674
9675         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
9676                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
9677                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
9678         }
9679         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
9680                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
9681                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9682         }
9683         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
9684                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
9685                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
9686         }
9687         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
9688                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
9689                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
9690         }
9691         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
9692                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
9693                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
9694         }
9695
9696         /* If there's a recovery already waiting, it takes
9697          * precedence before starting a new reset sequence.
9698          */
9699         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
9700                 i40e_prep_for_reset(pf);
9701                 i40e_reset(pf);
9702                 i40e_rebuild(pf, false, false);
9703         }
9704
9705         /* If we're already down or resetting, just bail */
9706         if (reset_flags &&
9707             !test_bit(__I40E_DOWN, pf->state) &&
9708             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
9709                 i40e_do_reset(pf, reset_flags, false);
9710         }
9711 }
9712
9713 /**
9714  * i40e_handle_link_event - Handle link event
9715  * @pf: board private structure
9716  * @e: event info posted on ARQ
9717  **/
9718 static void i40e_handle_link_event(struct i40e_pf *pf,
9719                                    struct i40e_arq_event_info *e)
9720 {
9721         struct i40e_aqc_get_link_status *status =
9722                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
9723
9724         /* Do a new status request to re-enable LSE reporting
9725          * and load new status information into the hw struct
9726          * This completely ignores any state information
9727          * in the ARQ event info, instead choosing to always
9728          * issue the AQ update link status command.
9729          */
9730         i40e_link_event(pf);
9731
9732         /* Check if module meets thermal requirements */
9733         if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
9734                 dev_err(&pf->pdev->dev,
9735                         "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
9736                 dev_err(&pf->pdev->dev,
9737                         "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9738         } else {
9739                 /* check for unqualified module, if link is down, suppress
9740                  * the message if link was forced to be down.
9741                  */
9742                 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
9743                     (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
9744                     (!(status->link_info & I40E_AQ_LINK_UP)) &&
9745                     (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
9746                         dev_err(&pf->pdev->dev,
9747                                 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
9748                         dev_err(&pf->pdev->dev,
9749                                 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9750                 }
9751         }
9752 }
9753
9754 /**
9755  * i40e_clean_adminq_subtask - Clean the AdminQ rings
9756  * @pf: board private structure
9757  **/
9758 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
9759 {
9760         struct i40e_arq_event_info event;
9761         struct i40e_hw *hw = &pf->hw;
9762         u16 pending, i = 0;
9763         i40e_status ret;
9764         u16 opcode;
9765         u32 oldval;
9766         u32 val;
9767
9768         /* Do not run clean AQ when PF reset fails */
9769         if (test_bit(__I40E_RESET_FAILED, pf->state))
9770                 return;
9771
9772         /* check for error indications */
9773         val = rd32(&pf->hw, pf->hw.aq.arq.len);
9774         oldval = val;
9775         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
9776                 if (hw->debug_mask & I40E_DEBUG_AQ)
9777                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
9778                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
9779         }
9780         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
9781                 if (hw->debug_mask & I40E_DEBUG_AQ)
9782                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
9783                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
9784                 pf->arq_overflows++;
9785         }
9786         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
9787                 if (hw->debug_mask & I40E_DEBUG_AQ)
9788                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
9789                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
9790         }
9791         if (oldval != val)
9792                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
9793
9794         val = rd32(&pf->hw, pf->hw.aq.asq.len);
9795         oldval = val;
9796         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
9797                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9798                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
9799                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
9800         }
9801         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
9802                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9803                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
9804                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
9805         }
9806         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
9807                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9808                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
9809                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
9810         }
9811         if (oldval != val)
9812                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
9813
9814         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
9815         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
9816         if (!event.msg_buf)
9817                 return;
9818
9819         do {
9820                 ret = i40e_clean_arq_element(hw, &event, &pending);
9821                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
9822                         break;
9823                 else if (ret) {
9824                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
9825                         break;
9826                 }
9827
9828                 opcode = le16_to_cpu(event.desc.opcode);
9829                 switch (opcode) {
9830
9831                 case i40e_aqc_opc_get_link_status:
9832                         rtnl_lock();
9833                         i40e_handle_link_event(pf, &event);
9834                         rtnl_unlock();
9835                         break;
9836                 case i40e_aqc_opc_send_msg_to_pf:
9837                         ret = i40e_vc_process_vf_msg(pf,
9838                                         le16_to_cpu(event.desc.retval),
9839                                         le32_to_cpu(event.desc.cookie_high),
9840                                         le32_to_cpu(event.desc.cookie_low),
9841                                         event.msg_buf,
9842                                         event.msg_len);
9843                         break;
9844                 case i40e_aqc_opc_lldp_update_mib:
9845                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
9846 #ifdef CONFIG_I40E_DCB
9847                         rtnl_lock();
9848                         i40e_handle_lldp_event(pf, &event);
9849                         rtnl_unlock();
9850 #endif /* CONFIG_I40E_DCB */
9851                         break;
9852                 case i40e_aqc_opc_event_lan_overflow:
9853                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
9854                         i40e_handle_lan_overflow_event(pf, &event);
9855                         break;
9856                 case i40e_aqc_opc_send_msg_to_peer:
9857                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
9858                         break;
9859                 case i40e_aqc_opc_nvm_erase:
9860                 case i40e_aqc_opc_nvm_update:
9861                 case i40e_aqc_opc_oem_post_update:
9862                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
9863                                    "ARQ NVM operation 0x%04x completed\n",
9864                                    opcode);
9865                         break;
9866                 default:
9867                         dev_info(&pf->pdev->dev,
9868                                  "ARQ: Unknown event 0x%04x ignored\n",
9869                                  opcode);
9870                         break;
9871                 }
9872         } while (i++ < pf->adminq_work_limit);
9873
9874         if (i < pf->adminq_work_limit)
9875                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
9876
9877         /* re-enable Admin queue interrupt cause */
9878         val = rd32(hw, I40E_PFINT_ICR0_ENA);
9879         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
9880         wr32(hw, I40E_PFINT_ICR0_ENA, val);
9881         i40e_flush(hw);
9882
9883         kfree(event.msg_buf);
9884 }
9885
9886 /**
9887  * i40e_verify_eeprom - make sure eeprom is good to use
9888  * @pf: board private structure
9889  **/
9890 static void i40e_verify_eeprom(struct i40e_pf *pf)
9891 {
9892         int err;
9893
9894         err = i40e_diag_eeprom_test(&pf->hw);
9895         if (err) {
9896                 /* retry in case of garbage read */
9897                 err = i40e_diag_eeprom_test(&pf->hw);
9898                 if (err) {
9899                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
9900                                  err);
9901                         set_bit(__I40E_BAD_EEPROM, pf->state);
9902                 }
9903         }
9904
9905         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
9906                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
9907                 clear_bit(__I40E_BAD_EEPROM, pf->state);
9908         }
9909 }
9910
9911 /**
9912  * i40e_enable_pf_switch_lb
9913  * @pf: pointer to the PF structure
9914  *
9915  * enable switch loop back or die - no point in a return value
9916  **/
9917 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
9918 {
9919         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9920         struct i40e_vsi_context ctxt;
9921         int ret;
9922
9923         ctxt.seid = pf->main_vsi_seid;
9924         ctxt.pf_num = pf->hw.pf_id;
9925         ctxt.vf_num = 0;
9926         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9927         if (ret) {
9928                 dev_info(&pf->pdev->dev,
9929                          "couldn't get PF vsi config, err %s aq_err %s\n",
9930                          i40e_stat_str(&pf->hw, ret),
9931                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9932                 return;
9933         }
9934         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9935         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9936         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9937
9938         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
9939         if (ret) {
9940                 dev_info(&pf->pdev->dev,
9941                          "update vsi switch failed, err %s aq_err %s\n",
9942                          i40e_stat_str(&pf->hw, ret),
9943                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9944         }
9945 }
9946
9947 /**
9948  * i40e_disable_pf_switch_lb
9949  * @pf: pointer to the PF structure
9950  *
9951  * disable switch loop back or die - no point in a return value
9952  **/
9953 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
9954 {
9955         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9956         struct i40e_vsi_context ctxt;
9957         int ret;
9958
9959         ctxt.seid = pf->main_vsi_seid;
9960         ctxt.pf_num = pf->hw.pf_id;
9961         ctxt.vf_num = 0;
9962         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9963         if (ret) {
9964                 dev_info(&pf->pdev->dev,
9965                          "couldn't get PF vsi config, err %s aq_err %s\n",
9966                          i40e_stat_str(&pf->hw, ret),
9967                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9968                 return;
9969         }
9970         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9971         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9972         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9973
9974         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
9975         if (ret) {
9976                 dev_info(&pf->pdev->dev,
9977                          "update vsi switch failed, err %s aq_err %s\n",
9978                          i40e_stat_str(&pf->hw, ret),
9979                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9980         }
9981 }
9982
9983 /**
9984  * i40e_config_bridge_mode - Configure the HW bridge mode
9985  * @veb: pointer to the bridge instance
9986  *
9987  * Configure the loop back mode for the LAN VSI that is downlink to the
9988  * specified HW bridge instance. It is expected this function is called
9989  * when a new HW bridge is instantiated.
9990  **/
9991 static void i40e_config_bridge_mode(struct i40e_veb *veb)
9992 {
9993         struct i40e_pf *pf = veb->pf;
9994
9995         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
9996                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
9997                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
9998         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
9999                 i40e_disable_pf_switch_lb(pf);
10000         else
10001                 i40e_enable_pf_switch_lb(pf);
10002 }
10003
10004 /**
10005  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
10006  * @veb: pointer to the VEB instance
10007  *
10008  * This is a recursive function that first builds the attached VSIs then
10009  * recurses in to build the next layer of VEB.  We track the connections
10010  * through our own index numbers because the seid's from the HW could
10011  * change across the reset.
10012  **/
10013 static int i40e_reconstitute_veb(struct i40e_veb *veb)
10014 {
10015         struct i40e_vsi *ctl_vsi = NULL;
10016         struct i40e_pf *pf = veb->pf;
10017         int v, veb_idx;
10018         int ret;
10019
10020         /* build VSI that owns this VEB, temporarily attached to base VEB */
10021         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
10022                 if (pf->vsi[v] &&
10023                     pf->vsi[v]->veb_idx == veb->idx &&
10024                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
10025                         ctl_vsi = pf->vsi[v];
10026                         break;
10027                 }
10028         }
10029         if (!ctl_vsi) {
10030                 dev_info(&pf->pdev->dev,
10031                          "missing owner VSI for veb_idx %d\n", veb->idx);
10032                 ret = -ENOENT;
10033                 goto end_reconstitute;
10034         }
10035         if (ctl_vsi != pf->vsi[pf->lan_vsi])
10036                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10037         ret = i40e_add_vsi(ctl_vsi);
10038         if (ret) {
10039                 dev_info(&pf->pdev->dev,
10040                          "rebuild of veb_idx %d owner VSI failed: %d\n",
10041                          veb->idx, ret);
10042                 goto end_reconstitute;
10043         }
10044         i40e_vsi_reset_stats(ctl_vsi);
10045
10046         /* create the VEB in the switch and move the VSI onto the VEB */
10047         ret = i40e_add_veb(veb, ctl_vsi);
10048         if (ret)
10049                 goto end_reconstitute;
10050
10051         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
10052                 veb->bridge_mode = BRIDGE_MODE_VEB;
10053         else
10054                 veb->bridge_mode = BRIDGE_MODE_VEPA;
10055         i40e_config_bridge_mode(veb);
10056
10057         /* create the remaining VSIs attached to this VEB */
10058         for (v = 0; v < pf->num_alloc_vsi; v++) {
10059                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
10060                         continue;
10061
10062                 if (pf->vsi[v]->veb_idx == veb->idx) {
10063                         struct i40e_vsi *vsi = pf->vsi[v];
10064
10065                         vsi->uplink_seid = veb->seid;
10066                         ret = i40e_add_vsi(vsi);
10067                         if (ret) {
10068                                 dev_info(&pf->pdev->dev,
10069                                          "rebuild of vsi_idx %d failed: %d\n",
10070                                          v, ret);
10071                                 goto end_reconstitute;
10072                         }
10073                         i40e_vsi_reset_stats(vsi);
10074                 }
10075         }
10076
10077         /* create any VEBs attached to this VEB - RECURSION */
10078         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10079                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
10080                         pf->veb[veb_idx]->uplink_seid = veb->seid;
10081                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
10082                         if (ret)
10083                                 break;
10084                 }
10085         }
10086
10087 end_reconstitute:
10088         return ret;
10089 }
10090
10091 /**
10092  * i40e_get_capabilities - get info about the HW
10093  * @pf: the PF struct
10094  * @list_type: AQ capability to be queried
10095  **/
10096 static int i40e_get_capabilities(struct i40e_pf *pf,
10097                                  enum i40e_admin_queue_opc list_type)
10098 {
10099         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
10100         u16 data_size;
10101         int buf_len;
10102         int err;
10103
10104         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
10105         do {
10106                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
10107                 if (!cap_buf)
10108                         return -ENOMEM;
10109
10110                 /* this loads the data into the hw struct for us */
10111                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
10112                                                     &data_size, list_type,
10113                                                     NULL);
10114                 /* data loaded, buffer no longer needed */
10115                 kfree(cap_buf);
10116
10117                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
10118                         /* retry with a larger buffer */
10119                         buf_len = data_size;
10120                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
10121                         dev_info(&pf->pdev->dev,
10122                                  "capability discovery failed, err %s aq_err %s\n",
10123                                  i40e_stat_str(&pf->hw, err),
10124                                  i40e_aq_str(&pf->hw,
10125                                              pf->hw.aq.asq_last_status));
10126                         return -ENODEV;
10127                 }
10128         } while (err);
10129
10130         if (pf->hw.debug_mask & I40E_DEBUG_USER) {
10131                 if (list_type == i40e_aqc_opc_list_func_capabilities) {
10132                         dev_info(&pf->pdev->dev,
10133                                  "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
10134                                  pf->hw.pf_id, pf->hw.func_caps.num_vfs,
10135                                  pf->hw.func_caps.num_msix_vectors,
10136                                  pf->hw.func_caps.num_msix_vectors_vf,
10137                                  pf->hw.func_caps.fd_filters_guaranteed,
10138                                  pf->hw.func_caps.fd_filters_best_effort,
10139                                  pf->hw.func_caps.num_tx_qp,
10140                                  pf->hw.func_caps.num_vsis);
10141                 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
10142                         dev_info(&pf->pdev->dev,
10143                                  "switch_mode=0x%04x, function_valid=0x%08x\n",
10144                                  pf->hw.dev_caps.switch_mode,
10145                                  pf->hw.dev_caps.valid_functions);
10146                         dev_info(&pf->pdev->dev,
10147                                  "SR-IOV=%d, num_vfs for all function=%u\n",
10148                                  pf->hw.dev_caps.sr_iov_1_1,
10149                                  pf->hw.dev_caps.num_vfs);
10150                         dev_info(&pf->pdev->dev,
10151                                  "num_vsis=%u, num_rx:%u, num_tx=%u\n",
10152                                  pf->hw.dev_caps.num_vsis,
10153                                  pf->hw.dev_caps.num_rx_qp,
10154                                  pf->hw.dev_caps.num_tx_qp);
10155                 }
10156         }
10157         if (list_type == i40e_aqc_opc_list_func_capabilities) {
10158 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
10159                        + pf->hw.func_caps.num_vfs)
10160                 if (pf->hw.revision_id == 0 &&
10161                     pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
10162                         dev_info(&pf->pdev->dev,
10163                                  "got num_vsis %d, setting num_vsis to %d\n",
10164                                  pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
10165                         pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
10166                 }
10167         }
10168         return 0;
10169 }
10170
10171 static int i40e_vsi_clear(struct i40e_vsi *vsi);
10172
10173 /**
10174  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10175  * @pf: board private structure
10176  **/
10177 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
10178 {
10179         struct i40e_vsi *vsi;
10180
10181         /* quick workaround for an NVM issue that leaves a critical register
10182          * uninitialized
10183          */
10184         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
10185                 static const u32 hkey[] = {
10186                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
10187                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
10188                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
10189                         0x95b3a76d};
10190                 int i;
10191
10192                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
10193                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
10194         }
10195
10196         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
10197                 return;
10198
10199         /* find existing VSI and see if it needs configuring */
10200         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10201
10202         /* create a new VSI if none exists */
10203         if (!vsi) {
10204                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
10205                                      pf->vsi[pf->lan_vsi]->seid, 0);
10206                 if (!vsi) {
10207                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
10208                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
10209                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
10210                         return;
10211                 }
10212         }
10213
10214         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
10215 }
10216
10217 /**
10218  * i40e_fdir_teardown - release the Flow Director resources
10219  * @pf: board private structure
10220  **/
10221 static void i40e_fdir_teardown(struct i40e_pf *pf)
10222 {
10223         struct i40e_vsi *vsi;
10224
10225         i40e_fdir_filter_exit(pf);
10226         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10227         if (vsi)
10228                 i40e_vsi_release(vsi);
10229 }
10230
10231 /**
10232  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10233  * @vsi: PF main vsi
10234  * @seid: seid of main or channel VSIs
10235  *
10236  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
10237  * existed before reset
10238  **/
10239 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
10240 {
10241         struct i40e_cloud_filter *cfilter;
10242         struct i40e_pf *pf = vsi->back;
10243         struct hlist_node *node;
10244         i40e_status ret;
10245
10246         /* Add cloud filters back if they exist */
10247         hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
10248                                   cloud_node) {
10249                 if (cfilter->seid != seid)
10250                         continue;
10251
10252                 if (cfilter->dst_port)
10253                         ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
10254                                                                 true);
10255                 else
10256                         ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
10257
10258                 if (ret) {
10259                         dev_dbg(&pf->pdev->dev,
10260                                 "Failed to rebuild cloud filter, err %s aq_err %s\n",
10261                                 i40e_stat_str(&pf->hw, ret),
10262                                 i40e_aq_str(&pf->hw,
10263                                             pf->hw.aq.asq_last_status));
10264                         return ret;
10265                 }
10266         }
10267         return 0;
10268 }
10269
10270 /**
10271  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10272  * @vsi: PF main vsi
10273  *
10274  * Rebuilds channel VSIs if they existed before reset
10275  **/
10276 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10277 {
10278         struct i40e_channel *ch, *ch_tmp;
10279         i40e_status ret;
10280
10281         if (list_empty(&vsi->ch_list))
10282                 return 0;
10283
10284         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10285                 if (!ch->initialized)
10286                         break;
10287                 /* Proceed with creation of channel (VMDq2) VSI */
10288                 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
10289                 if (ret) {
10290                         dev_info(&vsi->back->pdev->dev,
10291                                  "failed to rebuild channels using uplink_seid %u\n",
10292                                  vsi->uplink_seid);
10293                         return ret;
10294                 }
10295                 /* Reconfigure TX queues using QTX_CTL register */
10296                 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
10297                 if (ret) {
10298                         dev_info(&vsi->back->pdev->dev,
10299                                  "failed to configure TX rings for channel %u\n",
10300                                  ch->seid);
10301                         return ret;
10302                 }
10303                 /* update 'next_base_queue' */
10304                 vsi->next_base_queue = vsi->next_base_queue +
10305                                                         ch->num_queue_pairs;
10306                 if (ch->max_tx_rate) {
10307                         u64 credits = ch->max_tx_rate;
10308
10309                         if (i40e_set_bw_limit(vsi, ch->seid,
10310                                               ch->max_tx_rate))
10311                                 return -EINVAL;
10312
10313                         do_div(credits, I40E_BW_CREDIT_DIVISOR);
10314                         dev_dbg(&vsi->back->pdev->dev,
10315                                 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10316                                 ch->max_tx_rate,
10317                                 credits,
10318                                 ch->seid);
10319                 }
10320                 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
10321                 if (ret) {
10322                         dev_dbg(&vsi->back->pdev->dev,
10323                                 "Failed to rebuild cloud filters for channel VSI %u\n",
10324                                 ch->seid);
10325                         return ret;
10326                 }
10327         }
10328         return 0;
10329 }
10330
10331 /**
10332  * i40e_prep_for_reset - prep for the core to reset
10333  * @pf: board private structure
10334  *
10335  * Close up the VFs and other things in prep for PF Reset.
10336   **/
10337 static void i40e_prep_for_reset(struct i40e_pf *pf)
10338 {
10339         struct i40e_hw *hw = &pf->hw;
10340         i40e_status ret = 0;
10341         u32 v;
10342
10343         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
10344         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10345                 return;
10346         if (i40e_check_asq_alive(&pf->hw))
10347                 i40e_vc_notify_reset(pf);
10348
10349         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10350
10351         /* quiesce the VSIs and their queues that are not already DOWN */
10352         i40e_pf_quiesce_all_vsi(pf);
10353
10354         for (v = 0; v < pf->num_alloc_vsi; v++) {
10355                 if (pf->vsi[v])
10356                         pf->vsi[v]->seid = 0;
10357         }
10358
10359         i40e_shutdown_adminq(&pf->hw);
10360
10361         /* call shutdown HMC */
10362         if (hw->hmc.hmc_obj) {
10363                 ret = i40e_shutdown_lan_hmc(hw);
10364                 if (ret)
10365                         dev_warn(&pf->pdev->dev,
10366                                  "shutdown_lan_hmc failed: %d\n", ret);
10367         }
10368
10369         /* Save the current PTP time so that we can restore the time after the
10370          * reset completes.
10371          */
10372         i40e_ptp_save_hw_time(pf);
10373 }
10374
10375 /**
10376  * i40e_send_version - update firmware with driver version
10377  * @pf: PF struct
10378  */
10379 static void i40e_send_version(struct i40e_pf *pf)
10380 {
10381         struct i40e_driver_version dv;
10382
10383         dv.major_version = 0xff;
10384         dv.minor_version = 0xff;
10385         dv.build_version = 0xff;
10386         dv.subbuild_version = 0;
10387         strlcpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10388         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
10389 }
10390
10391 /**
10392  * i40e_get_oem_version - get OEM specific version information
10393  * @hw: pointer to the hardware structure
10394  **/
10395 static void i40e_get_oem_version(struct i40e_hw *hw)
10396 {
10397         u16 block_offset = 0xffff;
10398         u16 block_length = 0;
10399         u16 capabilities = 0;
10400         u16 gen_snap = 0;
10401         u16 release = 0;
10402
10403 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
10404 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
10405 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
10406 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
10407 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
10408 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
10409 #define I40E_NVM_OEM_LENGTH                     3
10410
10411         /* Check if pointer to OEM version block is valid. */
10412         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
10413         if (block_offset == 0xffff)
10414                 return;
10415
10416         /* Check if OEM version block has correct length. */
10417         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10418                            &block_length);
10419         if (block_length < I40E_NVM_OEM_LENGTH)
10420                 return;
10421
10422         /* Check if OEM version format is as expected. */
10423         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10424                            &capabilities);
10425         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10426                 return;
10427
10428         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
10429                            &gen_snap);
10430         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10431                            &release);
10432         hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
10433         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10434 }
10435
10436 /**
10437  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10438  * @pf: board private structure
10439  **/
10440 static int i40e_reset(struct i40e_pf *pf)
10441 {
10442         struct i40e_hw *hw = &pf->hw;
10443         i40e_status ret;
10444
10445         ret = i40e_pf_reset(hw);
10446         if (ret) {
10447                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10448                 set_bit(__I40E_RESET_FAILED, pf->state);
10449                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10450         } else {
10451                 pf->pfr_count++;
10452         }
10453         return ret;
10454 }
10455
10456 /**
10457  * i40e_rebuild - rebuild using a saved config
10458  * @pf: board private structure
10459  * @reinit: if the Main VSI needs to re-initialized.
10460  * @lock_acquired: indicates whether or not the lock has been acquired
10461  * before this function was called.
10462  **/
10463 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10464 {
10465         int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state);
10466         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10467         struct i40e_hw *hw = &pf->hw;
10468         i40e_status ret;
10469         u32 val;
10470         int v;
10471
10472         if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10473             i40e_check_recovery_mode(pf)) {
10474                 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev);
10475         }
10476
10477         if (test_bit(__I40E_DOWN, pf->state) &&
10478             !test_bit(__I40E_RECOVERY_MODE, pf->state) &&
10479             !old_recovery_mode_bit)
10480                 goto clear_recovery;
10481         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10482
10483         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10484         ret = i40e_init_adminq(&pf->hw);
10485         if (ret) {
10486                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
10487                          i40e_stat_str(&pf->hw, ret),
10488                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10489                 goto clear_recovery;
10490         }
10491         i40e_get_oem_version(&pf->hw);
10492
10493         if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10494             ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) ||
10495              hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) {
10496                 /* The following delay is necessary for 4.33 firmware and older
10497                  * to recover after EMP reset. 200 ms should suffice but we
10498                  * put here 300 ms to be sure that FW is ready to operate
10499                  * after reset.
10500                  */
10501                 mdelay(300);
10502         }
10503
10504         /* re-verify the eeprom if we just had an EMP reset */
10505         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
10506                 i40e_verify_eeprom(pf);
10507
10508         /* if we are going out of or into recovery mode we have to act
10509          * accordingly with regard to resources initialization
10510          * and deinitialization
10511          */
10512         if (test_bit(__I40E_RECOVERY_MODE, pf->state) ||
10513             old_recovery_mode_bit) {
10514                 if (i40e_get_capabilities(pf,
10515                                           i40e_aqc_opc_list_func_capabilities))
10516                         goto end_unlock;
10517
10518                 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10519                         /* we're staying in recovery mode so we'll reinitialize
10520                          * misc vector here
10521                          */
10522                         if (i40e_setup_misc_vector_for_recovery_mode(pf))
10523                                 goto end_unlock;
10524                 } else {
10525                         if (!lock_acquired)
10526                                 rtnl_lock();
10527                         /* we're going out of recovery mode so we'll free
10528                          * the IRQ allocated specifically for recovery mode
10529                          * and restore the interrupt scheme
10530                          */
10531                         free_irq(pf->pdev->irq, pf);
10532                         i40e_clear_interrupt_scheme(pf);
10533                         if (i40e_restore_interrupt_scheme(pf))
10534                                 goto end_unlock;
10535                 }
10536
10537                 /* tell the firmware that we're starting */
10538                 i40e_send_version(pf);
10539
10540                 /* bail out in case recovery mode was detected, as there is
10541                  * no need for further configuration.
10542                  */
10543                 goto end_unlock;
10544         }
10545
10546         i40e_clear_pxe_mode(hw);
10547         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10548         if (ret)
10549                 goto end_core_reset;
10550
10551         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10552                                 hw->func_caps.num_rx_qp, 0, 0);
10553         if (ret) {
10554                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10555                 goto end_core_reset;
10556         }
10557         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10558         if (ret) {
10559                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10560                 goto end_core_reset;
10561         }
10562
10563         if (!lock_acquired)
10564                 rtnl_lock();
10565         ret = i40e_setup_pf_switch(pf, reinit);
10566         if (ret)
10567                 goto end_unlock;
10568
10569 #ifdef CONFIG_I40E_DCB
10570         /* Enable FW to write a default DCB config on link-up
10571          * unless I40E_FLAG_TC_MQPRIO was enabled or DCB
10572          * is not supported with new link speed
10573          */
10574         if (pf->flags & I40E_FLAG_TC_MQPRIO) {
10575                 i40e_aq_set_dcb_parameters(hw, false, NULL);
10576         } else {
10577                 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
10578                     (hw->phy.link_info.link_speed &
10579                      (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
10580                         i40e_aq_set_dcb_parameters(hw, false, NULL);
10581                         dev_warn(&pf->pdev->dev,
10582                                  "DCB is not supported for X710-T*L 2.5/5G speeds\n");
10583                                  pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10584                 } else {
10585                         i40e_aq_set_dcb_parameters(hw, true, NULL);
10586                         ret = i40e_init_pf_dcb(pf);
10587                         if (ret) {
10588                                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
10589                                          ret);
10590                                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10591                                 /* Continue without DCB enabled */
10592                         }
10593                 }
10594         }
10595
10596 #endif /* CONFIG_I40E_DCB */
10597
10598         /* The driver only wants link up/down and module qualification
10599          * reports from firmware.  Note the negative logic.
10600          */
10601         ret = i40e_aq_set_phy_int_mask(&pf->hw,
10602                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
10603                                          I40E_AQ_EVENT_MEDIA_NA |
10604                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10605         if (ret)
10606                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
10607                          i40e_stat_str(&pf->hw, ret),
10608                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10609
10610         /* Rebuild the VSIs and VEBs that existed before reset.
10611          * They are still in our local switch element arrays, so only
10612          * need to rebuild the switch model in the HW.
10613          *
10614          * If there were VEBs but the reconstitution failed, we'll try
10615          * try to recover minimal use by getting the basic PF VSI working.
10616          */
10617         if (vsi->uplink_seid != pf->mac_seid) {
10618                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
10619                 /* find the one VEB connected to the MAC, and find orphans */
10620                 for (v = 0; v < I40E_MAX_VEB; v++) {
10621                         if (!pf->veb[v])
10622                                 continue;
10623
10624                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
10625                             pf->veb[v]->uplink_seid == 0) {
10626                                 ret = i40e_reconstitute_veb(pf->veb[v]);
10627
10628                                 if (!ret)
10629                                         continue;
10630
10631                                 /* If Main VEB failed, we're in deep doodoo,
10632                                  * so give up rebuilding the switch and set up
10633                                  * for minimal rebuild of PF VSI.
10634                                  * If orphan failed, we'll report the error
10635                                  * but try to keep going.
10636                                  */
10637                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
10638                                         dev_info(&pf->pdev->dev,
10639                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
10640                                                  ret);
10641                                         vsi->uplink_seid = pf->mac_seid;
10642                                         break;
10643                                 } else if (pf->veb[v]->uplink_seid == 0) {
10644                                         dev_info(&pf->pdev->dev,
10645                                                  "rebuild of orphan VEB failed: %d\n",
10646                                                  ret);
10647                                 }
10648                         }
10649                 }
10650         }
10651
10652         if (vsi->uplink_seid == pf->mac_seid) {
10653                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
10654                 /* no VEB, so rebuild only the Main VSI */
10655                 ret = i40e_add_vsi(vsi);
10656                 if (ret) {
10657                         dev_info(&pf->pdev->dev,
10658                                  "rebuild of Main VSI failed: %d\n", ret);
10659                         goto end_unlock;
10660                 }
10661         }
10662
10663         if (vsi->mqprio_qopt.max_rate[0]) {
10664                 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
10665                 u64 credits = 0;
10666
10667                 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
10668                 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
10669                 if (ret)
10670                         goto end_unlock;
10671
10672                 credits = max_tx_rate;
10673                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
10674                 dev_dbg(&vsi->back->pdev->dev,
10675                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10676                         max_tx_rate,
10677                         credits,
10678                         vsi->seid);
10679         }
10680
10681         ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
10682         if (ret)
10683                 goto end_unlock;
10684
10685         /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
10686          * for this main VSI if they exist
10687          */
10688         ret = i40e_rebuild_channels(vsi);
10689         if (ret)
10690                 goto end_unlock;
10691
10692         /* Reconfigure hardware for allowing smaller MSS in the case
10693          * of TSO, so that we avoid the MDD being fired and causing
10694          * a reset in the case of small MSS+TSO.
10695          */
10696 #define I40E_REG_MSS          0x000E64DC
10697 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
10698 #define I40E_64BYTE_MSS       0x400000
10699         val = rd32(hw, I40E_REG_MSS);
10700         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
10701                 val &= ~I40E_REG_MSS_MIN_MASK;
10702                 val |= I40E_64BYTE_MSS;
10703                 wr32(hw, I40E_REG_MSS, val);
10704         }
10705
10706         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
10707                 msleep(75);
10708                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
10709                 if (ret)
10710                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
10711                                  i40e_stat_str(&pf->hw, ret),
10712                                  i40e_aq_str(&pf->hw,
10713                                              pf->hw.aq.asq_last_status));
10714         }
10715         /* reinit the misc interrupt */
10716         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10717                 ret = i40e_setup_misc_vector(pf);
10718
10719         /* Add a filter to drop all Flow control frames from any VSI from being
10720          * transmitted. By doing so we stop a malicious VF from sending out
10721          * PAUSE or PFC frames and potentially controlling traffic for other
10722          * PF/VF VSIs.
10723          * The FW can still send Flow control frames if enabled.
10724          */
10725         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
10726                                                        pf->main_vsi_seid);
10727 #ifdef CONFIG_I40E_DCB
10728         if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)
10729                 i40e_set_lldp_forwarding(pf, true);
10730 #endif /* CONFIG_I40E_DCB */
10731
10732         /* restart the VSIs that were rebuilt and running before the reset */
10733         i40e_pf_unquiesce_all_vsi(pf);
10734
10735         /* Release the RTNL lock before we start resetting VFs */
10736         if (!lock_acquired)
10737                 rtnl_unlock();
10738
10739         /* Restore promiscuous settings */
10740         ret = i40e_set_promiscuous(pf, pf->cur_promisc);
10741         if (ret)
10742                 dev_warn(&pf->pdev->dev,
10743                          "Failed to restore promiscuous setting: %s, err %s aq_err %s\n",
10744                          pf->cur_promisc ? "on" : "off",
10745                          i40e_stat_str(&pf->hw, ret),
10746                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10747
10748         i40e_reset_all_vfs(pf, true);
10749
10750         /* tell the firmware that we're starting */
10751         i40e_send_version(pf);
10752
10753         /* We've already released the lock, so don't do it again */
10754         goto end_core_reset;
10755
10756 end_unlock:
10757         if (!lock_acquired)
10758                 rtnl_unlock();
10759 end_core_reset:
10760         clear_bit(__I40E_RESET_FAILED, pf->state);
10761 clear_recovery:
10762         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10763         clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
10764 }
10765
10766 /**
10767  * i40e_reset_and_rebuild - reset and rebuild using a saved config
10768  * @pf: board private structure
10769  * @reinit: if the Main VSI needs to re-initialized.
10770  * @lock_acquired: indicates whether or not the lock has been acquired
10771  * before this function was called.
10772  **/
10773 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
10774                                    bool lock_acquired)
10775 {
10776         int ret;
10777         /* Now we wait for GRST to settle out.
10778          * We don't have to delete the VEBs or VSIs from the hw switch
10779          * because the reset will make them disappear.
10780          */
10781         ret = i40e_reset(pf);
10782         if (!ret)
10783                 i40e_rebuild(pf, reinit, lock_acquired);
10784 }
10785
10786 /**
10787  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
10788  * @pf: board private structure
10789  *
10790  * Close up the VFs and other things in prep for a Core Reset,
10791  * then get ready to rebuild the world.
10792  * @lock_acquired: indicates whether or not the lock has been acquired
10793  * before this function was called.
10794  **/
10795 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
10796 {
10797         i40e_prep_for_reset(pf);
10798         i40e_reset_and_rebuild(pf, false, lock_acquired);
10799 }
10800
10801 /**
10802  * i40e_handle_mdd_event
10803  * @pf: pointer to the PF structure
10804  *
10805  * Called from the MDD irq handler to identify possibly malicious vfs
10806  **/
10807 static void i40e_handle_mdd_event(struct i40e_pf *pf)
10808 {
10809         struct i40e_hw *hw = &pf->hw;
10810         bool mdd_detected = false;
10811         struct i40e_vf *vf;
10812         u32 reg;
10813         int i;
10814
10815         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
10816                 return;
10817
10818         /* find what triggered the MDD event */
10819         reg = rd32(hw, I40E_GL_MDET_TX);
10820         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
10821                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
10822                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
10823                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
10824                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
10825                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
10826                                 I40E_GL_MDET_TX_EVENT_SHIFT;
10827                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
10828                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
10829                                 pf->hw.func_caps.base_queue;
10830                 if (netif_msg_tx_err(pf))
10831                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
10832                                  event, queue, pf_num, vf_num);
10833                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
10834                 mdd_detected = true;
10835         }
10836         reg = rd32(hw, I40E_GL_MDET_RX);
10837         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
10838                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
10839                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
10840                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
10841                                 I40E_GL_MDET_RX_EVENT_SHIFT;
10842                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
10843                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
10844                                 pf->hw.func_caps.base_queue;
10845                 if (netif_msg_rx_err(pf))
10846                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
10847                                  event, queue, func);
10848                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
10849                 mdd_detected = true;
10850         }
10851
10852         if (mdd_detected) {
10853                 reg = rd32(hw, I40E_PF_MDET_TX);
10854                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
10855                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
10856                         dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
10857                 }
10858                 reg = rd32(hw, I40E_PF_MDET_RX);
10859                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
10860                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
10861                         dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
10862                 }
10863         }
10864
10865         /* see if one of the VFs needs its hand slapped */
10866         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
10867                 vf = &(pf->vf[i]);
10868                 reg = rd32(hw, I40E_VP_MDET_TX(i));
10869                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
10870                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
10871                         vf->num_mdd_events++;
10872                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
10873                                  i);
10874                         dev_info(&pf->pdev->dev,
10875                                  "Use PF Control I/F to re-enable the VF\n");
10876                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10877                 }
10878
10879                 reg = rd32(hw, I40E_VP_MDET_RX(i));
10880                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
10881                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
10882                         vf->num_mdd_events++;
10883                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
10884                                  i);
10885                         dev_info(&pf->pdev->dev,
10886                                  "Use PF Control I/F to re-enable the VF\n");
10887                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10888                 }
10889         }
10890
10891         /* re-enable mdd interrupt cause */
10892         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
10893         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
10894         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
10895         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
10896         i40e_flush(hw);
10897 }
10898
10899 /**
10900  * i40e_service_task - Run the driver's async subtasks
10901  * @work: pointer to work_struct containing our data
10902  **/
10903 static void i40e_service_task(struct work_struct *work)
10904 {
10905         struct i40e_pf *pf = container_of(work,
10906                                           struct i40e_pf,
10907                                           service_task);
10908         unsigned long start_time = jiffies;
10909
10910         /* don't bother with service tasks if a reset is in progress */
10911         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
10912             test_bit(__I40E_SUSPENDED, pf->state))
10913                 return;
10914
10915         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
10916                 return;
10917
10918         if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10919                 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
10920                 i40e_sync_filters_subtask(pf);
10921                 i40e_reset_subtask(pf);
10922                 i40e_handle_mdd_event(pf);
10923                 i40e_vc_process_vflr_event(pf);
10924                 i40e_watchdog_subtask(pf);
10925                 i40e_fdir_reinit_subtask(pf);
10926                 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
10927                         /* Client subtask will reopen next time through. */
10928                         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi],
10929                                                            true);
10930                 } else {
10931                         i40e_client_subtask(pf);
10932                         if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
10933                                                pf->state))
10934                                 i40e_notify_client_of_l2_param_changes(
10935                                                                 pf->vsi[pf->lan_vsi]);
10936                 }
10937                 i40e_sync_filters_subtask(pf);
10938         } else {
10939                 i40e_reset_subtask(pf);
10940         }
10941
10942         i40e_clean_adminq_subtask(pf);
10943
10944         /* flush memory to make sure state is correct before next watchdog */
10945         smp_mb__before_atomic();
10946         clear_bit(__I40E_SERVICE_SCHED, pf->state);
10947
10948         /* If the tasks have taken longer than one timer cycle or there
10949          * is more work to be done, reschedule the service task now
10950          * rather than wait for the timer to tick again.
10951          */
10952         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
10953             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
10954             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
10955             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
10956                 i40e_service_event_schedule(pf);
10957 }
10958
10959 /**
10960  * i40e_service_timer - timer callback
10961  * @t: timer list pointer
10962  **/
10963 static void i40e_service_timer(struct timer_list *t)
10964 {
10965         struct i40e_pf *pf = from_timer(pf, t, service_timer);
10966
10967         mod_timer(&pf->service_timer,
10968                   round_jiffies(jiffies + pf->service_timer_period));
10969         i40e_service_event_schedule(pf);
10970 }
10971
10972 /**
10973  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
10974  * @vsi: the VSI being configured
10975  **/
10976 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
10977 {
10978         struct i40e_pf *pf = vsi->back;
10979
10980         switch (vsi->type) {
10981         case I40E_VSI_MAIN:
10982                 vsi->alloc_queue_pairs = pf->num_lan_qps;
10983                 if (!vsi->num_tx_desc)
10984                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10985                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
10986                 if (!vsi->num_rx_desc)
10987                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10988                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
10989                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10990                         vsi->num_q_vectors = pf->num_lan_msix;
10991                 else
10992                         vsi->num_q_vectors = 1;
10993
10994                 break;
10995
10996         case I40E_VSI_FDIR:
10997                 vsi->alloc_queue_pairs = 1;
10998                 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
10999                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
11000                 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11001                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
11002                 vsi->num_q_vectors = pf->num_fdsb_msix;
11003                 break;
11004
11005         case I40E_VSI_VMDQ2:
11006                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
11007                 if (!vsi->num_tx_desc)
11008                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11009                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11010                 if (!vsi->num_rx_desc)
11011                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11012                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11013                 vsi->num_q_vectors = pf->num_vmdq_msix;
11014                 break;
11015
11016         case I40E_VSI_SRIOV:
11017                 vsi->alloc_queue_pairs = pf->num_vf_qps;
11018                 if (!vsi->num_tx_desc)
11019                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11020                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11021                 if (!vsi->num_rx_desc)
11022                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11023                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11024                 break;
11025
11026         default:
11027                 WARN_ON(1);
11028                 return -ENODATA;
11029         }
11030
11031         return 0;
11032 }
11033
11034 /**
11035  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11036  * @vsi: VSI pointer
11037  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
11038  *
11039  * On error: returns error code (negative)
11040  * On success: returns 0
11041  **/
11042 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
11043 {
11044         struct i40e_ring **next_rings;
11045         int size;
11046         int ret = 0;
11047
11048         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
11049         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
11050                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
11051         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
11052         if (!vsi->tx_rings)
11053                 return -ENOMEM;
11054         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
11055         if (i40e_enabled_xdp_vsi(vsi)) {
11056                 vsi->xdp_rings = next_rings;
11057                 next_rings += vsi->alloc_queue_pairs;
11058         }
11059         vsi->rx_rings = next_rings;
11060
11061         if (alloc_qvectors) {
11062                 /* allocate memory for q_vector pointers */
11063                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
11064                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
11065                 if (!vsi->q_vectors) {
11066                         ret = -ENOMEM;
11067                         goto err_vectors;
11068                 }
11069         }
11070         return ret;
11071
11072 err_vectors:
11073         kfree(vsi->tx_rings);
11074         return ret;
11075 }
11076
11077 /**
11078  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11079  * @pf: board private structure
11080  * @type: type of VSI
11081  *
11082  * On error: returns error code (negative)
11083  * On success: returns vsi index in PF (positive)
11084  **/
11085 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
11086 {
11087         int ret = -ENODEV;
11088         struct i40e_vsi *vsi;
11089         int vsi_idx;
11090         int i;
11091
11092         /* Need to protect the allocation of the VSIs at the PF level */
11093         mutex_lock(&pf->switch_mutex);
11094
11095         /* VSI list may be fragmented if VSI creation/destruction has
11096          * been happening.  We can afford to do a quick scan to look
11097          * for any free VSIs in the list.
11098          *
11099          * find next empty vsi slot, looping back around if necessary
11100          */
11101         i = pf->next_vsi;
11102         while (i < pf->num_alloc_vsi && pf->vsi[i])
11103                 i++;
11104         if (i >= pf->num_alloc_vsi) {
11105                 i = 0;
11106                 while (i < pf->next_vsi && pf->vsi[i])
11107                         i++;
11108         }
11109
11110         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
11111                 vsi_idx = i;             /* Found one! */
11112         } else {
11113                 ret = -ENODEV;
11114                 goto unlock_pf;  /* out of VSI slots! */
11115         }
11116         pf->next_vsi = ++i;
11117
11118         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
11119         if (!vsi) {
11120                 ret = -ENOMEM;
11121                 goto unlock_pf;
11122         }
11123         vsi->type = type;
11124         vsi->back = pf;
11125         set_bit(__I40E_VSI_DOWN, vsi->state);
11126         vsi->flags = 0;
11127         vsi->idx = vsi_idx;
11128         vsi->int_rate_limit = 0;
11129         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
11130                                 pf->rss_table_size : 64;
11131         vsi->netdev_registered = false;
11132         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
11133         hash_init(vsi->mac_filter_hash);
11134         vsi->irqs_ready = false;
11135
11136         if (type == I40E_VSI_MAIN) {
11137                 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
11138                 if (!vsi->af_xdp_zc_qps)
11139                         goto err_rings;
11140         }
11141
11142         ret = i40e_set_num_rings_in_vsi(vsi);
11143         if (ret)
11144                 goto err_rings;
11145
11146         ret = i40e_vsi_alloc_arrays(vsi, true);
11147         if (ret)
11148                 goto err_rings;
11149
11150         /* Setup default MSIX irq handler for VSI */
11151         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
11152
11153         /* Initialize VSI lock */
11154         spin_lock_init(&vsi->mac_filter_hash_lock);
11155         pf->vsi[vsi_idx] = vsi;
11156         ret = vsi_idx;
11157         goto unlock_pf;
11158
11159 err_rings:
11160         bitmap_free(vsi->af_xdp_zc_qps);
11161         pf->next_vsi = i - 1;
11162         kfree(vsi);
11163 unlock_pf:
11164         mutex_unlock(&pf->switch_mutex);
11165         return ret;
11166 }
11167
11168 /**
11169  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11170  * @vsi: VSI pointer
11171  * @free_qvectors: a bool to specify if q_vectors need to be freed.
11172  *
11173  * On error: returns error code (negative)
11174  * On success: returns 0
11175  **/
11176 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
11177 {
11178         /* free the ring and vector containers */
11179         if (free_qvectors) {
11180                 kfree(vsi->q_vectors);
11181                 vsi->q_vectors = NULL;
11182         }
11183         kfree(vsi->tx_rings);
11184         vsi->tx_rings = NULL;
11185         vsi->rx_rings = NULL;
11186         vsi->xdp_rings = NULL;
11187 }
11188
11189 /**
11190  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
11191  * and lookup table
11192  * @vsi: Pointer to VSI structure
11193  */
11194 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
11195 {
11196         if (!vsi)
11197                 return;
11198
11199         kfree(vsi->rss_hkey_user);
11200         vsi->rss_hkey_user = NULL;
11201
11202         kfree(vsi->rss_lut_user);
11203         vsi->rss_lut_user = NULL;
11204 }
11205
11206 /**
11207  * i40e_vsi_clear - Deallocate the VSI provided
11208  * @vsi: the VSI being un-configured
11209  **/
11210 static int i40e_vsi_clear(struct i40e_vsi *vsi)
11211 {
11212         struct i40e_pf *pf;
11213
11214         if (!vsi)
11215                 return 0;
11216
11217         if (!vsi->back)
11218                 goto free_vsi;
11219         pf = vsi->back;
11220
11221         mutex_lock(&pf->switch_mutex);
11222         if (!pf->vsi[vsi->idx]) {
11223                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
11224                         vsi->idx, vsi->idx, vsi->type);
11225                 goto unlock_vsi;
11226         }
11227
11228         if (pf->vsi[vsi->idx] != vsi) {
11229                 dev_err(&pf->pdev->dev,
11230                         "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
11231                         pf->vsi[vsi->idx]->idx,
11232                         pf->vsi[vsi->idx]->type,
11233                         vsi->idx, vsi->type);
11234                 goto unlock_vsi;
11235         }
11236
11237         /* updates the PF for this cleared vsi */
11238         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
11239         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
11240
11241         bitmap_free(vsi->af_xdp_zc_qps);
11242         i40e_vsi_free_arrays(vsi, true);
11243         i40e_clear_rss_config_user(vsi);
11244
11245         pf->vsi[vsi->idx] = NULL;
11246         if (vsi->idx < pf->next_vsi)
11247                 pf->next_vsi = vsi->idx;
11248
11249 unlock_vsi:
11250         mutex_unlock(&pf->switch_mutex);
11251 free_vsi:
11252         kfree(vsi);
11253
11254         return 0;
11255 }
11256
11257 /**
11258  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11259  * @vsi: the VSI being cleaned
11260  **/
11261 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11262 {
11263         int i;
11264
11265         if (vsi->tx_rings && vsi->tx_rings[0]) {
11266                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11267                         kfree_rcu(vsi->tx_rings[i], rcu);
11268                         WRITE_ONCE(vsi->tx_rings[i], NULL);
11269                         WRITE_ONCE(vsi->rx_rings[i], NULL);
11270                         if (vsi->xdp_rings)
11271                                 WRITE_ONCE(vsi->xdp_rings[i], NULL);
11272                 }
11273         }
11274 }
11275
11276 /**
11277  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11278  * @vsi: the VSI being configured
11279  **/
11280 static int i40e_alloc_rings(struct i40e_vsi *vsi)
11281 {
11282         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11283         struct i40e_pf *pf = vsi->back;
11284         struct i40e_ring *ring;
11285
11286         /* Set basic values in the rings to be used later during open() */
11287         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11288                 /* allocate space for both Tx and Rx in one shot */
11289                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
11290                 if (!ring)
11291                         goto err_out;
11292
11293                 ring->queue_index = i;
11294                 ring->reg_idx = vsi->base_queue + i;
11295                 ring->ring_active = false;
11296                 ring->vsi = vsi;
11297                 ring->netdev = vsi->netdev;
11298                 ring->dev = &pf->pdev->dev;
11299                 ring->count = vsi->num_tx_desc;
11300                 ring->size = 0;
11301                 ring->dcb_tc = 0;
11302                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11303                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11304                 ring->itr_setting = pf->tx_itr_default;
11305                 WRITE_ONCE(vsi->tx_rings[i], ring++);
11306
11307                 if (!i40e_enabled_xdp_vsi(vsi))
11308                         goto setup_rx;
11309
11310                 ring->queue_index = vsi->alloc_queue_pairs + i;
11311                 ring->reg_idx = vsi->base_queue + ring->queue_index;
11312                 ring->ring_active = false;
11313                 ring->vsi = vsi;
11314                 ring->netdev = NULL;
11315                 ring->dev = &pf->pdev->dev;
11316                 ring->count = vsi->num_tx_desc;
11317                 ring->size = 0;
11318                 ring->dcb_tc = 0;
11319                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11320                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11321                 set_ring_xdp(ring);
11322                 ring->itr_setting = pf->tx_itr_default;
11323                 WRITE_ONCE(vsi->xdp_rings[i], ring++);
11324
11325 setup_rx:
11326                 ring->queue_index = i;
11327                 ring->reg_idx = vsi->base_queue + i;
11328                 ring->ring_active = false;
11329                 ring->vsi = vsi;
11330                 ring->netdev = vsi->netdev;
11331                 ring->dev = &pf->pdev->dev;
11332                 ring->count = vsi->num_rx_desc;
11333                 ring->size = 0;
11334                 ring->dcb_tc = 0;
11335                 ring->itr_setting = pf->rx_itr_default;
11336                 WRITE_ONCE(vsi->rx_rings[i], ring);
11337         }
11338
11339         return 0;
11340
11341 err_out:
11342         i40e_vsi_clear_rings(vsi);
11343         return -ENOMEM;
11344 }
11345
11346 /**
11347  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11348  * @pf: board private structure
11349  * @vectors: the number of MSI-X vectors to request
11350  *
11351  * Returns the number of vectors reserved, or error
11352  **/
11353 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11354 {
11355         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
11356                                         I40E_MIN_MSIX, vectors);
11357         if (vectors < 0) {
11358                 dev_info(&pf->pdev->dev,
11359                          "MSI-X vector reservation failed: %d\n", vectors);
11360                 vectors = 0;
11361         }
11362
11363         return vectors;
11364 }
11365
11366 /**
11367  * i40e_init_msix - Setup the MSIX capability
11368  * @pf: board private structure
11369  *
11370  * Work with the OS to set up the MSIX vectors needed.
11371  *
11372  * Returns the number of vectors reserved or negative on failure
11373  **/
11374 static int i40e_init_msix(struct i40e_pf *pf)
11375 {
11376         struct i40e_hw *hw = &pf->hw;
11377         int cpus, extra_vectors;
11378         int vectors_left;
11379         int v_budget, i;
11380         int v_actual;
11381         int iwarp_requested = 0;
11382
11383         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
11384                 return -ENODEV;
11385
11386         /* The number of vectors we'll request will be comprised of:
11387          *   - Add 1 for "other" cause for Admin Queue events, etc.
11388          *   - The number of LAN queue pairs
11389          *      - Queues being used for RSS.
11390          *              We don't need as many as max_rss_size vectors.
11391          *              use rss_size instead in the calculation since that
11392          *              is governed by number of cpus in the system.
11393          *      - assumes symmetric Tx/Rx pairing
11394          *   - The number of VMDq pairs
11395          *   - The CPU count within the NUMA node if iWARP is enabled
11396          * Once we count this up, try the request.
11397          *
11398          * If we can't get what we want, we'll simplify to nearly nothing
11399          * and try again.  If that still fails, we punt.
11400          */
11401         vectors_left = hw->func_caps.num_msix_vectors;
11402         v_budget = 0;
11403
11404         /* reserve one vector for miscellaneous handler */
11405         if (vectors_left) {
11406                 v_budget++;
11407                 vectors_left--;
11408         }
11409
11410         /* reserve some vectors for the main PF traffic queues. Initially we
11411          * only reserve at most 50% of the available vectors, in the case that
11412          * the number of online CPUs is large. This ensures that we can enable
11413          * extra features as well. Once we've enabled the other features, we
11414          * will use any remaining vectors to reach as close as we can to the
11415          * number of online CPUs.
11416          */
11417         cpus = num_online_cpus();
11418         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11419         vectors_left -= pf->num_lan_msix;
11420
11421         /* reserve one vector for sideband flow director */
11422         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11423                 if (vectors_left) {
11424                         pf->num_fdsb_msix = 1;
11425                         v_budget++;
11426                         vectors_left--;
11427                 } else {
11428                         pf->num_fdsb_msix = 0;
11429                 }
11430         }
11431
11432         /* can we reserve enough for iWARP? */
11433         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11434                 iwarp_requested = pf->num_iwarp_msix;
11435
11436                 if (!vectors_left)
11437                         pf->num_iwarp_msix = 0;
11438                 else if (vectors_left < pf->num_iwarp_msix)
11439                         pf->num_iwarp_msix = 1;
11440                 v_budget += pf->num_iwarp_msix;
11441                 vectors_left -= pf->num_iwarp_msix;
11442         }
11443
11444         /* any vectors left over go for VMDq support */
11445         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
11446                 if (!vectors_left) {
11447                         pf->num_vmdq_msix = 0;
11448                         pf->num_vmdq_qps = 0;
11449                 } else {
11450                         int vmdq_vecs_wanted =
11451                                 pf->num_vmdq_vsis * pf->num_vmdq_qps;
11452                         int vmdq_vecs =
11453                                 min_t(int, vectors_left, vmdq_vecs_wanted);
11454
11455                         /* if we're short on vectors for what's desired, we limit
11456                          * the queues per vmdq.  If this is still more than are
11457                          * available, the user will need to change the number of
11458                          * queues/vectors used by the PF later with the ethtool
11459                          * channels command
11460                          */
11461                         if (vectors_left < vmdq_vecs_wanted) {
11462                                 pf->num_vmdq_qps = 1;
11463                                 vmdq_vecs_wanted = pf->num_vmdq_vsis;
11464                                 vmdq_vecs = min_t(int,
11465                                                   vectors_left,
11466                                                   vmdq_vecs_wanted);
11467                         }
11468                         pf->num_vmdq_msix = pf->num_vmdq_qps;
11469
11470                         v_budget += vmdq_vecs;
11471                         vectors_left -= vmdq_vecs;
11472                 }
11473         }
11474
11475         /* On systems with a large number of SMP cores, we previously limited
11476          * the number of vectors for num_lan_msix to be at most 50% of the
11477          * available vectors, to allow for other features. Now, we add back
11478          * the remaining vectors. However, we ensure that the total
11479          * num_lan_msix will not exceed num_online_cpus(). To do this, we
11480          * calculate the number of vectors we can add without going over the
11481          * cap of CPUs. For systems with a small number of CPUs this will be
11482          * zero.
11483          */
11484         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11485         pf->num_lan_msix += extra_vectors;
11486         vectors_left -= extra_vectors;
11487
11488         WARN(vectors_left < 0,
11489              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11490
11491         v_budget += pf->num_lan_msix;
11492         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11493                                    GFP_KERNEL);
11494         if (!pf->msix_entries)
11495                 return -ENOMEM;
11496
11497         for (i = 0; i < v_budget; i++)
11498                 pf->msix_entries[i].entry = i;
11499         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11500
11501         if (v_actual < I40E_MIN_MSIX) {
11502                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
11503                 kfree(pf->msix_entries);
11504                 pf->msix_entries = NULL;
11505                 pci_disable_msix(pf->pdev);
11506                 return -ENODEV;
11507
11508         } else if (v_actual == I40E_MIN_MSIX) {
11509                 /* Adjust for minimal MSIX use */
11510                 pf->num_vmdq_vsis = 0;
11511                 pf->num_vmdq_qps = 0;
11512                 pf->num_lan_qps = 1;
11513                 pf->num_lan_msix = 1;
11514
11515         } else if (v_actual != v_budget) {
11516                 /* If we have limited resources, we will start with no vectors
11517                  * for the special features and then allocate vectors to some
11518                  * of these features based on the policy and at the end disable
11519                  * the features that did not get any vectors.
11520                  */
11521                 int vec;
11522
11523                 dev_info(&pf->pdev->dev,
11524                          "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11525                          v_actual, v_budget);
11526                 /* reserve the misc vector */
11527                 vec = v_actual - 1;
11528
11529                 /* Scale vector usage down */
11530                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
11531                 pf->num_vmdq_vsis = 1;
11532                 pf->num_vmdq_qps = 1;
11533
11534                 /* partition out the remaining vectors */
11535                 switch (vec) {
11536                 case 2:
11537                         pf->num_lan_msix = 1;
11538                         break;
11539                 case 3:
11540                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11541                                 pf->num_lan_msix = 1;
11542                                 pf->num_iwarp_msix = 1;
11543                         } else {
11544                                 pf->num_lan_msix = 2;
11545                         }
11546                         break;
11547                 default:
11548                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11549                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
11550                                                  iwarp_requested);
11551                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
11552                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11553                         } else {
11554                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
11555                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11556                         }
11557                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11558                                 pf->num_fdsb_msix = 1;
11559                                 vec--;
11560                         }
11561                         pf->num_lan_msix = min_t(int,
11562                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11563                                                               pf->num_lan_msix);
11564                         pf->num_lan_qps = pf->num_lan_msix;
11565                         break;
11566                 }
11567         }
11568
11569         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
11570             (pf->num_fdsb_msix == 0)) {
11571                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11572                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11573                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11574         }
11575         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11576             (pf->num_vmdq_msix == 0)) {
11577                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11578                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
11579         }
11580
11581         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
11582             (pf->num_iwarp_msix == 0)) {
11583                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11584                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11585         }
11586         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11587                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11588                    pf->num_lan_msix,
11589                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
11590                    pf->num_fdsb_msix,
11591                    pf->num_iwarp_msix);
11592
11593         return v_actual;
11594 }
11595
11596 /**
11597  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
11598  * @vsi: the VSI being configured
11599  * @v_idx: index of the vector in the vsi struct
11600  *
11601  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
11602  **/
11603 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
11604 {
11605         struct i40e_q_vector *q_vector;
11606
11607         /* allocate q_vector */
11608         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
11609         if (!q_vector)
11610                 return -ENOMEM;
11611
11612         q_vector->vsi = vsi;
11613         q_vector->v_idx = v_idx;
11614         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
11615
11616         if (vsi->netdev)
11617                 netif_napi_add(vsi->netdev, &q_vector->napi,
11618                                i40e_napi_poll, NAPI_POLL_WEIGHT);
11619
11620         /* tie q_vector and vsi together */
11621         vsi->q_vectors[v_idx] = q_vector;
11622
11623         return 0;
11624 }
11625
11626 /**
11627  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
11628  * @vsi: the VSI being configured
11629  *
11630  * We allocate one q_vector per queue interrupt.  If allocation fails we
11631  * return -ENOMEM.
11632  **/
11633 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
11634 {
11635         struct i40e_pf *pf = vsi->back;
11636         int err, v_idx, num_q_vectors;
11637
11638         /* if not MSIX, give the one vector only to the LAN VSI */
11639         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11640                 num_q_vectors = vsi->num_q_vectors;
11641         else if (vsi == pf->vsi[pf->lan_vsi])
11642                 num_q_vectors = 1;
11643         else
11644                 return -EINVAL;
11645
11646         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
11647                 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
11648                 if (err)
11649                         goto err_out;
11650         }
11651
11652         return 0;
11653
11654 err_out:
11655         while (v_idx--)
11656                 i40e_free_q_vector(vsi, v_idx);
11657
11658         return err;
11659 }
11660
11661 /**
11662  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
11663  * @pf: board private structure to initialize
11664  **/
11665 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
11666 {
11667         int vectors = 0;
11668         ssize_t size;
11669
11670         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11671                 vectors = i40e_init_msix(pf);
11672                 if (vectors < 0) {
11673                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
11674                                        I40E_FLAG_IWARP_ENABLED  |
11675                                        I40E_FLAG_RSS_ENABLED    |
11676                                        I40E_FLAG_DCB_CAPABLE    |
11677                                        I40E_FLAG_DCB_ENABLED    |
11678                                        I40E_FLAG_SRIOV_ENABLED  |
11679                                        I40E_FLAG_FD_SB_ENABLED  |
11680                                        I40E_FLAG_FD_ATR_ENABLED |
11681                                        I40E_FLAG_VMDQ_ENABLED);
11682                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11683
11684                         /* rework the queue expectations without MSIX */
11685                         i40e_determine_queue_usage(pf);
11686                 }
11687         }
11688
11689         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11690             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
11691                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
11692                 vectors = pci_enable_msi(pf->pdev);
11693                 if (vectors < 0) {
11694                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
11695                                  vectors);
11696                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
11697                 }
11698                 vectors = 1;  /* one MSI or Legacy vector */
11699         }
11700
11701         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
11702                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
11703
11704         /* set up vector assignment tracking */
11705         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
11706         pf->irq_pile = kzalloc(size, GFP_KERNEL);
11707         if (!pf->irq_pile)
11708                 return -ENOMEM;
11709
11710         pf->irq_pile->num_entries = vectors;
11711         pf->irq_pile->search_hint = 0;
11712
11713         /* track first vector for misc interrupts, ignore return */
11714         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
11715
11716         return 0;
11717 }
11718
11719 /**
11720  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
11721  * @pf: private board data structure
11722  *
11723  * Restore the interrupt scheme that was cleared when we suspended the
11724  * device. This should be called during resume to re-allocate the q_vectors
11725  * and reacquire IRQs.
11726  */
11727 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
11728 {
11729         int err, i;
11730
11731         /* We cleared the MSI and MSI-X flags when disabling the old interrupt
11732          * scheme. We need to re-enabled them here in order to attempt to
11733          * re-acquire the MSI or MSI-X vectors
11734          */
11735         pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
11736
11737         err = i40e_init_interrupt_scheme(pf);
11738         if (err)
11739                 return err;
11740
11741         /* Now that we've re-acquired IRQs, we need to remap the vectors and
11742          * rings together again.
11743          */
11744         for (i = 0; i < pf->num_alloc_vsi; i++) {
11745                 if (pf->vsi[i]) {
11746                         err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
11747                         if (err)
11748                                 goto err_unwind;
11749                         i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
11750                 }
11751         }
11752
11753         err = i40e_setup_misc_vector(pf);
11754         if (err)
11755                 goto err_unwind;
11756
11757         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
11758                 i40e_client_update_msix_info(pf);
11759
11760         return 0;
11761
11762 err_unwind:
11763         while (i--) {
11764                 if (pf->vsi[i])
11765                         i40e_vsi_free_q_vectors(pf->vsi[i]);
11766         }
11767
11768         return err;
11769 }
11770
11771 /**
11772  * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
11773  * non queue events in recovery mode
11774  * @pf: board private structure
11775  *
11776  * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
11777  * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
11778  * This is handled differently than in recovery mode since no Tx/Rx resources
11779  * are being allocated.
11780  **/
11781 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
11782 {
11783         int err;
11784
11785         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11786                 err = i40e_setup_misc_vector(pf);
11787
11788                 if (err) {
11789                         dev_info(&pf->pdev->dev,
11790                                  "MSI-X misc vector request failed, error %d\n",
11791                                  err);
11792                         return err;
11793                 }
11794         } else {
11795                 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED;
11796
11797                 err = request_irq(pf->pdev->irq, i40e_intr, flags,
11798                                   pf->int_name, pf);
11799
11800                 if (err) {
11801                         dev_info(&pf->pdev->dev,
11802                                  "MSI/legacy misc vector request failed, error %d\n",
11803                                  err);
11804                         return err;
11805                 }
11806                 i40e_enable_misc_int_causes(pf);
11807                 i40e_irq_dynamic_enable_icr0(pf);
11808         }
11809
11810         return 0;
11811 }
11812
11813 /**
11814  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
11815  * @pf: board private structure
11816  *
11817  * This sets up the handler for MSIX 0, which is used to manage the
11818  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
11819  * when in MSI or Legacy interrupt mode.
11820  **/
11821 static int i40e_setup_misc_vector(struct i40e_pf *pf)
11822 {
11823         struct i40e_hw *hw = &pf->hw;
11824         int err = 0;
11825
11826         /* Only request the IRQ once, the first time through. */
11827         if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
11828                 err = request_irq(pf->msix_entries[0].vector,
11829                                   i40e_intr, 0, pf->int_name, pf);
11830                 if (err) {
11831                         clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
11832                         dev_info(&pf->pdev->dev,
11833                                  "request_irq for %s failed: %d\n",
11834                                  pf->int_name, err);
11835                         return -EFAULT;
11836                 }
11837         }
11838
11839         i40e_enable_misc_int_causes(pf);
11840
11841         /* associate no queues to the misc vector */
11842         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
11843         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
11844
11845         i40e_flush(hw);
11846
11847         i40e_irq_dynamic_enable_icr0(pf);
11848
11849         return err;
11850 }
11851
11852 /**
11853  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
11854  * @vsi: Pointer to vsi structure
11855  * @seed: Buffter to store the hash keys
11856  * @lut: Buffer to store the lookup table entries
11857  * @lut_size: Size of buffer to store the lookup table entries
11858  *
11859  * Return 0 on success, negative on failure
11860  */
11861 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
11862                            u8 *lut, u16 lut_size)
11863 {
11864         struct i40e_pf *pf = vsi->back;
11865         struct i40e_hw *hw = &pf->hw;
11866         int ret = 0;
11867
11868         if (seed) {
11869                 ret = i40e_aq_get_rss_key(hw, vsi->id,
11870                         (struct i40e_aqc_get_set_rss_key_data *)seed);
11871                 if (ret) {
11872                         dev_info(&pf->pdev->dev,
11873                                  "Cannot get RSS key, err %s aq_err %s\n",
11874                                  i40e_stat_str(&pf->hw, ret),
11875                                  i40e_aq_str(&pf->hw,
11876                                              pf->hw.aq.asq_last_status));
11877                         return ret;
11878                 }
11879         }
11880
11881         if (lut) {
11882                 bool pf_lut = vsi->type == I40E_VSI_MAIN;
11883
11884                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
11885                 if (ret) {
11886                         dev_info(&pf->pdev->dev,
11887                                  "Cannot get RSS lut, err %s aq_err %s\n",
11888                                  i40e_stat_str(&pf->hw, ret),
11889                                  i40e_aq_str(&pf->hw,
11890                                              pf->hw.aq.asq_last_status));
11891                         return ret;
11892                 }
11893         }
11894
11895         return ret;
11896 }
11897
11898 /**
11899  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
11900  * @vsi: Pointer to vsi structure
11901  * @seed: RSS hash seed
11902  * @lut: Lookup table
11903  * @lut_size: Lookup table size
11904  *
11905  * Returns 0 on success, negative on failure
11906  **/
11907 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
11908                                const u8 *lut, u16 lut_size)
11909 {
11910         struct i40e_pf *pf = vsi->back;
11911         struct i40e_hw *hw = &pf->hw;
11912         u16 vf_id = vsi->vf_id;
11913         u8 i;
11914
11915         /* Fill out hash function seed */
11916         if (seed) {
11917                 u32 *seed_dw = (u32 *)seed;
11918
11919                 if (vsi->type == I40E_VSI_MAIN) {
11920                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
11921                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
11922                 } else if (vsi->type == I40E_VSI_SRIOV) {
11923                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
11924                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
11925                 } else {
11926                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
11927                 }
11928         }
11929
11930         if (lut) {
11931                 u32 *lut_dw = (u32 *)lut;
11932
11933                 if (vsi->type == I40E_VSI_MAIN) {
11934                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
11935                                 return -EINVAL;
11936                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11937                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
11938                 } else if (vsi->type == I40E_VSI_SRIOV) {
11939                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
11940                                 return -EINVAL;
11941                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
11942                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
11943                 } else {
11944                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
11945                 }
11946         }
11947         i40e_flush(hw);
11948
11949         return 0;
11950 }
11951
11952 /**
11953  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
11954  * @vsi: Pointer to VSI structure
11955  * @seed: Buffer to store the keys
11956  * @lut: Buffer to store the lookup table entries
11957  * @lut_size: Size of buffer to store the lookup table entries
11958  *
11959  * Returns 0 on success, negative on failure
11960  */
11961 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
11962                             u8 *lut, u16 lut_size)
11963 {
11964         struct i40e_pf *pf = vsi->back;
11965         struct i40e_hw *hw = &pf->hw;
11966         u16 i;
11967
11968         if (seed) {
11969                 u32 *seed_dw = (u32 *)seed;
11970
11971                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
11972                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
11973         }
11974         if (lut) {
11975                 u32 *lut_dw = (u32 *)lut;
11976
11977                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
11978                         return -EINVAL;
11979                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11980                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
11981         }
11982
11983         return 0;
11984 }
11985
11986 /**
11987  * i40e_config_rss - Configure RSS keys and lut
11988  * @vsi: Pointer to VSI structure
11989  * @seed: RSS hash seed
11990  * @lut: Lookup table
11991  * @lut_size: Lookup table size
11992  *
11993  * Returns 0 on success, negative on failure
11994  */
11995 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
11996 {
11997         struct i40e_pf *pf = vsi->back;
11998
11999         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
12000                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
12001         else
12002                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
12003 }
12004
12005 /**
12006  * i40e_get_rss - Get RSS keys and lut
12007  * @vsi: Pointer to VSI structure
12008  * @seed: Buffer to store the keys
12009  * @lut: Buffer to store the lookup table entries
12010  * @lut_size: Size of buffer to store the lookup table entries
12011  *
12012  * Returns 0 on success, negative on failure
12013  */
12014 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12015 {
12016         struct i40e_pf *pf = vsi->back;
12017
12018         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
12019                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
12020         else
12021                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
12022 }
12023
12024 /**
12025  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
12026  * @pf: Pointer to board private structure
12027  * @lut: Lookup table
12028  * @rss_table_size: Lookup table size
12029  * @rss_size: Range of queue number for hashing
12030  */
12031 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
12032                        u16 rss_table_size, u16 rss_size)
12033 {
12034         u16 i;
12035
12036         for (i = 0; i < rss_table_size; i++)
12037                 lut[i] = i % rss_size;
12038 }
12039
12040 /**
12041  * i40e_pf_config_rss - Prepare for RSS if used
12042  * @pf: board private structure
12043  **/
12044 static int i40e_pf_config_rss(struct i40e_pf *pf)
12045 {
12046         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12047         u8 seed[I40E_HKEY_ARRAY_SIZE];
12048         u8 *lut;
12049         struct i40e_hw *hw = &pf->hw;
12050         u32 reg_val;
12051         u64 hena;
12052         int ret;
12053
12054         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
12055         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
12056                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
12057         hena |= i40e_pf_get_default_rss_hena(pf);
12058
12059         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
12060         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
12061
12062         /* Determine the RSS table size based on the hardware capabilities */
12063         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
12064         reg_val = (pf->rss_table_size == 512) ?
12065                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
12066                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
12067         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
12068
12069         /* Determine the RSS size of the VSI */
12070         if (!vsi->rss_size) {
12071                 u16 qcount;
12072                 /* If the firmware does something weird during VSI init, we
12073                  * could end up with zero TCs. Check for that to avoid
12074                  * divide-by-zero. It probably won't pass traffic, but it also
12075                  * won't panic.
12076                  */
12077                 qcount = vsi->num_queue_pairs /
12078                          (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
12079                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12080         }
12081         if (!vsi->rss_size)
12082                 return -EINVAL;
12083
12084         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
12085         if (!lut)
12086                 return -ENOMEM;
12087
12088         /* Use user configured lut if there is one, otherwise use default */
12089         if (vsi->rss_lut_user)
12090                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
12091         else
12092                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
12093
12094         /* Use user configured hash key if there is one, otherwise
12095          * use default.
12096          */
12097         if (vsi->rss_hkey_user)
12098                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
12099         else
12100                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
12101         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
12102         kfree(lut);
12103
12104         return ret;
12105 }
12106
12107 /**
12108  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12109  * @pf: board private structure
12110  * @queue_count: the requested queue count for rss.
12111  *
12112  * returns 0 if rss is not enabled, if enabled returns the final rss queue
12113  * count which may be different from the requested queue count.
12114  * Note: expects to be called while under rtnl_lock()
12115  **/
12116 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
12117 {
12118         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12119         int new_rss_size;
12120
12121         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
12122                 return 0;
12123
12124         queue_count = min_t(int, queue_count, num_online_cpus());
12125         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
12126
12127         if (queue_count != vsi->num_queue_pairs) {
12128                 u16 qcount;
12129
12130                 vsi->req_queue_pairs = queue_count;
12131                 i40e_prep_for_reset(pf);
12132
12133                 pf->alloc_rss_size = new_rss_size;
12134
12135                 i40e_reset_and_rebuild(pf, true, true);
12136
12137                 /* Discard the user configured hash keys and lut, if less
12138                  * queues are enabled.
12139                  */
12140                 if (queue_count < vsi->rss_size) {
12141                         i40e_clear_rss_config_user(vsi);
12142                         dev_dbg(&pf->pdev->dev,
12143                                 "discard user configured hash keys and lut\n");
12144                 }
12145
12146                 /* Reset vsi->rss_size, as number of enabled queues changed */
12147                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
12148                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12149
12150                 i40e_pf_config_rss(pf);
12151         }
12152         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
12153                  vsi->req_queue_pairs, pf->rss_size_max);
12154         return pf->alloc_rss_size;
12155 }
12156
12157 /**
12158  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12159  * @pf: board private structure
12160  **/
12161 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
12162 {
12163         i40e_status status;
12164         bool min_valid, max_valid;
12165         u32 max_bw, min_bw;
12166
12167         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
12168                                            &min_valid, &max_valid);
12169
12170         if (!status) {
12171                 if (min_valid)
12172                         pf->min_bw = min_bw;
12173                 if (max_valid)
12174                         pf->max_bw = max_bw;
12175         }
12176
12177         return status;
12178 }
12179
12180 /**
12181  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
12182  * @pf: board private structure
12183  **/
12184 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
12185 {
12186         struct i40e_aqc_configure_partition_bw_data bw_data;
12187         i40e_status status;
12188
12189         memset(&bw_data, 0, sizeof(bw_data));
12190
12191         /* Set the valid bit for this PF */
12192         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
12193         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
12194         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
12195
12196         /* Set the new bandwidths */
12197         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
12198
12199         return status;
12200 }
12201
12202 /**
12203  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
12204  * @pf: board private structure
12205  **/
12206 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
12207 {
12208         /* Commit temporary BW setting to permanent NVM image */
12209         enum i40e_admin_queue_err last_aq_status;
12210         i40e_status ret;
12211         u16 nvm_word;
12212
12213         if (pf->hw.partition_id != 1) {
12214                 dev_info(&pf->pdev->dev,
12215                          "Commit BW only works on partition 1! This is partition %d",
12216                          pf->hw.partition_id);
12217                 ret = I40E_NOT_SUPPORTED;
12218                 goto bw_commit_out;
12219         }
12220
12221         /* Acquire NVM for read access */
12222         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
12223         last_aq_status = pf->hw.aq.asq_last_status;
12224         if (ret) {
12225                 dev_info(&pf->pdev->dev,
12226                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
12227                          i40e_stat_str(&pf->hw, ret),
12228                          i40e_aq_str(&pf->hw, last_aq_status));
12229                 goto bw_commit_out;
12230         }
12231
12232         /* Read word 0x10 of NVM - SW compatibility word 1 */
12233         ret = i40e_aq_read_nvm(&pf->hw,
12234                                I40E_SR_NVM_CONTROL_WORD,
12235                                0x10, sizeof(nvm_word), &nvm_word,
12236                                false, NULL);
12237         /* Save off last admin queue command status before releasing
12238          * the NVM
12239          */
12240         last_aq_status = pf->hw.aq.asq_last_status;
12241         i40e_release_nvm(&pf->hw);
12242         if (ret) {
12243                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
12244                          i40e_stat_str(&pf->hw, ret),
12245                          i40e_aq_str(&pf->hw, last_aq_status));
12246                 goto bw_commit_out;
12247         }
12248
12249         /* Wait a bit for NVM release to complete */
12250         msleep(50);
12251
12252         /* Acquire NVM for write access */
12253         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
12254         last_aq_status = pf->hw.aq.asq_last_status;
12255         if (ret) {
12256                 dev_info(&pf->pdev->dev,
12257                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
12258                          i40e_stat_str(&pf->hw, ret),
12259                          i40e_aq_str(&pf->hw, last_aq_status));
12260                 goto bw_commit_out;
12261         }
12262         /* Write it back out unchanged to initiate update NVM,
12263          * which will force a write of the shadow (alt) RAM to
12264          * the NVM - thus storing the bandwidth values permanently.
12265          */
12266         ret = i40e_aq_update_nvm(&pf->hw,
12267                                  I40E_SR_NVM_CONTROL_WORD,
12268                                  0x10, sizeof(nvm_word),
12269                                  &nvm_word, true, 0, NULL);
12270         /* Save off last admin queue command status before releasing
12271          * the NVM
12272          */
12273         last_aq_status = pf->hw.aq.asq_last_status;
12274         i40e_release_nvm(&pf->hw);
12275         if (ret)
12276                 dev_info(&pf->pdev->dev,
12277                          "BW settings NOT SAVED, err %s aq_err %s\n",
12278                          i40e_stat_str(&pf->hw, ret),
12279                          i40e_aq_str(&pf->hw, last_aq_status));
12280 bw_commit_out:
12281
12282         return ret;
12283 }
12284
12285 /**
12286  * i40e_is_total_port_shutdown_enabled - read NVM and return value
12287  * if total port shutdown feature is enabled for this PF
12288  * @pf: board private structure
12289  **/
12290 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12291 {
12292 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED        BIT(4)
12293 #define I40E_FEATURES_ENABLE_PTR                0x2A
12294 #define I40E_CURRENT_SETTING_PTR                0x2B
12295 #define I40E_LINK_BEHAVIOR_WORD_OFFSET          0x2D
12296 #define I40E_LINK_BEHAVIOR_WORD_LENGTH          0x1
12297 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED    BIT(0)
12298 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH      4
12299         i40e_status read_status = I40E_SUCCESS;
12300         u16 sr_emp_sr_settings_ptr = 0;
12301         u16 features_enable = 0;
12302         u16 link_behavior = 0;
12303         bool ret = false;
12304
12305         read_status = i40e_read_nvm_word(&pf->hw,
12306                                          I40E_SR_EMP_SR_SETTINGS_PTR,
12307                                          &sr_emp_sr_settings_ptr);
12308         if (read_status)
12309                 goto err_nvm;
12310         read_status = i40e_read_nvm_word(&pf->hw,
12311                                          sr_emp_sr_settings_ptr +
12312                                          I40E_FEATURES_ENABLE_PTR,
12313                                          &features_enable);
12314         if (read_status)
12315                 goto err_nvm;
12316         if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12317                 read_status = i40e_read_nvm_module_data(&pf->hw,
12318                                                         I40E_SR_EMP_SR_SETTINGS_PTR,
12319                                                         I40E_CURRENT_SETTING_PTR,
12320                                                         I40E_LINK_BEHAVIOR_WORD_OFFSET,
12321                                                         I40E_LINK_BEHAVIOR_WORD_LENGTH,
12322                                                         &link_behavior);
12323                 if (read_status)
12324                         goto err_nvm;
12325                 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12326                 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12327         }
12328         return ret;
12329
12330 err_nvm:
12331         dev_warn(&pf->pdev->dev,
12332                  "total-port-shutdown feature is off due to read nvm error: %s\n",
12333                  i40e_stat_str(&pf->hw, read_status));
12334         return ret;
12335 }
12336
12337 /**
12338  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12339  * @pf: board private structure to initialize
12340  *
12341  * i40e_sw_init initializes the Adapter private data structure.
12342  * Fields are initialized based on PCI device information and
12343  * OS network device settings (MTU size).
12344  **/
12345 static int i40e_sw_init(struct i40e_pf *pf)
12346 {
12347         int err = 0;
12348         int size;
12349
12350         /* Set default capability flags */
12351         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
12352                     I40E_FLAG_MSI_ENABLED     |
12353                     I40E_FLAG_MSIX_ENABLED;
12354
12355         /* Set default ITR */
12356         pf->rx_itr_default = I40E_ITR_RX_DEF;
12357         pf->tx_itr_default = I40E_ITR_TX_DEF;
12358
12359         /* Depending on PF configurations, it is possible that the RSS
12360          * maximum might end up larger than the available queues
12361          */
12362         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12363         pf->alloc_rss_size = 1;
12364         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12365         pf->rss_size_max = min_t(int, pf->rss_size_max,
12366                                  pf->hw.func_caps.num_tx_qp);
12367         if (pf->hw.func_caps.rss) {
12368                 pf->flags |= I40E_FLAG_RSS_ENABLED;
12369                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12370                                            num_online_cpus());
12371         }
12372
12373         /* MFP mode enabled */
12374         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12375                 pf->flags |= I40E_FLAG_MFP_ENABLED;
12376                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12377                 if (i40e_get_partition_bw_setting(pf)) {
12378                         dev_warn(&pf->pdev->dev,
12379                                  "Could not get partition bw settings\n");
12380                 } else {
12381                         dev_info(&pf->pdev->dev,
12382                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
12383                                  pf->min_bw, pf->max_bw);
12384
12385                         /* nudge the Tx scheduler */
12386                         i40e_set_partition_bw_setting(pf);
12387                 }
12388         }
12389
12390         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12391             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
12392                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
12393                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
12394                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
12395                     pf->hw.num_partitions > 1)
12396                         dev_info(&pf->pdev->dev,
12397                                  "Flow Director Sideband mode Disabled in MFP mode\n");
12398                 else
12399                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12400                 pf->fdir_pf_filter_count =
12401                                  pf->hw.func_caps.fd_filters_guaranteed;
12402                 pf->hw.fdir_shared_filter_count =
12403                                  pf->hw.func_caps.fd_filters_best_effort;
12404         }
12405
12406         if (pf->hw.mac.type == I40E_MAC_X722) {
12407                 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
12408                                     I40E_HW_128_QP_RSS_CAPABLE |
12409                                     I40E_HW_ATR_EVICT_CAPABLE |
12410                                     I40E_HW_WB_ON_ITR_CAPABLE |
12411                                     I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
12412                                     I40E_HW_NO_PCI_LINK_CHECK |
12413                                     I40E_HW_USE_SET_LLDP_MIB |
12414                                     I40E_HW_GENEVE_OFFLOAD_CAPABLE |
12415                                     I40E_HW_PTP_L4_CAPABLE |
12416                                     I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
12417                                     I40E_HW_OUTER_UDP_CSUM_CAPABLE);
12418
12419 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
12420                 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
12421                     I40E_FDEVICT_PCTYPE_DEFAULT) {
12422                         dev_warn(&pf->pdev->dev,
12423                                  "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
12424                         pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
12425                 }
12426         } else if ((pf->hw.aq.api_maj_ver > 1) ||
12427                    ((pf->hw.aq.api_maj_ver == 1) &&
12428                     (pf->hw.aq.api_min_ver > 4))) {
12429                 /* Supported in FW API version higher than 1.4 */
12430                 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
12431         }
12432
12433         /* Enable HW ATR eviction if possible */
12434         if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
12435                 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
12436
12437         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12438             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
12439             (pf->hw.aq.fw_maj_ver < 4))) {
12440                 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
12441                 /* No DCB support  for FW < v4.33 */
12442                 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
12443         }
12444
12445         /* Disable FW LLDP if FW < v4.3 */
12446         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12447             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
12448             (pf->hw.aq.fw_maj_ver < 4)))
12449                 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
12450
12451         /* Use the FW Set LLDP MIB API if FW > v4.40 */
12452         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12453             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
12454             (pf->hw.aq.fw_maj_ver >= 5)))
12455                 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
12456
12457         /* Enable PTP L4 if FW > v6.0 */
12458         if (pf->hw.mac.type == I40E_MAC_XL710 &&
12459             pf->hw.aq.fw_maj_ver >= 6)
12460                 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
12461
12462         if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12463                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12464                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
12465                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12466         }
12467
12468         if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12469                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
12470                 /* IWARP needs one extra vector for CQP just like MISC.*/
12471                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12472         }
12473         /* Stopping FW LLDP engine is supported on XL710 and X722
12474          * starting from FW versions determined in i40e_init_adminq.
12475          * Stopping the FW LLDP engine is not supported on XL710
12476          * if NPAR is functioning so unset this hw flag in this case.
12477          */
12478         if (pf->hw.mac.type == I40E_MAC_XL710 &&
12479             pf->hw.func_caps.npar_enable &&
12480             (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
12481                 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE;
12482
12483 #ifdef CONFIG_PCI_IOV
12484         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12485                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12486                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
12487                 pf->num_req_vfs = min_t(int,
12488                                         pf->hw.func_caps.num_vfs,
12489                                         I40E_MAX_VF_COUNT);
12490         }
12491 #endif /* CONFIG_PCI_IOV */
12492         pf->eeprom_version = 0xDEAD;
12493         pf->lan_veb = I40E_NO_VEB;
12494         pf->lan_vsi = I40E_NO_VSI;
12495
12496         /* By default FW has this off for performance reasons */
12497         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
12498
12499         /* set up queue assignment tracking */
12500         size = sizeof(struct i40e_lump_tracking)
12501                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12502         pf->qp_pile = kzalloc(size, GFP_KERNEL);
12503         if (!pf->qp_pile) {
12504                 err = -ENOMEM;
12505                 goto sw_init_done;
12506         }
12507         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12508         pf->qp_pile->search_hint = 0;
12509
12510         pf->tx_timeout_recovery_level = 1;
12511
12512         if (pf->hw.mac.type != I40E_MAC_X722 &&
12513             i40e_is_total_port_shutdown_enabled(pf)) {
12514                 /* Link down on close must be on when total port shutdown
12515                  * is enabled for a given port
12516                  */
12517                 pf->flags |= (I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED |
12518                               I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED);
12519                 dev_info(&pf->pdev->dev,
12520                          "total-port-shutdown was enabled, link-down-on-close is forced on\n");
12521         }
12522         mutex_init(&pf->switch_mutex);
12523
12524 sw_init_done:
12525         return err;
12526 }
12527
12528 /**
12529  * i40e_set_ntuple - set the ntuple feature flag and take action
12530  * @pf: board private structure to initialize
12531  * @features: the feature set that the stack is suggesting
12532  *
12533  * returns a bool to indicate if reset needs to happen
12534  **/
12535 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12536 {
12537         bool need_reset = false;
12538
12539         /* Check if Flow Director n-tuple support was enabled or disabled.  If
12540          * the state changed, we need to reset.
12541          */
12542         if (features & NETIF_F_NTUPLE) {
12543                 /* Enable filters and mark for reset */
12544                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
12545                         need_reset = true;
12546                 /* enable FD_SB only if there is MSI-X vector and no cloud
12547                  * filters exist
12548                  */
12549                 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12550                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12551                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
12552                 }
12553         } else {
12554                 /* turn off filters, mark for reset and clear SW filter list */
12555                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
12556                         need_reset = true;
12557                         i40e_fdir_filter_exit(pf);
12558                 }
12559                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
12560                 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12561                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
12562
12563                 /* reset fd counters */
12564                 pf->fd_add_err = 0;
12565                 pf->fd_atr_cnt = 0;
12566                 /* if ATR was auto disabled it can be re-enabled. */
12567                 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12568                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
12569                             (I40E_DEBUG_FD & pf->hw.debug_mask))
12570                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12571         }
12572         return need_reset;
12573 }
12574
12575 /**
12576  * i40e_clear_rss_lut - clear the rx hash lookup table
12577  * @vsi: the VSI being configured
12578  **/
12579 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12580 {
12581         struct i40e_pf *pf = vsi->back;
12582         struct i40e_hw *hw = &pf->hw;
12583         u16 vf_id = vsi->vf_id;
12584         u8 i;
12585
12586         if (vsi->type == I40E_VSI_MAIN) {
12587                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12588                         wr32(hw, I40E_PFQF_HLUT(i), 0);
12589         } else if (vsi->type == I40E_VSI_SRIOV) {
12590                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12591                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12592         } else {
12593                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12594         }
12595 }
12596
12597 /**
12598  * i40e_set_features - set the netdev feature flags
12599  * @netdev: ptr to the netdev being adjusted
12600  * @features: the feature set that the stack is suggesting
12601  * Note: expects to be called while under rtnl_lock()
12602  **/
12603 static int i40e_set_features(struct net_device *netdev,
12604                              netdev_features_t features)
12605 {
12606         struct i40e_netdev_priv *np = netdev_priv(netdev);
12607         struct i40e_vsi *vsi = np->vsi;
12608         struct i40e_pf *pf = vsi->back;
12609         bool need_reset;
12610
12611         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12612                 i40e_pf_config_rss(pf);
12613         else if (!(features & NETIF_F_RXHASH) &&
12614                  netdev->features & NETIF_F_RXHASH)
12615                 i40e_clear_rss_lut(vsi);
12616
12617         if (features & NETIF_F_HW_VLAN_CTAG_RX)
12618                 i40e_vlan_stripping_enable(vsi);
12619         else
12620                 i40e_vlan_stripping_disable(vsi);
12621
12622         if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12623                 dev_err(&pf->pdev->dev,
12624                         "Offloaded tc filters active, can't turn hw_tc_offload off");
12625                 return -EINVAL;
12626         }
12627
12628         if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12629                 i40e_del_all_macvlans(vsi);
12630
12631         need_reset = i40e_set_ntuple(pf, features);
12632
12633         if (need_reset)
12634                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12635
12636         return 0;
12637 }
12638
12639 static int i40e_udp_tunnel_set_port(struct net_device *netdev,
12640                                     unsigned int table, unsigned int idx,
12641                                     struct udp_tunnel_info *ti)
12642 {
12643         struct i40e_netdev_priv *np = netdev_priv(netdev);
12644         struct i40e_hw *hw = &np->vsi->back->hw;
12645         u8 type, filter_index;
12646         i40e_status ret;
12647
12648         type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
12649                                                    I40E_AQC_TUNNEL_TYPE_NGE;
12650
12651         ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,
12652                                      NULL);
12653         if (ret) {
12654                 netdev_info(netdev, "add UDP port failed, err %s aq_err %s\n",
12655                             i40e_stat_str(hw, ret),
12656                             i40e_aq_str(hw, hw->aq.asq_last_status));
12657                 return -EIO;
12658         }
12659
12660         udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);
12661         return 0;
12662 }
12663
12664 static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
12665                                       unsigned int table, unsigned int idx,
12666                                       struct udp_tunnel_info *ti)
12667 {
12668         struct i40e_netdev_priv *np = netdev_priv(netdev);
12669         struct i40e_hw *hw = &np->vsi->back->hw;
12670         i40e_status ret;
12671
12672         ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);
12673         if (ret) {
12674                 netdev_info(netdev, "delete UDP port failed, err %s aq_err %s\n",
12675                             i40e_stat_str(hw, ret),
12676                             i40e_aq_str(hw, hw->aq.asq_last_status));
12677                 return -EIO;
12678         }
12679
12680         return 0;
12681 }
12682
12683 static int i40e_get_phys_port_id(struct net_device *netdev,
12684                                  struct netdev_phys_item_id *ppid)
12685 {
12686         struct i40e_netdev_priv *np = netdev_priv(netdev);
12687         struct i40e_pf *pf = np->vsi->back;
12688         struct i40e_hw *hw = &pf->hw;
12689
12690         if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
12691                 return -EOPNOTSUPP;
12692
12693         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
12694         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
12695
12696         return 0;
12697 }
12698
12699 /**
12700  * i40e_ndo_fdb_add - add an entry to the hardware database
12701  * @ndm: the input from the stack
12702  * @tb: pointer to array of nladdr (unused)
12703  * @dev: the net device pointer
12704  * @addr: the MAC address entry being added
12705  * @vid: VLAN ID
12706  * @flags: instructions from stack about fdb operation
12707  * @extack: netlink extended ack, unused currently
12708  */
12709 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
12710                             struct net_device *dev,
12711                             const unsigned char *addr, u16 vid,
12712                             u16 flags,
12713                             struct netlink_ext_ack *extack)
12714 {
12715         struct i40e_netdev_priv *np = netdev_priv(dev);
12716         struct i40e_pf *pf = np->vsi->back;
12717         int err = 0;
12718
12719         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
12720                 return -EOPNOTSUPP;
12721
12722         if (vid) {
12723                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
12724                 return -EINVAL;
12725         }
12726
12727         /* Hardware does not support aging addresses so if a
12728          * ndm_state is given only allow permanent addresses
12729          */
12730         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
12731                 netdev_info(dev, "FDB only supports static addresses\n");
12732                 return -EINVAL;
12733         }
12734
12735         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
12736                 err = dev_uc_add_excl(dev, addr);
12737         else if (is_multicast_ether_addr(addr))
12738                 err = dev_mc_add_excl(dev, addr);
12739         else
12740                 err = -EINVAL;
12741
12742         /* Only return duplicate errors if NLM_F_EXCL is set */
12743         if (err == -EEXIST && !(flags & NLM_F_EXCL))
12744                 err = 0;
12745
12746         return err;
12747 }
12748
12749 /**
12750  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
12751  * @dev: the netdev being configured
12752  * @nlh: RTNL message
12753  * @flags: bridge flags
12754  * @extack: netlink extended ack
12755  *
12756  * Inserts a new hardware bridge if not already created and
12757  * enables the bridging mode requested (VEB or VEPA). If the
12758  * hardware bridge has already been inserted and the request
12759  * is to change the mode then that requires a PF reset to
12760  * allow rebuild of the components with required hardware
12761  * bridge mode enabled.
12762  *
12763  * Note: expects to be called while under rtnl_lock()
12764  **/
12765 static int i40e_ndo_bridge_setlink(struct net_device *dev,
12766                                    struct nlmsghdr *nlh,
12767                                    u16 flags,
12768                                    struct netlink_ext_ack *extack)
12769 {
12770         struct i40e_netdev_priv *np = netdev_priv(dev);
12771         struct i40e_vsi *vsi = np->vsi;
12772         struct i40e_pf *pf = vsi->back;
12773         struct i40e_veb *veb = NULL;
12774         struct nlattr *attr, *br_spec;
12775         int i, rem;
12776
12777         /* Only for PF VSI for now */
12778         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12779                 return -EOPNOTSUPP;
12780
12781         /* Find the HW bridge for PF VSI */
12782         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12783                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12784                         veb = pf->veb[i];
12785         }
12786
12787         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12788
12789         nla_for_each_nested(attr, br_spec, rem) {
12790                 __u16 mode;
12791
12792                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12793                         continue;
12794
12795                 mode = nla_get_u16(attr);
12796                 if ((mode != BRIDGE_MODE_VEPA) &&
12797                     (mode != BRIDGE_MODE_VEB))
12798                         return -EINVAL;
12799
12800                 /* Insert a new HW bridge */
12801                 if (!veb) {
12802                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
12803                                              vsi->tc_config.enabled_tc);
12804                         if (veb) {
12805                                 veb->bridge_mode = mode;
12806                                 i40e_config_bridge_mode(veb);
12807                         } else {
12808                                 /* No Bridge HW offload available */
12809                                 return -ENOENT;
12810                         }
12811                         break;
12812                 } else if (mode != veb->bridge_mode) {
12813                         /* Existing HW bridge but different mode needs reset */
12814                         veb->bridge_mode = mode;
12815                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
12816                         if (mode == BRIDGE_MODE_VEB)
12817                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
12818                         else
12819                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
12820                         i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12821                         break;
12822                 }
12823         }
12824
12825         return 0;
12826 }
12827
12828 /**
12829  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
12830  * @skb: skb buff
12831  * @pid: process id
12832  * @seq: RTNL message seq #
12833  * @dev: the netdev being configured
12834  * @filter_mask: unused
12835  * @nlflags: netlink flags passed in
12836  *
12837  * Return the mode in which the hardware bridge is operating in
12838  * i.e VEB or VEPA.
12839  **/
12840 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
12841                                    struct net_device *dev,
12842                                    u32 __always_unused filter_mask,
12843                                    int nlflags)
12844 {
12845         struct i40e_netdev_priv *np = netdev_priv(dev);
12846         struct i40e_vsi *vsi = np->vsi;
12847         struct i40e_pf *pf = vsi->back;
12848         struct i40e_veb *veb = NULL;
12849         int i;
12850
12851         /* Only for PF VSI for now */
12852         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12853                 return -EOPNOTSUPP;
12854
12855         /* Find the HW bridge for the PF VSI */
12856         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12857                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12858                         veb = pf->veb[i];
12859         }
12860
12861         if (!veb)
12862                 return 0;
12863
12864         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
12865                                        0, 0, nlflags, filter_mask, NULL);
12866 }
12867
12868 /**
12869  * i40e_features_check - Validate encapsulated packet conforms to limits
12870  * @skb: skb buff
12871  * @dev: This physical port's netdev
12872  * @features: Offload features that the stack believes apply
12873  **/
12874 static netdev_features_t i40e_features_check(struct sk_buff *skb,
12875                                              struct net_device *dev,
12876                                              netdev_features_t features)
12877 {
12878         size_t len;
12879
12880         /* No point in doing any of this if neither checksum nor GSO are
12881          * being requested for this frame.  We can rule out both by just
12882          * checking for CHECKSUM_PARTIAL
12883          */
12884         if (skb->ip_summed != CHECKSUM_PARTIAL)
12885                 return features;
12886
12887         /* We cannot support GSO if the MSS is going to be less than
12888          * 64 bytes.  If it is then we need to drop support for GSO.
12889          */
12890         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
12891                 features &= ~NETIF_F_GSO_MASK;
12892
12893         /* MACLEN can support at most 63 words */
12894         len = skb_network_header(skb) - skb->data;
12895         if (len & ~(63 * 2))
12896                 goto out_err;
12897
12898         /* IPLEN and EIPLEN can support at most 127 dwords */
12899         len = skb_transport_header(skb) - skb_network_header(skb);
12900         if (len & ~(127 * 4))
12901                 goto out_err;
12902
12903         if (skb->encapsulation) {
12904                 /* L4TUNLEN can support 127 words */
12905                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
12906                 if (len & ~(127 * 2))
12907                         goto out_err;
12908
12909                 /* IPLEN can support at most 127 dwords */
12910                 len = skb_inner_transport_header(skb) -
12911                       skb_inner_network_header(skb);
12912                 if (len & ~(127 * 4))
12913                         goto out_err;
12914         }
12915
12916         /* No need to validate L4LEN as TCP is the only protocol with a
12917          * a flexible value and we support all possible values supported
12918          * by TCP, which is at most 15 dwords
12919          */
12920
12921         return features;
12922 out_err:
12923         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
12924 }
12925
12926 /**
12927  * i40e_xdp_setup - add/remove an XDP program
12928  * @vsi: VSI to changed
12929  * @prog: XDP program
12930  * @extack: netlink extended ack
12931  **/
12932 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
12933                           struct netlink_ext_ack *extack)
12934 {
12935         int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
12936         struct i40e_pf *pf = vsi->back;
12937         struct bpf_prog *old_prog;
12938         bool need_reset;
12939         int i;
12940
12941         /* Don't allow frames that span over multiple buffers */
12942         if (frame_size > vsi->rx_buf_len) {
12943                 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
12944                 return -EINVAL;
12945         }
12946
12947         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
12948         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
12949
12950         if (need_reset)
12951                 i40e_prep_for_reset(pf);
12952
12953         old_prog = xchg(&vsi->xdp_prog, prog);
12954
12955         if (need_reset) {
12956                 if (!prog)
12957                         /* Wait until ndo_xsk_wakeup completes. */
12958                         synchronize_rcu();
12959                 i40e_reset_and_rebuild(pf, true, true);
12960         }
12961
12962         for (i = 0; i < vsi->num_queue_pairs; i++)
12963                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
12964
12965         if (old_prog)
12966                 bpf_prog_put(old_prog);
12967
12968         /* Kick start the NAPI context if there is an AF_XDP socket open
12969          * on that queue id. This so that receiving will start.
12970          */
12971         if (need_reset && prog)
12972                 for (i = 0; i < vsi->num_queue_pairs; i++)
12973                         if (vsi->xdp_rings[i]->xsk_pool)
12974                                 (void)i40e_xsk_wakeup(vsi->netdev, i,
12975                                                       XDP_WAKEUP_RX);
12976
12977         return 0;
12978 }
12979
12980 /**
12981  * i40e_enter_busy_conf - Enters busy config state
12982  * @vsi: vsi
12983  *
12984  * Returns 0 on success, <0 for failure.
12985  **/
12986 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
12987 {
12988         struct i40e_pf *pf = vsi->back;
12989         int timeout = 50;
12990
12991         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
12992                 timeout--;
12993                 if (!timeout)
12994                         return -EBUSY;
12995                 usleep_range(1000, 2000);
12996         }
12997
12998         return 0;
12999 }
13000
13001 /**
13002  * i40e_exit_busy_conf - Exits busy config state
13003  * @vsi: vsi
13004  **/
13005 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
13006 {
13007         struct i40e_pf *pf = vsi->back;
13008
13009         clear_bit(__I40E_CONFIG_BUSY, pf->state);
13010 }
13011
13012 /**
13013  * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13014  * @vsi: vsi
13015  * @queue_pair: queue pair
13016  **/
13017 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
13018 {
13019         memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
13020                sizeof(vsi->rx_rings[queue_pair]->rx_stats));
13021         memset(&vsi->tx_rings[queue_pair]->stats, 0,
13022                sizeof(vsi->tx_rings[queue_pair]->stats));
13023         if (i40e_enabled_xdp_vsi(vsi)) {
13024                 memset(&vsi->xdp_rings[queue_pair]->stats, 0,
13025                        sizeof(vsi->xdp_rings[queue_pair]->stats));
13026         }
13027 }
13028
13029 /**
13030  * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13031  * @vsi: vsi
13032  * @queue_pair: queue pair
13033  **/
13034 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
13035 {
13036         i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
13037         if (i40e_enabled_xdp_vsi(vsi)) {
13038                 /* Make sure that in-progress ndo_xdp_xmit calls are
13039                  * completed.
13040                  */
13041                 synchronize_rcu();
13042                 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
13043         }
13044         i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13045 }
13046
13047 /**
13048  * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13049  * @vsi: vsi
13050  * @queue_pair: queue pair
13051  * @enable: true for enable, false for disable
13052  **/
13053 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
13054                                         bool enable)
13055 {
13056         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13057         struct i40e_q_vector *q_vector = rxr->q_vector;
13058
13059         if (!vsi->netdev)
13060                 return;
13061
13062         /* All rings in a qp belong to the same qvector. */
13063         if (q_vector->rx.ring || q_vector->tx.ring) {
13064                 if (enable)
13065                         napi_enable(&q_vector->napi);
13066                 else
13067                         napi_disable(&q_vector->napi);
13068         }
13069 }
13070
13071 /**
13072  * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13073  * @vsi: vsi
13074  * @queue_pair: queue pair
13075  * @enable: true for enable, false for disable
13076  *
13077  * Returns 0 on success, <0 on failure.
13078  **/
13079 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
13080                                         bool enable)
13081 {
13082         struct i40e_pf *pf = vsi->back;
13083         int pf_q, ret = 0;
13084
13085         pf_q = vsi->base_queue + queue_pair;
13086         ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
13087                                      false /*is xdp*/, enable);
13088         if (ret) {
13089                 dev_info(&pf->pdev->dev,
13090                          "VSI seid %d Tx ring %d %sable timeout\n",
13091                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13092                 return ret;
13093         }
13094
13095         i40e_control_rx_q(pf, pf_q, enable);
13096         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
13097         if (ret) {
13098                 dev_info(&pf->pdev->dev,
13099                          "VSI seid %d Rx ring %d %sable timeout\n",
13100                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13101                 return ret;
13102         }
13103
13104         /* Due to HW errata, on Rx disable only, the register can
13105          * indicate done before it really is. Needs 50ms to be sure
13106          */
13107         if (!enable)
13108                 mdelay(50);
13109
13110         if (!i40e_enabled_xdp_vsi(vsi))
13111                 return ret;
13112
13113         ret = i40e_control_wait_tx_q(vsi->seid, pf,
13114                                      pf_q + vsi->alloc_queue_pairs,
13115                                      true /*is xdp*/, enable);
13116         if (ret) {
13117                 dev_info(&pf->pdev->dev,
13118                          "VSI seid %d XDP Tx ring %d %sable timeout\n",
13119                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13120         }
13121
13122         return ret;
13123 }
13124
13125 /**
13126  * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13127  * @vsi: vsi
13128  * @queue_pair: queue_pair
13129  **/
13130 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
13131 {
13132         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13133         struct i40e_pf *pf = vsi->back;
13134         struct i40e_hw *hw = &pf->hw;
13135
13136         /* All rings in a qp belong to the same qvector. */
13137         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
13138                 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
13139         else
13140                 i40e_irq_dynamic_enable_icr0(pf);
13141
13142         i40e_flush(hw);
13143 }
13144
13145 /**
13146  * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13147  * @vsi: vsi
13148  * @queue_pair: queue_pair
13149  **/
13150 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
13151 {
13152         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13153         struct i40e_pf *pf = vsi->back;
13154         struct i40e_hw *hw = &pf->hw;
13155
13156         /* For simplicity, instead of removing the qp interrupt causes
13157          * from the interrupt linked list, we simply disable the interrupt, and
13158          * leave the list intact.
13159          *
13160          * All rings in a qp belong to the same qvector.
13161          */
13162         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
13163                 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
13164
13165                 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
13166                 i40e_flush(hw);
13167                 synchronize_irq(pf->msix_entries[intpf].vector);
13168         } else {
13169                 /* Legacy and MSI mode - this stops all interrupt handling */
13170                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
13171                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
13172                 i40e_flush(hw);
13173                 synchronize_irq(pf->pdev->irq);
13174         }
13175 }
13176
13177 /**
13178  * i40e_queue_pair_disable - Disables a queue pair
13179  * @vsi: vsi
13180  * @queue_pair: queue pair
13181  *
13182  * Returns 0 on success, <0 on failure.
13183  **/
13184 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
13185 {
13186         int err;
13187
13188         err = i40e_enter_busy_conf(vsi);
13189         if (err)
13190                 return err;
13191
13192         i40e_queue_pair_disable_irq(vsi, queue_pair);
13193         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
13194         i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
13195         i40e_queue_pair_clean_rings(vsi, queue_pair);
13196         i40e_queue_pair_reset_stats(vsi, queue_pair);
13197
13198         return err;
13199 }
13200
13201 /**
13202  * i40e_queue_pair_enable - Enables a queue pair
13203  * @vsi: vsi
13204  * @queue_pair: queue pair
13205  *
13206  * Returns 0 on success, <0 on failure.
13207  **/
13208 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
13209 {
13210         int err;
13211
13212         err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
13213         if (err)
13214                 return err;
13215
13216         if (i40e_enabled_xdp_vsi(vsi)) {
13217                 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
13218                 if (err)
13219                         return err;
13220         }
13221
13222         err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
13223         if (err)
13224                 return err;
13225
13226         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
13227         i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
13228         i40e_queue_pair_enable_irq(vsi, queue_pair);
13229
13230         i40e_exit_busy_conf(vsi);
13231
13232         return err;
13233 }
13234
13235 /**
13236  * i40e_xdp - implements ndo_bpf for i40e
13237  * @dev: netdevice
13238  * @xdp: XDP command
13239  **/
13240 static int i40e_xdp(struct net_device *dev,
13241                     struct netdev_bpf *xdp)
13242 {
13243         struct i40e_netdev_priv *np = netdev_priv(dev);
13244         struct i40e_vsi *vsi = np->vsi;
13245
13246         if (vsi->type != I40E_VSI_MAIN)
13247                 return -EINVAL;
13248
13249         switch (xdp->command) {
13250         case XDP_SETUP_PROG:
13251                 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);
13252         case XDP_SETUP_XSK_POOL:
13253                 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,
13254                                            xdp->xsk.queue_id);
13255         default:
13256                 return -EINVAL;
13257         }
13258 }
13259
13260 static const struct net_device_ops i40e_netdev_ops = {
13261         .ndo_open               = i40e_open,
13262         .ndo_stop               = i40e_close,
13263         .ndo_start_xmit         = i40e_lan_xmit_frame,
13264         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
13265         .ndo_set_rx_mode        = i40e_set_rx_mode,
13266         .ndo_validate_addr      = eth_validate_addr,
13267         .ndo_set_mac_address    = i40e_set_mac,
13268         .ndo_change_mtu         = i40e_change_mtu,
13269         .ndo_do_ioctl           = i40e_ioctl,
13270         .ndo_tx_timeout         = i40e_tx_timeout,
13271         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
13272         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
13273 #ifdef CONFIG_NET_POLL_CONTROLLER
13274         .ndo_poll_controller    = i40e_netpoll,
13275 #endif
13276         .ndo_setup_tc           = __i40e_setup_tc,
13277         .ndo_set_features       = i40e_set_features,
13278         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
13279         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
13280         .ndo_get_vf_stats       = i40e_get_vf_stats,
13281         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
13282         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
13283         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
13284         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
13285         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
13286         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
13287         .ndo_fdb_add            = i40e_ndo_fdb_add,
13288         .ndo_features_check     = i40e_features_check,
13289         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
13290         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
13291         .ndo_bpf                = i40e_xdp,
13292         .ndo_xdp_xmit           = i40e_xdp_xmit,
13293         .ndo_xsk_wakeup         = i40e_xsk_wakeup,
13294         .ndo_dfwd_add_station   = i40e_fwd_add,
13295         .ndo_dfwd_del_station   = i40e_fwd_del,
13296 };
13297
13298 /**
13299  * i40e_config_netdev - Setup the netdev flags
13300  * @vsi: the VSI being configured
13301  *
13302  * Returns 0 on success, negative value on failure
13303  **/
13304 static int i40e_config_netdev(struct i40e_vsi *vsi)
13305 {
13306         struct i40e_pf *pf = vsi->back;
13307         struct i40e_hw *hw = &pf->hw;
13308         struct i40e_netdev_priv *np;
13309         struct net_device *netdev;
13310         u8 broadcast[ETH_ALEN];
13311         u8 mac_addr[ETH_ALEN];
13312         int etherdev_size;
13313         netdev_features_t hw_enc_features;
13314         netdev_features_t hw_features;
13315
13316         etherdev_size = sizeof(struct i40e_netdev_priv);
13317         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13318         if (!netdev)
13319                 return -ENOMEM;
13320
13321         vsi->netdev = netdev;
13322         np = netdev_priv(netdev);
13323         np->vsi = vsi;
13324
13325         hw_enc_features = NETIF_F_SG                    |
13326                           NETIF_F_IP_CSUM               |
13327                           NETIF_F_IPV6_CSUM             |
13328                           NETIF_F_HIGHDMA               |
13329                           NETIF_F_SOFT_FEATURES         |
13330                           NETIF_F_TSO                   |
13331                           NETIF_F_TSO_ECN               |
13332                           NETIF_F_TSO6                  |
13333                           NETIF_F_GSO_GRE               |
13334                           NETIF_F_GSO_GRE_CSUM          |
13335                           NETIF_F_GSO_PARTIAL           |
13336                           NETIF_F_GSO_IPXIP4            |
13337                           NETIF_F_GSO_IPXIP6            |
13338                           NETIF_F_GSO_UDP_TUNNEL        |
13339                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
13340                           NETIF_F_GSO_UDP_L4            |
13341                           NETIF_F_SCTP_CRC              |
13342                           NETIF_F_RXHASH                |
13343                           NETIF_F_RXCSUM                |
13344                           0;
13345
13346         if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
13347                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13348
13349         netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13350
13351         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13352
13353         netdev->hw_enc_features |= hw_enc_features;
13354
13355         /* record features VLANs can make use of */
13356         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13357
13358         /* enable macvlan offloads */
13359         netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13360
13361         hw_features = hw_enc_features           |
13362                       NETIF_F_HW_VLAN_CTAG_TX   |
13363                       NETIF_F_HW_VLAN_CTAG_RX;
13364
13365         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
13366                 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13367
13368         netdev->hw_features |= hw_features;
13369
13370         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13371         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13372
13373         if (vsi->type == I40E_VSI_MAIN) {
13374                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13375                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
13376                 /* The following steps are necessary for two reasons. First,
13377                  * some older NVM configurations load a default MAC-VLAN
13378                  * filter that will accept any tagged packet, and we want to
13379                  * replace this with a normal filter. Additionally, it is
13380                  * possible our MAC address was provided by the platform using
13381                  * Open Firmware or similar.
13382                  *
13383                  * Thus, we need to remove the default filter and install one
13384                  * specific to the MAC address.
13385                  */
13386                 i40e_rm_default_mac_filter(vsi, mac_addr);
13387                 spin_lock_bh(&vsi->mac_filter_hash_lock);
13388                 i40e_add_mac_filter(vsi, mac_addr);
13389                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13390         } else {
13391                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13392                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13393                  * the end, which is 4 bytes long, so force truncation of the
13394                  * original name by IFNAMSIZ - 4
13395                  */
13396                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
13397                          IFNAMSIZ - 4,
13398                          pf->vsi[pf->lan_vsi]->netdev->name);
13399                 eth_random_addr(mac_addr);
13400
13401                 spin_lock_bh(&vsi->mac_filter_hash_lock);
13402                 i40e_add_mac_filter(vsi, mac_addr);
13403                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13404         }
13405
13406         /* Add the broadcast filter so that we initially will receive
13407          * broadcast packets. Note that when a new VLAN is first added the
13408          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13409          * specific filters as part of transitioning into "vlan" operation.
13410          * When more VLANs are added, the driver will copy each existing MAC
13411          * filter and add it for the new VLAN.
13412          *
13413          * Broadcast filters are handled specially by
13414          * i40e_sync_filters_subtask, as the driver must to set the broadcast
13415          * promiscuous bit instead of adding this directly as a MAC/VLAN
13416          * filter. The subtask will update the correct broadcast promiscuous
13417          * bits as VLANs become active or inactive.
13418          */
13419         eth_broadcast_addr(broadcast);
13420         spin_lock_bh(&vsi->mac_filter_hash_lock);
13421         i40e_add_mac_filter(vsi, broadcast);
13422         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13423
13424         ether_addr_copy(netdev->dev_addr, mac_addr);
13425         ether_addr_copy(netdev->perm_addr, mac_addr);
13426
13427         /* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13428         netdev->neigh_priv_len = sizeof(u32) * 4;
13429
13430         netdev->priv_flags |= IFF_UNICAST_FLT;
13431         netdev->priv_flags |= IFF_SUPP_NOFCS;
13432         /* Setup netdev TC information */
13433         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13434
13435         netdev->netdev_ops = &i40e_netdev_ops;
13436         netdev->watchdog_timeo = 5 * HZ;
13437         i40e_set_ethtool_ops(netdev);
13438
13439         /* MTU range: 68 - 9706 */
13440         netdev->min_mtu = ETH_MIN_MTU;
13441         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13442
13443         return 0;
13444 }
13445
13446 /**
13447  * i40e_vsi_delete - Delete a VSI from the switch
13448  * @vsi: the VSI being removed
13449  *
13450  * Returns 0 on success, negative value on failure
13451  **/
13452 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13453 {
13454         /* remove default VSI is not allowed */
13455         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13456                 return;
13457
13458         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13459 }
13460
13461 /**
13462  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13463  * @vsi: the VSI being queried
13464  *
13465  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13466  **/
13467 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13468 {
13469         struct i40e_veb *veb;
13470         struct i40e_pf *pf = vsi->back;
13471
13472         /* Uplink is not a bridge so default to VEB */
13473         if (vsi->veb_idx >= I40E_MAX_VEB)
13474                 return 1;
13475
13476         veb = pf->veb[vsi->veb_idx];
13477         if (!veb) {
13478                 dev_info(&pf->pdev->dev,
13479                          "There is no veb associated with the bridge\n");
13480                 return -ENOENT;
13481         }
13482
13483         /* Uplink is a bridge in VEPA mode */
13484         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13485                 return 0;
13486         } else {
13487                 /* Uplink is a bridge in VEB mode */
13488                 return 1;
13489         }
13490
13491         /* VEPA is now default bridge, so return 0 */
13492         return 0;
13493 }
13494
13495 /**
13496  * i40e_add_vsi - Add a VSI to the switch
13497  * @vsi: the VSI being configured
13498  *
13499  * This initializes a VSI context depending on the VSI type to be added and
13500  * passes it down to the add_vsi aq command.
13501  **/
13502 static int i40e_add_vsi(struct i40e_vsi *vsi)
13503 {
13504         int ret = -ENODEV;
13505         struct i40e_pf *pf = vsi->back;
13506         struct i40e_hw *hw = &pf->hw;
13507         struct i40e_vsi_context ctxt;
13508         struct i40e_mac_filter *f;
13509         struct hlist_node *h;
13510         int bkt;
13511
13512         u8 enabled_tc = 0x1; /* TC0 enabled */
13513         int f_count = 0;
13514
13515         memset(&ctxt, 0, sizeof(ctxt));
13516         switch (vsi->type) {
13517         case I40E_VSI_MAIN:
13518                 /* The PF's main VSI is already setup as part of the
13519                  * device initialization, so we'll not bother with
13520                  * the add_vsi call, but we will retrieve the current
13521                  * VSI context.
13522                  */
13523                 ctxt.seid = pf->main_vsi_seid;
13524                 ctxt.pf_num = pf->hw.pf_id;
13525                 ctxt.vf_num = 0;
13526                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13527                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13528                 if (ret) {
13529                         dev_info(&pf->pdev->dev,
13530                                  "couldn't get PF vsi config, err %s aq_err %s\n",
13531                                  i40e_stat_str(&pf->hw, ret),
13532                                  i40e_aq_str(&pf->hw,
13533                                              pf->hw.aq.asq_last_status));
13534                         return -ENOENT;
13535                 }
13536                 vsi->info = ctxt.info;
13537                 vsi->info.valid_sections = 0;
13538
13539                 vsi->seid = ctxt.seid;
13540                 vsi->id = ctxt.vsi_number;
13541
13542                 enabled_tc = i40e_pf_get_tc_map(pf);
13543
13544                 /* Source pruning is enabled by default, so the flag is
13545                  * negative logic - if it's set, we need to fiddle with
13546                  * the VSI to disable source pruning.
13547                  */
13548                 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
13549                         memset(&ctxt, 0, sizeof(ctxt));
13550                         ctxt.seid = pf->main_vsi_seid;
13551                         ctxt.pf_num = pf->hw.pf_id;
13552                         ctxt.vf_num = 0;
13553                         ctxt.info.valid_sections |=
13554                                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13555                         ctxt.info.switch_id =
13556                                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13557                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13558                         if (ret) {
13559                                 dev_info(&pf->pdev->dev,
13560                                          "update vsi failed, err %s aq_err %s\n",
13561                                          i40e_stat_str(&pf->hw, ret),
13562                                          i40e_aq_str(&pf->hw,
13563                                                      pf->hw.aq.asq_last_status));
13564                                 ret = -ENOENT;
13565                                 goto err;
13566                         }
13567                 }
13568
13569                 /* MFP mode setup queue map and update VSI */
13570                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
13571                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13572                         memset(&ctxt, 0, sizeof(ctxt));
13573                         ctxt.seid = pf->main_vsi_seid;
13574                         ctxt.pf_num = pf->hw.pf_id;
13575                         ctxt.vf_num = 0;
13576                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13577                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13578                         if (ret) {
13579                                 dev_info(&pf->pdev->dev,
13580                                          "update vsi failed, err %s aq_err %s\n",
13581                                          i40e_stat_str(&pf->hw, ret),
13582                                          i40e_aq_str(&pf->hw,
13583                                                     pf->hw.aq.asq_last_status));
13584                                 ret = -ENOENT;
13585                                 goto err;
13586                         }
13587                         /* update the local VSI info queue map */
13588                         i40e_vsi_update_queue_map(vsi, &ctxt);
13589                         vsi->info.valid_sections = 0;
13590                 } else {
13591                         /* Default/Main VSI is only enabled for TC0
13592                          * reconfigure it to enable all TCs that are
13593                          * available on the port in SFP mode.
13594                          * For MFP case the iSCSI PF would use this
13595                          * flow to enable LAN+iSCSI TC.
13596                          */
13597                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
13598                         if (ret) {
13599                                 /* Single TC condition is not fatal,
13600                                  * message and continue
13601                                  */
13602                                 dev_info(&pf->pdev->dev,
13603                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
13604                                          enabled_tc,
13605                                          i40e_stat_str(&pf->hw, ret),
13606                                          i40e_aq_str(&pf->hw,
13607                                                     pf->hw.aq.asq_last_status));
13608                         }
13609                 }
13610                 break;
13611
13612         case I40E_VSI_FDIR:
13613                 ctxt.pf_num = hw->pf_id;
13614                 ctxt.vf_num = 0;
13615                 ctxt.uplink_seid = vsi->uplink_seid;
13616                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13617                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13618                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
13619                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
13620                         ctxt.info.valid_sections |=
13621                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13622                         ctxt.info.switch_id =
13623                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13624                 }
13625                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13626                 break;
13627
13628         case I40E_VSI_VMDQ2:
13629                 ctxt.pf_num = hw->pf_id;
13630                 ctxt.vf_num = 0;
13631                 ctxt.uplink_seid = vsi->uplink_seid;
13632                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13633                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
13634
13635                 /* This VSI is connected to VEB so the switch_id
13636                  * should be set to zero by default.
13637                  */
13638                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13639                         ctxt.info.valid_sections |=
13640                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13641                         ctxt.info.switch_id =
13642                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13643                 }
13644
13645                 /* Setup the VSI tx/rx queue map for TC0 only for now */
13646                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13647                 break;
13648
13649         case I40E_VSI_SRIOV:
13650                 ctxt.pf_num = hw->pf_id;
13651                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
13652                 ctxt.uplink_seid = vsi->uplink_seid;
13653                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13654                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
13655
13656                 /* This VSI is connected to VEB so the switch_id
13657                  * should be set to zero by default.
13658                  */
13659                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13660                         ctxt.info.valid_sections |=
13661                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13662                         ctxt.info.switch_id =
13663                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13664                 }
13665
13666                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
13667                         ctxt.info.valid_sections |=
13668                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
13669                         ctxt.info.queueing_opt_flags |=
13670                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
13671                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
13672                 }
13673
13674                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
13675                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
13676                 if (pf->vf[vsi->vf_id].spoofchk) {
13677                         ctxt.info.valid_sections |=
13678                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
13679                         ctxt.info.sec_flags |=
13680                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
13681                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
13682                 }
13683                 /* Setup the VSI tx/rx queue map for TC0 only for now */
13684                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13685                 break;
13686
13687         case I40E_VSI_IWARP:
13688                 /* send down message to iWARP */
13689                 break;
13690
13691         default:
13692                 return -ENODEV;
13693         }
13694
13695         if (vsi->type != I40E_VSI_MAIN) {
13696                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
13697                 if (ret) {
13698                         dev_info(&vsi->back->pdev->dev,
13699                                  "add vsi failed, err %s aq_err %s\n",
13700                                  i40e_stat_str(&pf->hw, ret),
13701                                  i40e_aq_str(&pf->hw,
13702                                              pf->hw.aq.asq_last_status));
13703                         ret = -ENOENT;
13704                         goto err;
13705                 }
13706                 vsi->info = ctxt.info;
13707                 vsi->info.valid_sections = 0;
13708                 vsi->seid = ctxt.seid;
13709                 vsi->id = ctxt.vsi_number;
13710         }
13711
13712         vsi->active_filters = 0;
13713         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
13714         spin_lock_bh(&vsi->mac_filter_hash_lock);
13715         /* If macvlan filters already exist, force them to get loaded */
13716         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
13717                 f->state = I40E_FILTER_NEW;
13718                 f_count++;
13719         }
13720         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13721
13722         if (f_count) {
13723                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
13724                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
13725         }
13726
13727         /* Update VSI BW information */
13728         ret = i40e_vsi_get_bw_info(vsi);
13729         if (ret) {
13730                 dev_info(&pf->pdev->dev,
13731                          "couldn't get vsi bw info, err %s aq_err %s\n",
13732                          i40e_stat_str(&pf->hw, ret),
13733                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13734                 /* VSI is already added so not tearing that up */
13735                 ret = 0;
13736         }
13737
13738 err:
13739         return ret;
13740 }
13741
13742 /**
13743  * i40e_vsi_release - Delete a VSI and free its resources
13744  * @vsi: the VSI being removed
13745  *
13746  * Returns 0 on success or < 0 on error
13747  **/
13748 int i40e_vsi_release(struct i40e_vsi *vsi)
13749 {
13750         struct i40e_mac_filter *f;
13751         struct hlist_node *h;
13752         struct i40e_veb *veb = NULL;
13753         struct i40e_pf *pf;
13754         u16 uplink_seid;
13755         int i, n, bkt;
13756
13757         pf = vsi->back;
13758
13759         /* release of a VEB-owner or last VSI is not allowed */
13760         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
13761                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
13762                          vsi->seid, vsi->uplink_seid);
13763                 return -ENODEV;
13764         }
13765         if (vsi == pf->vsi[pf->lan_vsi] &&
13766             !test_bit(__I40E_DOWN, pf->state)) {
13767                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
13768                 return -ENODEV;
13769         }
13770
13771         uplink_seid = vsi->uplink_seid;
13772         if (vsi->type != I40E_VSI_SRIOV) {
13773                 if (vsi->netdev_registered) {
13774                         vsi->netdev_registered = false;
13775                         if (vsi->netdev) {
13776                                 /* results in a call to i40e_close() */
13777                                 unregister_netdev(vsi->netdev);
13778                         }
13779                 } else {
13780                         i40e_vsi_close(vsi);
13781                 }
13782                 i40e_vsi_disable_irq(vsi);
13783         }
13784
13785         spin_lock_bh(&vsi->mac_filter_hash_lock);
13786
13787         /* clear the sync flag on all filters */
13788         if (vsi->netdev) {
13789                 __dev_uc_unsync(vsi->netdev, NULL);
13790                 __dev_mc_unsync(vsi->netdev, NULL);
13791         }
13792
13793         /* make sure any remaining filters are marked for deletion */
13794         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
13795                 __i40e_del_filter(vsi, f);
13796
13797         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13798
13799         i40e_sync_vsi_filters(vsi);
13800
13801         i40e_vsi_delete(vsi);
13802         i40e_vsi_free_q_vectors(vsi);
13803         if (vsi->netdev) {
13804                 free_netdev(vsi->netdev);
13805                 vsi->netdev = NULL;
13806         }
13807         i40e_vsi_clear_rings(vsi);
13808         i40e_vsi_clear(vsi);
13809
13810         /* If this was the last thing on the VEB, except for the
13811          * controlling VSI, remove the VEB, which puts the controlling
13812          * VSI onto the next level down in the switch.
13813          *
13814          * Well, okay, there's one more exception here: don't remove
13815          * the orphan VEBs yet.  We'll wait for an explicit remove request
13816          * from up the network stack.
13817          */
13818         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
13819                 if (pf->vsi[i] &&
13820                     pf->vsi[i]->uplink_seid == uplink_seid &&
13821                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
13822                         n++;      /* count the VSIs */
13823                 }
13824         }
13825         for (i = 0; i < I40E_MAX_VEB; i++) {
13826                 if (!pf->veb[i])
13827                         continue;
13828                 if (pf->veb[i]->uplink_seid == uplink_seid)
13829                         n++;     /* count the VEBs */
13830                 if (pf->veb[i]->seid == uplink_seid)
13831                         veb = pf->veb[i];
13832         }
13833         if (n == 0 && veb && veb->uplink_seid != 0)
13834                 i40e_veb_release(veb);
13835
13836         return 0;
13837 }
13838
13839 /**
13840  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
13841  * @vsi: ptr to the VSI
13842  *
13843  * This should only be called after i40e_vsi_mem_alloc() which allocates the
13844  * corresponding SW VSI structure and initializes num_queue_pairs for the
13845  * newly allocated VSI.
13846  *
13847  * Returns 0 on success or negative on failure
13848  **/
13849 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
13850 {
13851         int ret = -ENOENT;
13852         struct i40e_pf *pf = vsi->back;
13853
13854         if (vsi->q_vectors[0]) {
13855                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
13856                          vsi->seid);
13857                 return -EEXIST;
13858         }
13859
13860         if (vsi->base_vector) {
13861                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
13862                          vsi->seid, vsi->base_vector);
13863                 return -EEXIST;
13864         }
13865
13866         ret = i40e_vsi_alloc_q_vectors(vsi);
13867         if (ret) {
13868                 dev_info(&pf->pdev->dev,
13869                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
13870                          vsi->num_q_vectors, vsi->seid, ret);
13871                 vsi->num_q_vectors = 0;
13872                 goto vector_setup_out;
13873         }
13874
13875         /* In Legacy mode, we do not have to get any other vector since we
13876          * piggyback on the misc/ICR0 for queue interrupts.
13877         */
13878         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
13879                 return ret;
13880         if (vsi->num_q_vectors)
13881                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
13882                                                  vsi->num_q_vectors, vsi->idx);
13883         if (vsi->base_vector < 0) {
13884                 dev_info(&pf->pdev->dev,
13885                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
13886                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
13887                 i40e_vsi_free_q_vectors(vsi);
13888                 ret = -ENOENT;
13889                 goto vector_setup_out;
13890         }
13891
13892 vector_setup_out:
13893         return ret;
13894 }
13895
13896 /**
13897  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
13898  * @vsi: pointer to the vsi.
13899  *
13900  * This re-allocates a vsi's queue resources.
13901  *
13902  * Returns pointer to the successfully allocated and configured VSI sw struct
13903  * on success, otherwise returns NULL on failure.
13904  **/
13905 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
13906 {
13907         u16 alloc_queue_pairs;
13908         struct i40e_pf *pf;
13909         u8 enabled_tc;
13910         int ret;
13911
13912         if (!vsi)
13913                 return NULL;
13914
13915         pf = vsi->back;
13916
13917         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
13918         i40e_vsi_clear_rings(vsi);
13919
13920         i40e_vsi_free_arrays(vsi, false);
13921         i40e_set_num_rings_in_vsi(vsi);
13922         ret = i40e_vsi_alloc_arrays(vsi, false);
13923         if (ret)
13924                 goto err_vsi;
13925
13926         alloc_queue_pairs = vsi->alloc_queue_pairs *
13927                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
13928
13929         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
13930         if (ret < 0) {
13931                 dev_info(&pf->pdev->dev,
13932                          "failed to get tracking for %d queues for VSI %d err %d\n",
13933                          alloc_queue_pairs, vsi->seid, ret);
13934                 goto err_vsi;
13935         }
13936         vsi->base_queue = ret;
13937
13938         /* Update the FW view of the VSI. Force a reset of TC and queue
13939          * layout configurations.
13940          */
13941         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
13942         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
13943         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
13944         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
13945         if (vsi->type == I40E_VSI_MAIN)
13946                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
13947
13948         /* assign it some queues */
13949         ret = i40e_alloc_rings(vsi);
13950         if (ret)
13951                 goto err_rings;
13952
13953         /* map all of the rings to the q_vectors */
13954         i40e_vsi_map_rings_to_vectors(vsi);
13955         return vsi;
13956
13957 err_rings:
13958         i40e_vsi_free_q_vectors(vsi);
13959         if (vsi->netdev_registered) {
13960                 vsi->netdev_registered = false;
13961                 unregister_netdev(vsi->netdev);
13962                 free_netdev(vsi->netdev);
13963                 vsi->netdev = NULL;
13964         }
13965         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
13966 err_vsi:
13967         i40e_vsi_clear(vsi);
13968         return NULL;
13969 }
13970
13971 /**
13972  * i40e_vsi_setup - Set up a VSI by a given type
13973  * @pf: board private structure
13974  * @type: VSI type
13975  * @uplink_seid: the switch element to link to
13976  * @param1: usage depends upon VSI type. For VF types, indicates VF id
13977  *
13978  * This allocates the sw VSI structure and its queue resources, then add a VSI
13979  * to the identified VEB.
13980  *
13981  * Returns pointer to the successfully allocated and configure VSI sw struct on
13982  * success, otherwise returns NULL on failure.
13983  **/
13984 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
13985                                 u16 uplink_seid, u32 param1)
13986 {
13987         struct i40e_vsi *vsi = NULL;
13988         struct i40e_veb *veb = NULL;
13989         u16 alloc_queue_pairs;
13990         int ret, i;
13991         int v_idx;
13992
13993         /* The requested uplink_seid must be either
13994          *     - the PF's port seid
13995          *              no VEB is needed because this is the PF
13996          *              or this is a Flow Director special case VSI
13997          *     - seid of an existing VEB
13998          *     - seid of a VSI that owns an existing VEB
13999          *     - seid of a VSI that doesn't own a VEB
14000          *              a new VEB is created and the VSI becomes the owner
14001          *     - seid of the PF VSI, which is what creates the first VEB
14002          *              this is a special case of the previous
14003          *
14004          * Find which uplink_seid we were given and create a new VEB if needed
14005          */
14006         for (i = 0; i < I40E_MAX_VEB; i++) {
14007                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
14008                         veb = pf->veb[i];
14009                         break;
14010                 }
14011         }
14012
14013         if (!veb && uplink_seid != pf->mac_seid) {
14014
14015                 for (i = 0; i < pf->num_alloc_vsi; i++) {
14016                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
14017                                 vsi = pf->vsi[i];
14018                                 break;
14019                         }
14020                 }
14021                 if (!vsi) {
14022                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
14023                                  uplink_seid);
14024                         return NULL;
14025                 }
14026
14027                 if (vsi->uplink_seid == pf->mac_seid)
14028                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
14029                                              vsi->tc_config.enabled_tc);
14030                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14031                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
14032                                              vsi->tc_config.enabled_tc);
14033                 if (veb) {
14034                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
14035                                 dev_info(&vsi->back->pdev->dev,
14036                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
14037                                 return NULL;
14038                         }
14039                         /* We come up by default in VEPA mode if SRIOV is not
14040                          * already enabled, in which case we can't force VEPA
14041                          * mode.
14042                          */
14043                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
14044                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
14045                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
14046                         }
14047                         i40e_config_bridge_mode(veb);
14048                 }
14049                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
14050                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
14051                                 veb = pf->veb[i];
14052                 }
14053                 if (!veb) {
14054                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
14055                         return NULL;
14056                 }
14057
14058                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14059                 uplink_seid = veb->seid;
14060         }
14061
14062         /* get vsi sw struct */
14063         v_idx = i40e_vsi_mem_alloc(pf, type);
14064         if (v_idx < 0)
14065                 goto err_alloc;
14066         vsi = pf->vsi[v_idx];
14067         if (!vsi)
14068                 goto err_alloc;
14069         vsi->type = type;
14070         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
14071
14072         if (type == I40E_VSI_MAIN)
14073                 pf->lan_vsi = v_idx;
14074         else if (type == I40E_VSI_SRIOV)
14075                 vsi->vf_id = param1;
14076         /* assign it some queues */
14077         alloc_queue_pairs = vsi->alloc_queue_pairs *
14078                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14079
14080         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14081         if (ret < 0) {
14082                 dev_info(&pf->pdev->dev,
14083                          "failed to get tracking for %d queues for VSI %d err=%d\n",
14084                          alloc_queue_pairs, vsi->seid, ret);
14085                 goto err_vsi;
14086         }
14087         vsi->base_queue = ret;
14088
14089         /* get a VSI from the hardware */
14090         vsi->uplink_seid = uplink_seid;
14091         ret = i40e_add_vsi(vsi);
14092         if (ret)
14093                 goto err_vsi;
14094
14095         switch (vsi->type) {
14096         /* setup the netdev if needed */
14097         case I40E_VSI_MAIN:
14098         case I40E_VSI_VMDQ2:
14099                 ret = i40e_config_netdev(vsi);
14100                 if (ret)
14101                         goto err_netdev;
14102                 ret = register_netdev(vsi->netdev);
14103                 if (ret)
14104                         goto err_netdev;
14105                 vsi->netdev_registered = true;
14106                 netif_carrier_off(vsi->netdev);
14107 #ifdef CONFIG_I40E_DCB
14108                 /* Setup DCB netlink interface */
14109                 i40e_dcbnl_setup(vsi);
14110 #endif /* CONFIG_I40E_DCB */
14111                 fallthrough;
14112         case I40E_VSI_FDIR:
14113                 /* set up vectors and rings if needed */
14114                 ret = i40e_vsi_setup_vectors(vsi);
14115                 if (ret)
14116                         goto err_msix;
14117
14118                 ret = i40e_alloc_rings(vsi);
14119                 if (ret)
14120                         goto err_rings;
14121
14122                 /* map all of the rings to the q_vectors */
14123                 i40e_vsi_map_rings_to_vectors(vsi);
14124
14125                 i40e_vsi_reset_stats(vsi);
14126                 break;
14127         default:
14128                 /* no netdev or rings for the other VSI types */
14129                 break;
14130         }
14131
14132         if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
14133             (vsi->type == I40E_VSI_VMDQ2)) {
14134                 ret = i40e_vsi_config_rss(vsi);
14135         }
14136         return vsi;
14137
14138 err_rings:
14139         i40e_vsi_free_q_vectors(vsi);
14140 err_msix:
14141         if (vsi->netdev_registered) {
14142                 vsi->netdev_registered = false;
14143                 unregister_netdev(vsi->netdev);
14144                 free_netdev(vsi->netdev);
14145                 vsi->netdev = NULL;
14146         }
14147 err_netdev:
14148         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14149 err_vsi:
14150         i40e_vsi_clear(vsi);
14151 err_alloc:
14152         return NULL;
14153 }
14154
14155 /**
14156  * i40e_veb_get_bw_info - Query VEB BW information
14157  * @veb: the veb to query
14158  *
14159  * Query the Tx scheduler BW configuration data for given VEB
14160  **/
14161 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
14162 {
14163         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
14164         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
14165         struct i40e_pf *pf = veb->pf;
14166         struct i40e_hw *hw = &pf->hw;
14167         u32 tc_bw_max;
14168         int ret = 0;
14169         int i;
14170
14171         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
14172                                                   &bw_data, NULL);
14173         if (ret) {
14174                 dev_info(&pf->pdev->dev,
14175                          "query veb bw config failed, err %s aq_err %s\n",
14176                          i40e_stat_str(&pf->hw, ret),
14177                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14178                 goto out;
14179         }
14180
14181         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
14182                                                    &ets_data, NULL);
14183         if (ret) {
14184                 dev_info(&pf->pdev->dev,
14185                          "query veb bw ets config failed, err %s aq_err %s\n",
14186                          i40e_stat_str(&pf->hw, ret),
14187                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14188                 goto out;
14189         }
14190
14191         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
14192         veb->bw_max_quanta = ets_data.tc_bw_max;
14193         veb->is_abs_credits = bw_data.absolute_credits_enable;
14194         veb->enabled_tc = ets_data.tc_valid_bits;
14195         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
14196                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
14197         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
14198                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
14199                 veb->bw_tc_limit_credits[i] =
14200                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
14201                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
14202         }
14203
14204 out:
14205         return ret;
14206 }
14207
14208 /**
14209  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14210  * @pf: board private structure
14211  *
14212  * On error: returns error code (negative)
14213  * On success: returns vsi index in PF (positive)
14214  **/
14215 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
14216 {
14217         int ret = -ENOENT;
14218         struct i40e_veb *veb;
14219         int i;
14220
14221         /* Need to protect the allocation of switch elements at the PF level */
14222         mutex_lock(&pf->switch_mutex);
14223
14224         /* VEB list may be fragmented if VEB creation/destruction has
14225          * been happening.  We can afford to do a quick scan to look
14226          * for any free slots in the list.
14227          *
14228          * find next empty veb slot, looping back around if necessary
14229          */
14230         i = 0;
14231         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
14232                 i++;
14233         if (i >= I40E_MAX_VEB) {
14234                 ret = -ENOMEM;
14235                 goto err_alloc_veb;  /* out of VEB slots! */
14236         }
14237
14238         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
14239         if (!veb) {
14240                 ret = -ENOMEM;
14241                 goto err_alloc_veb;
14242         }
14243         veb->pf = pf;
14244         veb->idx = i;
14245         veb->enabled_tc = 1;
14246
14247         pf->veb[i] = veb;
14248         ret = i;
14249 err_alloc_veb:
14250         mutex_unlock(&pf->switch_mutex);
14251         return ret;
14252 }
14253
14254 /**
14255  * i40e_switch_branch_release - Delete a branch of the switch tree
14256  * @branch: where to start deleting
14257  *
14258  * This uses recursion to find the tips of the branch to be
14259  * removed, deleting until we get back to and can delete this VEB.
14260  **/
14261 static void i40e_switch_branch_release(struct i40e_veb *branch)
14262 {
14263         struct i40e_pf *pf = branch->pf;
14264         u16 branch_seid = branch->seid;
14265         u16 veb_idx = branch->idx;
14266         int i;
14267
14268         /* release any VEBs on this VEB - RECURSION */
14269         for (i = 0; i < I40E_MAX_VEB; i++) {
14270                 if (!pf->veb[i])
14271                         continue;
14272                 if (pf->veb[i]->uplink_seid == branch->seid)
14273                         i40e_switch_branch_release(pf->veb[i]);
14274         }
14275
14276         /* Release the VSIs on this VEB, but not the owner VSI.
14277          *
14278          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14279          *       the VEB itself, so don't use (*branch) after this loop.
14280          */
14281         for (i = 0; i < pf->num_alloc_vsi; i++) {
14282                 if (!pf->vsi[i])
14283                         continue;
14284                 if (pf->vsi[i]->uplink_seid == branch_seid &&
14285                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
14286                         i40e_vsi_release(pf->vsi[i]);
14287                 }
14288         }
14289
14290         /* There's one corner case where the VEB might not have been
14291          * removed, so double check it here and remove it if needed.
14292          * This case happens if the veb was created from the debugfs
14293          * commands and no VSIs were added to it.
14294          */
14295         if (pf->veb[veb_idx])
14296                 i40e_veb_release(pf->veb[veb_idx]);
14297 }
14298
14299 /**
14300  * i40e_veb_clear - remove veb struct
14301  * @veb: the veb to remove
14302  **/
14303 static void i40e_veb_clear(struct i40e_veb *veb)
14304 {
14305         if (!veb)
14306                 return;
14307
14308         if (veb->pf) {
14309                 struct i40e_pf *pf = veb->pf;
14310
14311                 mutex_lock(&pf->switch_mutex);
14312                 if (pf->veb[veb->idx] == veb)
14313                         pf->veb[veb->idx] = NULL;
14314                 mutex_unlock(&pf->switch_mutex);
14315         }
14316
14317         kfree(veb);
14318 }
14319
14320 /**
14321  * i40e_veb_release - Delete a VEB and free its resources
14322  * @veb: the VEB being removed
14323  **/
14324 void i40e_veb_release(struct i40e_veb *veb)
14325 {
14326         struct i40e_vsi *vsi = NULL;
14327         struct i40e_pf *pf;
14328         int i, n = 0;
14329
14330         pf = veb->pf;
14331
14332         /* find the remaining VSI and check for extras */
14333         for (i = 0; i < pf->num_alloc_vsi; i++) {
14334                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
14335                         n++;
14336                         vsi = pf->vsi[i];
14337                 }
14338         }
14339         if (n != 1) {
14340                 dev_info(&pf->pdev->dev,
14341                          "can't remove VEB %d with %d VSIs left\n",
14342                          veb->seid, n);
14343                 return;
14344         }
14345
14346         /* move the remaining VSI to uplink veb */
14347         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14348         if (veb->uplink_seid) {
14349                 vsi->uplink_seid = veb->uplink_seid;
14350                 if (veb->uplink_seid == pf->mac_seid)
14351                         vsi->veb_idx = I40E_NO_VEB;
14352                 else
14353                         vsi->veb_idx = veb->veb_idx;
14354         } else {
14355                 /* floating VEB */
14356                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
14357                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
14358         }
14359
14360         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14361         i40e_veb_clear(veb);
14362 }
14363
14364 /**
14365  * i40e_add_veb - create the VEB in the switch
14366  * @veb: the VEB to be instantiated
14367  * @vsi: the controlling VSI
14368  **/
14369 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14370 {
14371         struct i40e_pf *pf = veb->pf;
14372         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
14373         int ret;
14374
14375         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
14376                               veb->enabled_tc, false,
14377                               &veb->seid, enable_stats, NULL);
14378
14379         /* get a VEB from the hardware */
14380         if (ret) {
14381                 dev_info(&pf->pdev->dev,
14382                          "couldn't add VEB, err %s aq_err %s\n",
14383                          i40e_stat_str(&pf->hw, ret),
14384                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14385                 return -EPERM;
14386         }
14387
14388         /* get statistics counter */
14389         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
14390                                          &veb->stats_idx, NULL, NULL, NULL);
14391         if (ret) {
14392                 dev_info(&pf->pdev->dev,
14393                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
14394                          i40e_stat_str(&pf->hw, ret),
14395                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14396                 return -EPERM;
14397         }
14398         ret = i40e_veb_get_bw_info(veb);
14399         if (ret) {
14400                 dev_info(&pf->pdev->dev,
14401                          "couldn't get VEB bw info, err %s aq_err %s\n",
14402                          i40e_stat_str(&pf->hw, ret),
14403                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14404                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14405                 return -ENOENT;
14406         }
14407
14408         vsi->uplink_seid = veb->seid;
14409         vsi->veb_idx = veb->idx;
14410         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14411
14412         return 0;
14413 }
14414
14415 /**
14416  * i40e_veb_setup - Set up a VEB
14417  * @pf: board private structure
14418  * @flags: VEB setup flags
14419  * @uplink_seid: the switch element to link to
14420  * @vsi_seid: the initial VSI seid
14421  * @enabled_tc: Enabled TC bit-map
14422  *
14423  * This allocates the sw VEB structure and links it into the switch
14424  * It is possible and legal for this to be a duplicate of an already
14425  * existing VEB.  It is also possible for both uplink and vsi seids
14426  * to be zero, in order to create a floating VEB.
14427  *
14428  * Returns pointer to the successfully allocated VEB sw struct on
14429  * success, otherwise returns NULL on failure.
14430  **/
14431 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
14432                                 u16 uplink_seid, u16 vsi_seid,
14433                                 u8 enabled_tc)
14434 {
14435         struct i40e_veb *veb, *uplink_veb = NULL;
14436         int vsi_idx, veb_idx;
14437         int ret;
14438
14439         /* if one seid is 0, the other must be 0 to create a floating relay */
14440         if ((uplink_seid == 0 || vsi_seid == 0) &&
14441             (uplink_seid + vsi_seid != 0)) {
14442                 dev_info(&pf->pdev->dev,
14443                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
14444                          uplink_seid, vsi_seid);
14445                 return NULL;
14446         }
14447
14448         /* make sure there is such a vsi and uplink */
14449         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
14450                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
14451                         break;
14452         if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) {
14453                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
14454                          vsi_seid);
14455                 return NULL;
14456         }
14457
14458         if (uplink_seid && uplink_seid != pf->mac_seid) {
14459                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
14460                         if (pf->veb[veb_idx] &&
14461                             pf->veb[veb_idx]->seid == uplink_seid) {
14462                                 uplink_veb = pf->veb[veb_idx];
14463                                 break;
14464                         }
14465                 }
14466                 if (!uplink_veb) {
14467                         dev_info(&pf->pdev->dev,
14468                                  "uplink seid %d not found\n", uplink_seid);
14469                         return NULL;
14470                 }
14471         }
14472
14473         /* get veb sw struct */
14474         veb_idx = i40e_veb_mem_alloc(pf);
14475         if (veb_idx < 0)
14476                 goto err_alloc;
14477         veb = pf->veb[veb_idx];
14478         veb->flags = flags;
14479         veb->uplink_seid = uplink_seid;
14480         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
14481         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14482
14483         /* create the VEB in the switch */
14484         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
14485         if (ret)
14486                 goto err_veb;
14487         if (vsi_idx == pf->lan_vsi)
14488                 pf->lan_veb = veb->idx;
14489
14490         return veb;
14491
14492 err_veb:
14493         i40e_veb_clear(veb);
14494 err_alloc:
14495         return NULL;
14496 }
14497
14498 /**
14499  * i40e_setup_pf_switch_element - set PF vars based on switch type
14500  * @pf: board private structure
14501  * @ele: element we are building info from
14502  * @num_reported: total number of elements
14503  * @printconfig: should we print the contents
14504  *
14505  * helper function to assist in extracting a few useful SEID values.
14506  **/
14507 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14508                                 struct i40e_aqc_switch_config_element_resp *ele,
14509                                 u16 num_reported, bool printconfig)
14510 {
14511         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14512         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14513         u8 element_type = ele->element_type;
14514         u16 seid = le16_to_cpu(ele->seid);
14515
14516         if (printconfig)
14517                 dev_info(&pf->pdev->dev,
14518                          "type=%d seid=%d uplink=%d downlink=%d\n",
14519                          element_type, seid, uplink_seid, downlink_seid);
14520
14521         switch (element_type) {
14522         case I40E_SWITCH_ELEMENT_TYPE_MAC:
14523                 pf->mac_seid = seid;
14524                 break;
14525         case I40E_SWITCH_ELEMENT_TYPE_VEB:
14526                 /* Main VEB? */
14527                 if (uplink_seid != pf->mac_seid)
14528                         break;
14529                 if (pf->lan_veb >= I40E_MAX_VEB) {
14530                         int v;
14531
14532                         /* find existing or else empty VEB */
14533                         for (v = 0; v < I40E_MAX_VEB; v++) {
14534                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
14535                                         pf->lan_veb = v;
14536                                         break;
14537                                 }
14538                         }
14539                         if (pf->lan_veb >= I40E_MAX_VEB) {
14540                                 v = i40e_veb_mem_alloc(pf);
14541                                 if (v < 0)
14542                                         break;
14543                                 pf->lan_veb = v;
14544                         }
14545                 }
14546                 if (pf->lan_veb >= I40E_MAX_VEB)
14547                         break;
14548
14549                 pf->veb[pf->lan_veb]->seid = seid;
14550                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
14551                 pf->veb[pf->lan_veb]->pf = pf;
14552                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
14553                 break;
14554         case I40E_SWITCH_ELEMENT_TYPE_VSI:
14555                 if (num_reported != 1)
14556                         break;
14557                 /* This is immediately after a reset so we can assume this is
14558                  * the PF's VSI
14559                  */
14560                 pf->mac_seid = uplink_seid;
14561                 pf->pf_seid = downlink_seid;
14562                 pf->main_vsi_seid = seid;
14563                 if (printconfig)
14564                         dev_info(&pf->pdev->dev,
14565                                  "pf_seid=%d main_vsi_seid=%d\n",
14566                                  pf->pf_seid, pf->main_vsi_seid);
14567                 break;
14568         case I40E_SWITCH_ELEMENT_TYPE_PF:
14569         case I40E_SWITCH_ELEMENT_TYPE_VF:
14570         case I40E_SWITCH_ELEMENT_TYPE_EMP:
14571         case I40E_SWITCH_ELEMENT_TYPE_BMC:
14572         case I40E_SWITCH_ELEMENT_TYPE_PE:
14573         case I40E_SWITCH_ELEMENT_TYPE_PA:
14574                 /* ignore these for now */
14575                 break;
14576         default:
14577                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14578                          element_type, seid);
14579                 break;
14580         }
14581 }
14582
14583 /**
14584  * i40e_fetch_switch_configuration - Get switch config from firmware
14585  * @pf: board private structure
14586  * @printconfig: should we print the contents
14587  *
14588  * Get the current switch configuration from the device and
14589  * extract a few useful SEID values.
14590  **/
14591 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14592 {
14593         struct i40e_aqc_get_switch_config_resp *sw_config;
14594         u16 next_seid = 0;
14595         int ret = 0;
14596         u8 *aq_buf;
14597         int i;
14598
14599         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14600         if (!aq_buf)
14601                 return -ENOMEM;
14602
14603         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14604         do {
14605                 u16 num_reported, num_total;
14606
14607                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
14608                                                 I40E_AQ_LARGE_BUF,
14609                                                 &next_seid, NULL);
14610                 if (ret) {
14611                         dev_info(&pf->pdev->dev,
14612                                  "get switch config failed err %s aq_err %s\n",
14613                                  i40e_stat_str(&pf->hw, ret),
14614                                  i40e_aq_str(&pf->hw,
14615                                              pf->hw.aq.asq_last_status));
14616                         kfree(aq_buf);
14617                         return -ENOENT;
14618                 }
14619
14620                 num_reported = le16_to_cpu(sw_config->header.num_reported);
14621                 num_total = le16_to_cpu(sw_config->header.num_total);
14622
14623                 if (printconfig)
14624                         dev_info(&pf->pdev->dev,
14625                                  "header: %d reported %d total\n",
14626                                  num_reported, num_total);
14627
14628                 for (i = 0; i < num_reported; i++) {
14629                         struct i40e_aqc_switch_config_element_resp *ele =
14630                                 &sw_config->element[i];
14631
14632                         i40e_setup_pf_switch_element(pf, ele, num_reported,
14633                                                      printconfig);
14634                 }
14635         } while (next_seid != 0);
14636
14637         kfree(aq_buf);
14638         return ret;
14639 }
14640
14641 /**
14642  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
14643  * @pf: board private structure
14644  * @reinit: if the Main VSI needs to re-initialized.
14645  *
14646  * Returns 0 on success, negative value on failure
14647  **/
14648 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
14649 {
14650         u16 flags = 0;
14651         int ret;
14652
14653         /* find out what's out there already */
14654         ret = i40e_fetch_switch_configuration(pf, false);
14655         if (ret) {
14656                 dev_info(&pf->pdev->dev,
14657                          "couldn't fetch switch config, err %s aq_err %s\n",
14658                          i40e_stat_str(&pf->hw, ret),
14659                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14660                 return ret;
14661         }
14662         i40e_pf_reset_stats(pf);
14663
14664         /* set the switch config bit for the whole device to
14665          * support limited promisc or true promisc
14666          * when user requests promisc. The default is limited
14667          * promisc.
14668         */
14669
14670         if ((pf->hw.pf_id == 0) &&
14671             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
14672                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14673                 pf->last_sw_conf_flags = flags;
14674         }
14675
14676         if (pf->hw.pf_id == 0) {
14677                 u16 valid_flags;
14678
14679                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14680                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
14681                                                 NULL);
14682                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
14683                         dev_info(&pf->pdev->dev,
14684                                  "couldn't set switch config bits, err %s aq_err %s\n",
14685                                  i40e_stat_str(&pf->hw, ret),
14686                                  i40e_aq_str(&pf->hw,
14687                                              pf->hw.aq.asq_last_status));
14688                         /* not a fatal problem, just keep going */
14689                 }
14690                 pf->last_sw_conf_valid_flags = valid_flags;
14691         }
14692
14693         /* first time setup */
14694         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
14695                 struct i40e_vsi *vsi = NULL;
14696                 u16 uplink_seid;
14697
14698                 /* Set up the PF VSI associated with the PF's main VSI
14699                  * that is already in the HW switch
14700                  */
14701                 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
14702                         uplink_seid = pf->veb[pf->lan_veb]->seid;
14703                 else
14704                         uplink_seid = pf->mac_seid;
14705                 if (pf->lan_vsi == I40E_NO_VSI)
14706                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
14707                 else if (reinit)
14708                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
14709                 if (!vsi) {
14710                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
14711                         i40e_cloud_filter_exit(pf);
14712                         i40e_fdir_teardown(pf);
14713                         return -EAGAIN;
14714                 }
14715         } else {
14716                 /* force a reset of TC and queue layout configurations */
14717                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14718
14719                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14720                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14721                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
14722         }
14723         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
14724
14725         i40e_fdir_sb_setup(pf);
14726
14727         /* Setup static PF queue filter control settings */
14728         ret = i40e_setup_pf_filter_control(pf);
14729         if (ret) {
14730                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
14731                          ret);
14732                 /* Failure here should not stop continuing other steps */
14733         }
14734
14735         /* enable RSS in the HW, even for only one queue, as the stack can use
14736          * the hash
14737          */
14738         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
14739                 i40e_pf_config_rss(pf);
14740
14741         /* fill in link information and enable LSE reporting */
14742         i40e_link_event(pf);
14743
14744         /* Initialize user-specific link properties */
14745         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
14746                                   I40E_AQ_AN_COMPLETED) ? true : false);
14747
14748         i40e_ptp_init(pf);
14749
14750         /* repopulate tunnel port filters */
14751         udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev);
14752
14753         return ret;
14754 }
14755
14756 /**
14757  * i40e_determine_queue_usage - Work out queue distribution
14758  * @pf: board private structure
14759  **/
14760 static void i40e_determine_queue_usage(struct i40e_pf *pf)
14761 {
14762         int queues_left;
14763         int q_max;
14764
14765         pf->num_lan_qps = 0;
14766
14767         /* Find the max queues to be put into basic use.  We'll always be
14768          * using TC0, whether or not DCB is running, and TC0 will get the
14769          * big RSS set.
14770          */
14771         queues_left = pf->hw.func_caps.num_tx_qp;
14772
14773         if ((queues_left == 1) ||
14774             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
14775                 /* one qp for PF, no queues for anything else */
14776                 queues_left = 0;
14777                 pf->alloc_rss_size = pf->num_lan_qps = 1;
14778
14779                 /* make sure all the fancies are disabled */
14780                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
14781                                I40E_FLAG_IWARP_ENABLED  |
14782                                I40E_FLAG_FD_SB_ENABLED  |
14783                                I40E_FLAG_FD_ATR_ENABLED |
14784                                I40E_FLAG_DCB_CAPABLE    |
14785                                I40E_FLAG_DCB_ENABLED    |
14786                                I40E_FLAG_SRIOV_ENABLED  |
14787                                I40E_FLAG_VMDQ_ENABLED);
14788                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14789         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
14790                                   I40E_FLAG_FD_SB_ENABLED |
14791                                   I40E_FLAG_FD_ATR_ENABLED |
14792                                   I40E_FLAG_DCB_CAPABLE))) {
14793                 /* one qp for PF */
14794                 pf->alloc_rss_size = pf->num_lan_qps = 1;
14795                 queues_left -= pf->num_lan_qps;
14796
14797                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
14798                                I40E_FLAG_IWARP_ENABLED  |
14799                                I40E_FLAG_FD_SB_ENABLED  |
14800                                I40E_FLAG_FD_ATR_ENABLED |
14801                                I40E_FLAG_DCB_ENABLED    |
14802                                I40E_FLAG_VMDQ_ENABLED);
14803                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14804         } else {
14805                 /* Not enough queues for all TCs */
14806                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
14807                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
14808                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
14809                                         I40E_FLAG_DCB_ENABLED);
14810                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
14811                 }
14812
14813                 /* limit lan qps to the smaller of qps, cpus or msix */
14814                 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
14815                 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
14816                 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
14817                 pf->num_lan_qps = q_max;
14818
14819                 queues_left -= pf->num_lan_qps;
14820         }
14821
14822         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14823                 if (queues_left > 1) {
14824                         queues_left -= 1; /* save 1 queue for FD */
14825                 } else {
14826                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
14827                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14828                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
14829                 }
14830         }
14831
14832         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
14833             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
14834                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
14835                                         (queues_left / pf->num_vf_qps));
14836                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
14837         }
14838
14839         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
14840             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
14841                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
14842                                           (queues_left / pf->num_vmdq_qps));
14843                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
14844         }
14845
14846         pf->queues_left = queues_left;
14847         dev_dbg(&pf->pdev->dev,
14848                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
14849                 pf->hw.func_caps.num_tx_qp,
14850                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
14851                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
14852                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
14853                 queues_left);
14854 }
14855
14856 /**
14857  * i40e_setup_pf_filter_control - Setup PF static filter control
14858  * @pf: PF to be setup
14859  *
14860  * i40e_setup_pf_filter_control sets up a PF's initial filter control
14861  * settings. If PE/FCoE are enabled then it will also set the per PF
14862  * based filter sizes required for them. It also enables Flow director,
14863  * ethertype and macvlan type filter settings for the pf.
14864  *
14865  * Returns 0 on success, negative on failure
14866  **/
14867 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
14868 {
14869         struct i40e_filter_control_settings *settings = &pf->filter_settings;
14870
14871         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
14872
14873         /* Flow Director is enabled */
14874         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
14875                 settings->enable_fdir = true;
14876
14877         /* Ethtype and MACVLAN filters enabled for PF */
14878         settings->enable_ethtype = true;
14879         settings->enable_macvlan = true;
14880
14881         if (i40e_set_filter_control(&pf->hw, settings))
14882                 return -ENOENT;
14883
14884         return 0;
14885 }
14886
14887 #define INFO_STRING_LEN 255
14888 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
14889 static void i40e_print_features(struct i40e_pf *pf)
14890 {
14891         struct i40e_hw *hw = &pf->hw;
14892         char *buf;
14893         int i;
14894
14895         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
14896         if (!buf)
14897                 return;
14898
14899         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
14900 #ifdef CONFIG_PCI_IOV
14901         i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
14902 #endif
14903         i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
14904                       pf->hw.func_caps.num_vsis,
14905                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
14906         if (pf->flags & I40E_FLAG_RSS_ENABLED)
14907                 i += scnprintf(&buf[i], REMAIN(i), " RSS");
14908         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
14909                 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
14910         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14911                 i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
14912                 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
14913         }
14914         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
14915                 i += scnprintf(&buf[i], REMAIN(i), " DCB");
14916         i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
14917         i += scnprintf(&buf[i], REMAIN(i), " Geneve");
14918         if (pf->flags & I40E_FLAG_PTP)
14919                 i += scnprintf(&buf[i], REMAIN(i), " PTP");
14920         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
14921                 i += scnprintf(&buf[i], REMAIN(i), " VEB");
14922         else
14923                 i += scnprintf(&buf[i], REMAIN(i), " VEPA");
14924
14925         dev_info(&pf->pdev->dev, "%s\n", buf);
14926         kfree(buf);
14927         WARN_ON(i > INFO_STRING_LEN);
14928 }
14929
14930 /**
14931  * i40e_get_platform_mac_addr - get platform-specific MAC address
14932  * @pdev: PCI device information struct
14933  * @pf: board private structure
14934  *
14935  * Look up the MAC address for the device. First we'll try
14936  * eth_platform_get_mac_address, which will check Open Firmware, or arch
14937  * specific fallback. Otherwise, we'll default to the stored value in
14938  * firmware.
14939  **/
14940 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
14941 {
14942         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
14943                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
14944 }
14945
14946 /**
14947  * i40e_set_fec_in_flags - helper function for setting FEC options in flags
14948  * @fec_cfg: FEC option to set in flags
14949  * @flags: ptr to flags in which we set FEC option
14950  **/
14951 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags)
14952 {
14953         if (fec_cfg & I40E_AQ_SET_FEC_AUTO)
14954                 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC;
14955         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
14956             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
14957                 *flags |= I40E_FLAG_RS_FEC;
14958                 *flags &= ~I40E_FLAG_BASE_R_FEC;
14959         }
14960         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
14961             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
14962                 *flags |= I40E_FLAG_BASE_R_FEC;
14963                 *flags &= ~I40E_FLAG_RS_FEC;
14964         }
14965         if (fec_cfg == 0)
14966                 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC);
14967 }
14968
14969 /**
14970  * i40e_check_recovery_mode - check if we are running transition firmware
14971  * @pf: board private structure
14972  *
14973  * Check registers indicating the firmware runs in recovery mode. Sets the
14974  * appropriate driver state.
14975  *
14976  * Returns true if the recovery mode was detected, false otherwise
14977  **/
14978 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
14979 {
14980         u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
14981
14982         if (val & I40E_GL_FWSTS_FWS1B_MASK) {
14983                 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
14984                 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
14985                 set_bit(__I40E_RECOVERY_MODE, pf->state);
14986
14987                 return true;
14988         }
14989         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
14990                 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
14991
14992         return false;
14993 }
14994
14995 /**
14996  * i40e_pf_loop_reset - perform reset in a loop.
14997  * @pf: board private structure
14998  *
14999  * This function is useful when a NIC is about to enter recovery mode.
15000  * When a NIC's internal data structures are corrupted the NIC's
15001  * firmware is going to enter recovery mode.
15002  * Right after a POR it takes about 7 minutes for firmware to enter
15003  * recovery mode. Until that time a NIC is in some kind of intermediate
15004  * state. After that time period the NIC almost surely enters
15005  * recovery mode. The only way for a driver to detect intermediate
15006  * state is to issue a series of pf-resets and check a return value.
15007  * If a PF reset returns success then the firmware could be in recovery
15008  * mode so the caller of this code needs to check for recovery mode
15009  * if this function returns success. There is a little chance that
15010  * firmware will hang in intermediate state forever.
15011  * Since waiting 7 minutes is quite a lot of time this function waits
15012  * 10 seconds and then gives up by returning an error.
15013  *
15014  * Return 0 on success, negative on failure.
15015  **/
15016 static i40e_status i40e_pf_loop_reset(struct i40e_pf *pf)
15017 {
15018         /* wait max 10 seconds for PF reset to succeed */
15019         const unsigned long time_end = jiffies + 10 * HZ;
15020
15021         struct i40e_hw *hw = &pf->hw;
15022         i40e_status ret;
15023
15024         ret = i40e_pf_reset(hw);
15025         while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) {
15026                 usleep_range(10000, 20000);
15027                 ret = i40e_pf_reset(hw);
15028         }
15029
15030         if (ret == I40E_SUCCESS)
15031                 pf->pfr_count++;
15032         else
15033                 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
15034
15035         return ret;
15036 }
15037
15038 /**
15039  * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15040  * @pf: board private structure
15041  *
15042  * Check FW registers to determine if FW issued unexpected EMP Reset.
15043  * Every time when unexpected EMP Reset occurs the FW increments
15044  * a counter of unexpected EMP Resets. When the counter reaches 10
15045  * the FW should enter the Recovery mode
15046  *
15047  * Returns true if FW issued unexpected EMP Reset
15048  **/
15049 static bool i40e_check_fw_empr(struct i40e_pf *pf)
15050 {
15051         const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
15052                            I40E_GL_FWSTS_FWS1B_MASK;
15053         return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
15054                (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
15055 }
15056
15057 /**
15058  * i40e_handle_resets - handle EMP resets and PF resets
15059  * @pf: board private structure
15060  *
15061  * Handle both EMP resets and PF resets and conclude whether there are
15062  * any issues regarding these resets. If there are any issues then
15063  * generate log entry.
15064  *
15065  * Return 0 if NIC is healthy or negative value when there are issues
15066  * with resets
15067  **/
15068 static i40e_status i40e_handle_resets(struct i40e_pf *pf)
15069 {
15070         const i40e_status pfr = i40e_pf_loop_reset(pf);
15071         const bool is_empr = i40e_check_fw_empr(pf);
15072
15073         if (is_empr || pfr != I40E_SUCCESS)
15074                 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
15075
15076         return is_empr ? I40E_ERR_RESET_FAILED : pfr;
15077 }
15078
15079 /**
15080  * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15081  * @pf: board private structure
15082  * @hw: ptr to the hardware info
15083  *
15084  * This function does a minimal setup of all subsystems needed for running
15085  * recovery mode.
15086  *
15087  * Returns 0 on success, negative on failure
15088  **/
15089 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
15090 {
15091         struct i40e_vsi *vsi;
15092         int err;
15093         int v_idx;
15094
15095         pci_save_state(pf->pdev);
15096
15097         /* set up periodic task facility */
15098         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15099         pf->service_timer_period = HZ;
15100
15101         INIT_WORK(&pf->service_task, i40e_service_task);
15102         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15103
15104         err = i40e_init_interrupt_scheme(pf);
15105         if (err)
15106                 goto err_switch_setup;
15107
15108         /* The number of VSIs reported by the FW is the minimum guaranteed
15109          * to us; HW supports far more and we share the remaining pool with
15110          * the other PFs. We allocate space for more than the guarantee with
15111          * the understanding that we might not get them all later.
15112          */
15113         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15114                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15115         else
15116                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15117
15118         /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
15119         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15120                           GFP_KERNEL);
15121         if (!pf->vsi) {
15122                 err = -ENOMEM;
15123                 goto err_switch_setup;
15124         }
15125
15126         /* We allocate one VSI which is needed as absolute minimum
15127          * in order to register the netdev
15128          */
15129         v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
15130         if (v_idx < 0)
15131                 goto err_switch_setup;
15132         pf->lan_vsi = v_idx;
15133         vsi = pf->vsi[v_idx];
15134         if (!vsi)
15135                 goto err_switch_setup;
15136         vsi->alloc_queue_pairs = 1;
15137         err = i40e_config_netdev(vsi);
15138         if (err)
15139                 goto err_switch_setup;
15140         err = register_netdev(vsi->netdev);
15141         if (err)
15142                 goto err_switch_setup;
15143         vsi->netdev_registered = true;
15144         i40e_dbg_pf_init(pf);
15145
15146         err = i40e_setup_misc_vector_for_recovery_mode(pf);
15147         if (err)
15148                 goto err_switch_setup;
15149
15150         /* tell the firmware that we're starting */
15151         i40e_send_version(pf);
15152
15153         /* since everything's happy, start the service_task timer */
15154         mod_timer(&pf->service_timer,
15155                   round_jiffies(jiffies + pf->service_timer_period));
15156
15157         return 0;
15158
15159 err_switch_setup:
15160         i40e_reset_interrupt_capability(pf);
15161         del_timer_sync(&pf->service_timer);
15162         i40e_shutdown_adminq(hw);
15163         iounmap(hw->hw_addr);
15164         pci_disable_pcie_error_reporting(pf->pdev);
15165         pci_release_mem_regions(pf->pdev);
15166         pci_disable_device(pf->pdev);
15167         kfree(pf);
15168
15169         return err;
15170 }
15171
15172 /**
15173  * i40e_probe - Device initialization routine
15174  * @pdev: PCI device information struct
15175  * @ent: entry in i40e_pci_tbl
15176  *
15177  * i40e_probe initializes a PF identified by a pci_dev structure.
15178  * The OS initialization, configuring of the PF private structure,
15179  * and a hardware reset occur.
15180  *
15181  * Returns 0 on success, negative on failure
15182  **/
15183 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
15184 {
15185         struct i40e_aq_get_phy_abilities_resp abilities;
15186 #ifdef CONFIG_I40E_DCB
15187         enum i40e_get_fw_lldp_status_resp lldp_status;
15188         i40e_status status;
15189 #endif /* CONFIG_I40E_DCB */
15190         struct i40e_pf *pf;
15191         struct i40e_hw *hw;
15192         static u16 pfs_found;
15193         u16 wol_nvm_bits;
15194         u16 link_status;
15195         int err;
15196         u32 val;
15197         u32 i;
15198
15199         err = pci_enable_device_mem(pdev);
15200         if (err)
15201                 return err;
15202
15203         /* set up for high or low dma */
15204         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
15205         if (err) {
15206                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
15207                 if (err) {
15208                         dev_err(&pdev->dev,
15209                                 "DMA configuration failed: 0x%x\n", err);
15210                         goto err_dma;
15211                 }
15212         }
15213
15214         /* set up pci connections */
15215         err = pci_request_mem_regions(pdev, i40e_driver_name);
15216         if (err) {
15217                 dev_info(&pdev->dev,
15218                          "pci_request_selected_regions failed %d\n", err);
15219                 goto err_pci_reg;
15220         }
15221
15222         pci_enable_pcie_error_reporting(pdev);
15223         pci_set_master(pdev);
15224
15225         /* Now that we have a PCI connection, we need to do the
15226          * low level device setup.  This is primarily setting up
15227          * the Admin Queue structures and then querying for the
15228          * device's current profile information.
15229          */
15230         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
15231         if (!pf) {
15232                 err = -ENOMEM;
15233                 goto err_pf_alloc;
15234         }
15235         pf->next_vsi = 0;
15236         pf->pdev = pdev;
15237         set_bit(__I40E_DOWN, pf->state);
15238
15239         hw = &pf->hw;
15240         hw->back = pf;
15241
15242         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15243                                 I40E_MAX_CSR_SPACE);
15244         /* We believe that the highest register to read is
15245          * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15246          * is not less than that before mapping to prevent a
15247          * kernel panic.
15248          */
15249         if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15250                 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15251                         pf->ioremap_len);
15252                 err = -ENOMEM;
15253                 goto err_ioremap;
15254         }
15255         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
15256         if (!hw->hw_addr) {
15257                 err = -EIO;
15258                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15259                          (unsigned int)pci_resource_start(pdev, 0),
15260                          pf->ioremap_len, err);
15261                 goto err_ioremap;
15262         }
15263         hw->vendor_id = pdev->vendor;
15264         hw->device_id = pdev->device;
15265         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
15266         hw->subsystem_vendor_id = pdev->subsystem_vendor;
15267         hw->subsystem_device_id = pdev->subsystem_device;
15268         hw->bus.device = PCI_SLOT(pdev->devfn);
15269         hw->bus.func = PCI_FUNC(pdev->devfn);
15270         hw->bus.bus_id = pdev->bus->number;
15271         pf->instance = pfs_found;
15272
15273         /* Select something other than the 802.1ad ethertype for the
15274          * switch to use internally and drop on ingress.
15275          */
15276         hw->switch_tag = 0xffff;
15277         hw->first_tag = ETH_P_8021AD;
15278         hw->second_tag = ETH_P_8021Q;
15279
15280         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
15281         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
15282         INIT_LIST_HEAD(&pf->ddp_old_prof);
15283
15284         /* set up the locks for the AQ, do this only once in probe
15285          * and destroy them only once in remove
15286          */
15287         mutex_init(&hw->aq.asq_mutex);
15288         mutex_init(&hw->aq.arq_mutex);
15289
15290         pf->msg_enable = netif_msg_init(debug,
15291                                         NETIF_MSG_DRV |
15292                                         NETIF_MSG_PROBE |
15293                                         NETIF_MSG_LINK);
15294         if (debug < -1)
15295                 pf->hw.debug_mask = debug;
15296
15297         /* do a special CORER for clearing PXE mode once at init */
15298         if (hw->revision_id == 0 &&
15299             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15300                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15301                 i40e_flush(hw);
15302                 msleep(200);
15303                 pf->corer_count++;
15304
15305                 i40e_clear_pxe_mode(hw);
15306         }
15307
15308         /* Reset here to make sure all is clean and to define PF 'n' */
15309         i40e_clear_hw(hw);
15310
15311         err = i40e_set_mac_type(hw);
15312         if (err) {
15313                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15314                          err);
15315                 goto err_pf_reset;
15316         }
15317
15318         err = i40e_handle_resets(pf);
15319         if (err)
15320                 goto err_pf_reset;
15321
15322         i40e_check_recovery_mode(pf);
15323
15324         hw->aq.num_arq_entries = I40E_AQ_LEN;
15325         hw->aq.num_asq_entries = I40E_AQ_LEN;
15326         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15327         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15328         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
15329
15330         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
15331                  "%s-%s:misc",
15332                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
15333
15334         err = i40e_init_shared_code(hw);
15335         if (err) {
15336                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15337                          err);
15338                 goto err_pf_reset;
15339         }
15340
15341         /* set up a default setting for link flow control */
15342         pf->hw.fc.requested_mode = I40E_FC_NONE;
15343
15344         err = i40e_init_adminq(hw);
15345         if (err) {
15346                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
15347                         dev_info(&pdev->dev,
15348                                  "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15349                                  hw->aq.api_maj_ver,
15350                                  hw->aq.api_min_ver,
15351                                  I40E_FW_API_VERSION_MAJOR,
15352                                  I40E_FW_MINOR_VERSION(hw));
15353                 else
15354                         dev_info(&pdev->dev,
15355                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15356
15357                 goto err_pf_reset;
15358         }
15359         i40e_get_oem_version(hw);
15360
15361         /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15362         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15363                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15364                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
15365                  i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id,
15366                  hw->subsystem_vendor_id, hw->subsystem_device_id);
15367
15368         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
15369             hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
15370                 dev_info(&pdev->dev,
15371                          "The driver for the device detected a newer version of the NVM image v%u.%u than expected v%u.%u. Please install the most recent version of the network driver.\n",
15372                          hw->aq.api_maj_ver,
15373                          hw->aq.api_min_ver,
15374                          I40E_FW_API_VERSION_MAJOR,
15375                          I40E_FW_MINOR_VERSION(hw));
15376         else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
15377                 dev_info(&pdev->dev,
15378                          "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15379                          hw->aq.api_maj_ver,
15380                          hw->aq.api_min_ver,
15381                          I40E_FW_API_VERSION_MAJOR,
15382                          I40E_FW_MINOR_VERSION(hw));
15383
15384         i40e_verify_eeprom(pf);
15385
15386         /* Rev 0 hardware was never productized */
15387         if (hw->revision_id < 1)
15388                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15389
15390         i40e_clear_pxe_mode(hw);
15391
15392         err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
15393         if (err)
15394                 goto err_adminq_setup;
15395
15396         err = i40e_sw_init(pf);
15397         if (err) {
15398                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15399                 goto err_sw_init;
15400         }
15401
15402         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15403                 return i40e_init_recovery_mode(pf, hw);
15404
15405         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
15406                                 hw->func_caps.num_rx_qp, 0, 0);
15407         if (err) {
15408                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15409                 goto err_init_lan_hmc;
15410         }
15411
15412         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
15413         if (err) {
15414                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15415                 err = -ENOENT;
15416                 goto err_configure_lan_hmc;
15417         }
15418
15419         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
15420          * Ignore error return codes because if it was already disabled via
15421          * hardware settings this will fail
15422          */
15423         if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
15424                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15425                 i40e_aq_stop_lldp(hw, true, false, NULL);
15426         }
15427
15428         /* allow a platform config to override the HW addr */
15429         i40e_get_platform_mac_addr(pdev, pf);
15430
15431         if (!is_valid_ether_addr(hw->mac.addr)) {
15432                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15433                 err = -EIO;
15434                 goto err_mac_addr;
15435         }
15436         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15437         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15438         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15439         if (is_valid_ether_addr(hw->mac.port_addr))
15440                 pf->hw_features |= I40E_HW_PORT_ID_VALID;
15441
15442         pci_set_drvdata(pdev, pf);
15443         pci_save_state(pdev);
15444
15445 #ifdef CONFIG_I40E_DCB
15446         status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);
15447         (!status &&
15448          lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
15449                 (pf->flags &= ~I40E_FLAG_DISABLE_FW_LLDP) :
15450                 (pf->flags |= I40E_FLAG_DISABLE_FW_LLDP);
15451         dev_info(&pdev->dev,
15452                  (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ?
15453                         "FW LLDP is disabled\n" :
15454                         "FW LLDP is enabled\n");
15455
15456         /* Enable FW to write default DCB config on link-up */
15457         i40e_aq_set_dcb_parameters(hw, true, NULL);
15458
15459         err = i40e_init_pf_dcb(pf);
15460         if (err) {
15461                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15462                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
15463                 /* Continue without DCB enabled */
15464         }
15465 #endif /* CONFIG_I40E_DCB */
15466
15467         /* set up periodic task facility */
15468         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15469         pf->service_timer_period = HZ;
15470
15471         INIT_WORK(&pf->service_task, i40e_service_task);
15472         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15473
15474         /* NVM bit on means WoL disabled for the port */
15475         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15476         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15477                 pf->wol_en = false;
15478         else
15479                 pf->wol_en = true;
15480         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15481
15482         /* set up the main switch operations */
15483         i40e_determine_queue_usage(pf);
15484         err = i40e_init_interrupt_scheme(pf);
15485         if (err)
15486                 goto err_switch_setup;
15487
15488         pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15489         pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15490         pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15491         pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15492         pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15493         pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15494                                                     UDP_TUNNEL_TYPE_GENEVE;
15495
15496         /* The number of VSIs reported by the FW is the minimum guaranteed
15497          * to us; HW supports far more and we share the remaining pool with
15498          * the other PFs. We allocate space for more than the guarantee with
15499          * the understanding that we might not get them all later.
15500          */
15501         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15502                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15503         else
15504                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15505         if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15506                 dev_warn(&pf->pdev->dev,
15507                          "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15508                          pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15509                 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15510         }
15511
15512         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15513         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15514                           GFP_KERNEL);
15515         if (!pf->vsi) {
15516                 err = -ENOMEM;
15517                 goto err_switch_setup;
15518         }
15519
15520 #ifdef CONFIG_PCI_IOV
15521         /* prep for VF support */
15522         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15523             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15524             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15525                 if (pci_num_vf(pdev))
15526                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
15527         }
15528 #endif
15529         err = i40e_setup_pf_switch(pf, false);
15530         if (err) {
15531                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15532                 goto err_vsis;
15533         }
15534         INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
15535
15536         /* if FDIR VSI was set up, start it now */
15537         for (i = 0; i < pf->num_alloc_vsi; i++) {
15538                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
15539                         i40e_vsi_open(pf->vsi[i]);
15540                         break;
15541                 }
15542         }
15543
15544         /* The driver only wants link up/down and module qualification
15545          * reports from firmware.  Note the negative logic.
15546          */
15547         err = i40e_aq_set_phy_int_mask(&pf->hw,
15548                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
15549                                          I40E_AQ_EVENT_MEDIA_NA |
15550                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15551         if (err)
15552                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
15553                          i40e_stat_str(&pf->hw, err),
15554                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15555
15556         /* Reconfigure hardware for allowing smaller MSS in the case
15557          * of TSO, so that we avoid the MDD being fired and causing
15558          * a reset in the case of small MSS+TSO.
15559          */
15560         val = rd32(hw, I40E_REG_MSS);
15561         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15562                 val &= ~I40E_REG_MSS_MIN_MASK;
15563                 val |= I40E_64BYTE_MSS;
15564                 wr32(hw, I40E_REG_MSS, val);
15565         }
15566
15567         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
15568                 msleep(75);
15569                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
15570                 if (err)
15571                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
15572                                  i40e_stat_str(&pf->hw, err),
15573                                  i40e_aq_str(&pf->hw,
15574                                              pf->hw.aq.asq_last_status));
15575         }
15576         /* The main driver is (mostly) up and happy. We need to set this state
15577          * before setting up the misc vector or we get a race and the vector
15578          * ends up disabled forever.
15579          */
15580         clear_bit(__I40E_DOWN, pf->state);
15581
15582         /* In case of MSIX we are going to setup the misc vector right here
15583          * to handle admin queue events etc. In case of legacy and MSI
15584          * the misc functionality and queue processing is combined in
15585          * the same vector and that gets setup at open.
15586          */
15587         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
15588                 err = i40e_setup_misc_vector(pf);
15589                 if (err) {
15590                         dev_info(&pdev->dev,
15591                                  "setup of misc vector failed: %d\n", err);
15592                         i40e_cloud_filter_exit(pf);
15593                         i40e_fdir_teardown(pf);
15594                         goto err_vsis;
15595                 }
15596         }
15597
15598 #ifdef CONFIG_PCI_IOV
15599         /* prep for VF support */
15600         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15601             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15602             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15603                 /* disable link interrupts for VFs */
15604                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
15605                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
15606                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
15607                 i40e_flush(hw);
15608
15609                 if (pci_num_vf(pdev)) {
15610                         dev_info(&pdev->dev,
15611                                  "Active VFs found, allocating resources.\n");
15612                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
15613                         if (err)
15614                                 dev_info(&pdev->dev,
15615                                          "Error %d allocating resources for existing VFs\n",
15616                                          err);
15617                 }
15618         }
15619 #endif /* CONFIG_PCI_IOV */
15620
15621         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15622                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
15623                                                       pf->num_iwarp_msix,
15624                                                       I40E_IWARP_IRQ_PILE_ID);
15625                 if (pf->iwarp_base_vector < 0) {
15626                         dev_info(&pdev->dev,
15627                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
15628                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
15629                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
15630                 }
15631         }
15632
15633         i40e_dbg_pf_init(pf);
15634
15635         /* tell the firmware that we're starting */
15636         i40e_send_version(pf);
15637
15638         /* since everything's happy, start the service_task timer */
15639         mod_timer(&pf->service_timer,
15640                   round_jiffies(jiffies + pf->service_timer_period));
15641
15642         /* add this PF to client device list and launch a client service task */
15643         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15644                 err = i40e_lan_add_device(pf);
15645                 if (err)
15646                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
15647                                  err);
15648         }
15649
15650 #define PCI_SPEED_SIZE 8
15651 #define PCI_WIDTH_SIZE 8
15652         /* Devices on the IOSF bus do not have this information
15653          * and will report PCI Gen 1 x 1 by default so don't bother
15654          * checking them.
15655          */
15656         if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
15657                 char speed[PCI_SPEED_SIZE] = "Unknown";
15658                 char width[PCI_WIDTH_SIZE] = "Unknown";
15659
15660                 /* Get the negotiated link width and speed from PCI config
15661                  * space
15662                  */
15663                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
15664                                           &link_status);
15665
15666                 i40e_set_pci_config_data(hw, link_status);
15667
15668                 switch (hw->bus.speed) {
15669                 case i40e_bus_speed_8000:
15670                         strlcpy(speed, "8.0", PCI_SPEED_SIZE); break;
15671                 case i40e_bus_speed_5000:
15672                         strlcpy(speed, "5.0", PCI_SPEED_SIZE); break;
15673                 case i40e_bus_speed_2500:
15674                         strlcpy(speed, "2.5", PCI_SPEED_SIZE); break;
15675                 default:
15676                         break;
15677                 }
15678                 switch (hw->bus.width) {
15679                 case i40e_bus_width_pcie_x8:
15680                         strlcpy(width, "8", PCI_WIDTH_SIZE); break;
15681                 case i40e_bus_width_pcie_x4:
15682                         strlcpy(width, "4", PCI_WIDTH_SIZE); break;
15683                 case i40e_bus_width_pcie_x2:
15684                         strlcpy(width, "2", PCI_WIDTH_SIZE); break;
15685                 case i40e_bus_width_pcie_x1:
15686                         strlcpy(width, "1", PCI_WIDTH_SIZE); break;
15687                 default:
15688                         break;
15689                 }
15690
15691                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
15692                          speed, width);
15693
15694                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
15695                     hw->bus.speed < i40e_bus_speed_8000) {
15696                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
15697                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
15698                 }
15699         }
15700
15701         /* get the requested speeds from the fw */
15702         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
15703         if (err)
15704                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
15705                         i40e_stat_str(&pf->hw, err),
15706                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15707         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
15708
15709         /* set the FEC config due to the board capabilities */
15710         i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags);
15711
15712         /* get the supported phy types from the fw */
15713         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
15714         if (err)
15715                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
15716                         i40e_stat_str(&pf->hw, err),
15717                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15718
15719         /* make sure the MFS hasn't been set lower than the default */
15720 #define MAX_FRAME_SIZE_DEFAULT 0x2600
15721         val = (rd32(&pf->hw, I40E_PRTGL_SAH) &
15722                I40E_PRTGL_SAH_MFS_MASK) >> I40E_PRTGL_SAH_MFS_SHIFT;
15723         if (val < MAX_FRAME_SIZE_DEFAULT)
15724                 dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n",
15725                          i, val);
15726
15727         /* Add a filter to drop all Flow control frames from any VSI from being
15728          * transmitted. By doing so we stop a malicious VF from sending out
15729          * PAUSE or PFC frames and potentially controlling traffic for other
15730          * PF/VF VSIs.
15731          * The FW can still send Flow control frames if enabled.
15732          */
15733         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
15734                                                        pf->main_vsi_seid);
15735 #ifdef CONFIG_I40E_DCB
15736         if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)
15737                 i40e_set_lldp_forwarding(pf, true);
15738 #endif /* CONFIG_I40E_DCB */
15739
15740         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
15741                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
15742                 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
15743         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
15744                 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
15745         /* print a string summarizing features */
15746         i40e_print_features(pf);
15747
15748         return 0;
15749
15750         /* Unwind what we've done if something failed in the setup */
15751 err_vsis:
15752         set_bit(__I40E_DOWN, pf->state);
15753         i40e_clear_interrupt_scheme(pf);
15754         kfree(pf->vsi);
15755 err_switch_setup:
15756         i40e_reset_interrupt_capability(pf);
15757         del_timer_sync(&pf->service_timer);
15758 err_mac_addr:
15759 err_configure_lan_hmc:
15760         (void)i40e_shutdown_lan_hmc(hw);
15761 err_init_lan_hmc:
15762         kfree(pf->qp_pile);
15763 err_sw_init:
15764 err_adminq_setup:
15765 err_pf_reset:
15766         iounmap(hw->hw_addr);
15767 err_ioremap:
15768         kfree(pf);
15769 err_pf_alloc:
15770         pci_disable_pcie_error_reporting(pdev);
15771         pci_release_mem_regions(pdev);
15772 err_pci_reg:
15773 err_dma:
15774         pci_disable_device(pdev);
15775         return err;
15776 }
15777
15778 /**
15779  * i40e_remove - Device removal routine
15780  * @pdev: PCI device information struct
15781  *
15782  * i40e_remove is called by the PCI subsystem to alert the driver
15783  * that is should release a PCI device.  This could be caused by a
15784  * Hot-Plug event, or because the driver is going to be removed from
15785  * memory.
15786  **/
15787 static void i40e_remove(struct pci_dev *pdev)
15788 {
15789         struct i40e_pf *pf = pci_get_drvdata(pdev);
15790         struct i40e_hw *hw = &pf->hw;
15791         i40e_status ret_code;
15792         int i;
15793
15794         i40e_dbg_pf_exit(pf);
15795
15796         i40e_ptp_stop(pf);
15797
15798         /* Disable RSS in hw */
15799         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
15800         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
15801
15802         while (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
15803                 usleep_range(1000, 2000);
15804
15805         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
15806                 set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
15807                 i40e_free_vfs(pf);
15808                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
15809         }
15810         /* no more scheduling of any task */
15811         set_bit(__I40E_SUSPENDED, pf->state);
15812         set_bit(__I40E_DOWN, pf->state);
15813         if (pf->service_timer.function)
15814                 del_timer_sync(&pf->service_timer);
15815         if (pf->service_task.func)
15816                 cancel_work_sync(&pf->service_task);
15817
15818         if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
15819                 struct i40e_vsi *vsi = pf->vsi[0];
15820
15821                 /* We know that we have allocated only one vsi for this PF,
15822                  * it was just for registering netdevice, so the interface
15823                  * could be visible in the 'ifconfig' output
15824                  */
15825                 unregister_netdev(vsi->netdev);
15826                 free_netdev(vsi->netdev);
15827
15828                 goto unmap;
15829         }
15830
15831         /* Client close must be called explicitly here because the timer
15832          * has been stopped.
15833          */
15834         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15835
15836         i40e_fdir_teardown(pf);
15837
15838         /* If there is a switch structure or any orphans, remove them.
15839          * This will leave only the PF's VSI remaining.
15840          */
15841         for (i = 0; i < I40E_MAX_VEB; i++) {
15842                 if (!pf->veb[i])
15843                         continue;
15844
15845                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
15846                     pf->veb[i]->uplink_seid == 0)
15847                         i40e_switch_branch_release(pf->veb[i]);
15848         }
15849
15850         /* Now we can shutdown the PF's VSI, just before we kill
15851          * adminq and hmc.
15852          */
15853         if (pf->vsi[pf->lan_vsi])
15854                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
15855
15856         i40e_cloud_filter_exit(pf);
15857
15858         /* remove attached clients */
15859         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15860                 ret_code = i40e_lan_del_device(pf);
15861                 if (ret_code)
15862                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
15863                                  ret_code);
15864         }
15865
15866         /* shutdown and destroy the HMC */
15867         if (hw->hmc.hmc_obj) {
15868                 ret_code = i40e_shutdown_lan_hmc(hw);
15869                 if (ret_code)
15870                         dev_warn(&pdev->dev,
15871                                  "Failed to destroy the HMC resources: %d\n",
15872                                  ret_code);
15873         }
15874
15875 unmap:
15876         /* Free MSI/legacy interrupt 0 when in recovery mode. */
15877         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
15878             !(pf->flags & I40E_FLAG_MSIX_ENABLED))
15879                 free_irq(pf->pdev->irq, pf);
15880
15881         /* shutdown the adminq */
15882         i40e_shutdown_adminq(hw);
15883
15884         /* destroy the locks only once, here */
15885         mutex_destroy(&hw->aq.arq_mutex);
15886         mutex_destroy(&hw->aq.asq_mutex);
15887
15888         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
15889         rtnl_lock();
15890         i40e_clear_interrupt_scheme(pf);
15891         for (i = 0; i < pf->num_alloc_vsi; i++) {
15892                 if (pf->vsi[i]) {
15893                         if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
15894                                 i40e_vsi_clear_rings(pf->vsi[i]);
15895                         i40e_vsi_clear(pf->vsi[i]);
15896                         pf->vsi[i] = NULL;
15897                 }
15898         }
15899         rtnl_unlock();
15900
15901         for (i = 0; i < I40E_MAX_VEB; i++) {
15902                 kfree(pf->veb[i]);
15903                 pf->veb[i] = NULL;
15904         }
15905
15906         kfree(pf->qp_pile);
15907         kfree(pf->vsi);
15908
15909         iounmap(hw->hw_addr);
15910         kfree(pf);
15911         pci_release_mem_regions(pdev);
15912
15913         pci_disable_pcie_error_reporting(pdev);
15914         pci_disable_device(pdev);
15915 }
15916
15917 /**
15918  * i40e_pci_error_detected - warning that something funky happened in PCI land
15919  * @pdev: PCI device information struct
15920  * @error: the type of PCI error
15921  *
15922  * Called to warn that something happened and the error handling steps
15923  * are in progress.  Allows the driver to quiesce things, be ready for
15924  * remediation.
15925  **/
15926 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
15927                                                 pci_channel_state_t error)
15928 {
15929         struct i40e_pf *pf = pci_get_drvdata(pdev);
15930
15931         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
15932
15933         if (!pf) {
15934                 dev_info(&pdev->dev,
15935                          "Cannot recover - error happened during device probe\n");
15936                 return PCI_ERS_RESULT_DISCONNECT;
15937         }
15938
15939         /* shutdown all operations */
15940         if (!test_bit(__I40E_SUSPENDED, pf->state))
15941                 i40e_prep_for_reset(pf);
15942
15943         /* Request a slot reset */
15944         return PCI_ERS_RESULT_NEED_RESET;
15945 }
15946
15947 /**
15948  * i40e_pci_error_slot_reset - a PCI slot reset just happened
15949  * @pdev: PCI device information struct
15950  *
15951  * Called to find if the driver can work with the device now that
15952  * the pci slot has been reset.  If a basic connection seems good
15953  * (registers are readable and have sane content) then return a
15954  * happy little PCI_ERS_RESULT_xxx.
15955  **/
15956 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
15957 {
15958         struct i40e_pf *pf = pci_get_drvdata(pdev);
15959         pci_ers_result_t result;
15960         u32 reg;
15961
15962         dev_dbg(&pdev->dev, "%s\n", __func__);
15963         if (pci_enable_device_mem(pdev)) {
15964                 dev_info(&pdev->dev,
15965                          "Cannot re-enable PCI device after reset.\n");
15966                 result = PCI_ERS_RESULT_DISCONNECT;
15967         } else {
15968                 pci_set_master(pdev);
15969                 pci_restore_state(pdev);
15970                 pci_save_state(pdev);
15971                 pci_wake_from_d3(pdev, false);
15972
15973                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
15974                 if (reg == 0)
15975                         result = PCI_ERS_RESULT_RECOVERED;
15976                 else
15977                         result = PCI_ERS_RESULT_DISCONNECT;
15978         }
15979
15980         return result;
15981 }
15982
15983 /**
15984  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
15985  * @pdev: PCI device information struct
15986  */
15987 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
15988 {
15989         struct i40e_pf *pf = pci_get_drvdata(pdev);
15990
15991         i40e_prep_for_reset(pf);
15992 }
15993
15994 /**
15995  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
15996  * @pdev: PCI device information struct
15997  */
15998 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
15999 {
16000         struct i40e_pf *pf = pci_get_drvdata(pdev);
16001
16002         i40e_reset_and_rebuild(pf, false, false);
16003 }
16004
16005 /**
16006  * i40e_pci_error_resume - restart operations after PCI error recovery
16007  * @pdev: PCI device information struct
16008  *
16009  * Called to allow the driver to bring things back up after PCI error
16010  * and/or reset recovery has finished.
16011  **/
16012 static void i40e_pci_error_resume(struct pci_dev *pdev)
16013 {
16014         struct i40e_pf *pf = pci_get_drvdata(pdev);
16015
16016         dev_dbg(&pdev->dev, "%s\n", __func__);
16017         if (test_bit(__I40E_SUSPENDED, pf->state))
16018                 return;
16019
16020         i40e_handle_reset_warning(pf, false);
16021 }
16022
16023 /**
16024  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16025  * using the mac_address_write admin q function
16026  * @pf: pointer to i40e_pf struct
16027  **/
16028 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
16029 {
16030         struct i40e_hw *hw = &pf->hw;
16031         i40e_status ret;
16032         u8 mac_addr[6];
16033         u16 flags = 0;
16034
16035         /* Get current MAC address in case it's an LAA */
16036         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
16037                 ether_addr_copy(mac_addr,
16038                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
16039         } else {
16040                 dev_err(&pf->pdev->dev,
16041                         "Failed to retrieve MAC address; using default\n");
16042                 ether_addr_copy(mac_addr, hw->mac.addr);
16043         }
16044
16045         /* The FW expects the mac address write cmd to first be called with
16046          * one of these flags before calling it again with the multicast
16047          * enable flags.
16048          */
16049         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
16050
16051         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
16052                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
16053
16054         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16055         if (ret) {
16056                 dev_err(&pf->pdev->dev,
16057                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
16058                 return;
16059         }
16060
16061         flags = I40E_AQC_MC_MAG_EN
16062                         | I40E_AQC_WOL_PRESERVE_ON_PFR
16063                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
16064         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16065         if (ret)
16066                 dev_err(&pf->pdev->dev,
16067                         "Failed to enable Multicast Magic Packet wake up\n");
16068 }
16069
16070 /**
16071  * i40e_shutdown - PCI callback for shutting down
16072  * @pdev: PCI device information struct
16073  **/
16074 static void i40e_shutdown(struct pci_dev *pdev)
16075 {
16076         struct i40e_pf *pf = pci_get_drvdata(pdev);
16077         struct i40e_hw *hw = &pf->hw;
16078
16079         set_bit(__I40E_SUSPENDED, pf->state);
16080         set_bit(__I40E_DOWN, pf->state);
16081
16082         del_timer_sync(&pf->service_timer);
16083         cancel_work_sync(&pf->service_task);
16084         i40e_cloud_filter_exit(pf);
16085         i40e_fdir_teardown(pf);
16086
16087         /* Client close must be called explicitly here because the timer
16088          * has been stopped.
16089          */
16090         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16091
16092         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
16093                 i40e_enable_mc_magic_wake(pf);
16094
16095         i40e_prep_for_reset(pf);
16096
16097         wr32(hw, I40E_PFPM_APM,
16098              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16099         wr32(hw, I40E_PFPM_WUFC,
16100              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16101
16102         /* Free MSI/legacy interrupt 0 when in recovery mode. */
16103         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16104             !(pf->flags & I40E_FLAG_MSIX_ENABLED))
16105                 free_irq(pf->pdev->irq, pf);
16106
16107         /* Since we're going to destroy queues during the
16108          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16109          * whole section
16110          */
16111         rtnl_lock();
16112         i40e_clear_interrupt_scheme(pf);
16113         rtnl_unlock();
16114
16115         if (system_state == SYSTEM_POWER_OFF) {
16116                 pci_wake_from_d3(pdev, pf->wol_en);
16117                 pci_set_power_state(pdev, PCI_D3hot);
16118         }
16119 }
16120
16121 /**
16122  * i40e_suspend - PM callback for moving to D3
16123  * @dev: generic device information structure
16124  **/
16125 static int __maybe_unused i40e_suspend(struct device *dev)
16126 {
16127         struct i40e_pf *pf = dev_get_drvdata(dev);
16128         struct i40e_hw *hw = &pf->hw;
16129
16130         /* If we're already suspended, then there is nothing to do */
16131         if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
16132                 return 0;
16133
16134         set_bit(__I40E_DOWN, pf->state);
16135
16136         /* Ensure service task will not be running */
16137         del_timer_sync(&pf->service_timer);
16138         cancel_work_sync(&pf->service_task);
16139
16140         /* Client close must be called explicitly here because the timer
16141          * has been stopped.
16142          */
16143         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16144
16145         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
16146                 i40e_enable_mc_magic_wake(pf);
16147
16148         /* Since we're going to destroy queues during the
16149          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16150          * whole section
16151          */
16152         rtnl_lock();
16153
16154         i40e_prep_for_reset(pf);
16155
16156         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16157         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16158
16159         /* Clear the interrupt scheme and release our IRQs so that the system
16160          * can safely hibernate even when there are a large number of CPUs.
16161          * Otherwise hibernation might fail when mapping all the vectors back
16162          * to CPU0.
16163          */
16164         i40e_clear_interrupt_scheme(pf);
16165
16166         rtnl_unlock();
16167
16168         return 0;
16169 }
16170
16171 /**
16172  * i40e_resume - PM callback for waking up from D3
16173  * @dev: generic device information structure
16174  **/
16175 static int __maybe_unused i40e_resume(struct device *dev)
16176 {
16177         struct i40e_pf *pf = dev_get_drvdata(dev);
16178         int err;
16179
16180         /* If we're not suspended, then there is nothing to do */
16181         if (!test_bit(__I40E_SUSPENDED, pf->state))
16182                 return 0;
16183
16184         /* We need to hold the RTNL lock prior to restoring interrupt schemes,
16185          * since we're going to be restoring queues
16186          */
16187         rtnl_lock();
16188
16189         /* We cleared the interrupt scheme when we suspended, so we need to
16190          * restore it now to resume device functionality.
16191          */
16192         err = i40e_restore_interrupt_scheme(pf);
16193         if (err) {
16194                 dev_err(dev, "Cannot restore interrupt scheme: %d\n",
16195                         err);
16196         }
16197
16198         clear_bit(__I40E_DOWN, pf->state);
16199         i40e_reset_and_rebuild(pf, false, true);
16200
16201         rtnl_unlock();
16202
16203         /* Clear suspended state last after everything is recovered */
16204         clear_bit(__I40E_SUSPENDED, pf->state);
16205
16206         /* Restart the service task */
16207         mod_timer(&pf->service_timer,
16208                   round_jiffies(jiffies + pf->service_timer_period));
16209
16210         return 0;
16211 }
16212
16213 static const struct pci_error_handlers i40e_err_handler = {
16214         .error_detected = i40e_pci_error_detected,
16215         .slot_reset = i40e_pci_error_slot_reset,
16216         .reset_prepare = i40e_pci_error_reset_prepare,
16217         .reset_done = i40e_pci_error_reset_done,
16218         .resume = i40e_pci_error_resume,
16219 };
16220
16221 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
16222
16223 static struct pci_driver i40e_driver = {
16224         .name     = i40e_driver_name,
16225         .id_table = i40e_pci_tbl,
16226         .probe    = i40e_probe,
16227         .remove   = i40e_remove,
16228         .driver   = {
16229                 .pm = &i40e_pm_ops,
16230         },
16231         .shutdown = i40e_shutdown,
16232         .err_handler = &i40e_err_handler,
16233         .sriov_configure = i40e_pci_sriov_configure,
16234 };
16235
16236 /**
16237  * i40e_init_module - Driver registration routine
16238  *
16239  * i40e_init_module is the first routine called when the driver is
16240  * loaded. All it does is register with the PCI subsystem.
16241  **/
16242 static int __init i40e_init_module(void)
16243 {
16244         pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16245         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16246
16247         /* There is no need to throttle the number of active tasks because
16248          * each device limits its own task using a state bit for scheduling
16249          * the service task, and the device tasks do not interfere with each
16250          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16251          * since we need to be able to guarantee forward progress even under
16252          * memory pressure.
16253          */
16254         i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
16255         if (!i40e_wq) {
16256                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16257                 return -ENOMEM;
16258         }
16259
16260         i40e_dbg_init();
16261         return pci_register_driver(&i40e_driver);
16262 }
16263 module_init(i40e_init_module);
16264
16265 /**
16266  * i40e_exit_module - Driver exit cleanup routine
16267  *
16268  * i40e_exit_module is called just before the driver is removed
16269  * from memory.
16270  **/
16271 static void __exit i40e_exit_module(void)
16272 {
16273         pci_unregister_driver(&i40e_driver);
16274         destroy_workqueue(i40e_wq);
16275         i40e_dbg_exit();
16276 }
16277 module_exit(i40e_exit_module);