drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2017 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 #include <linux/etherdevice.h>
  28 #include <linux/of_net.h>
  29 #include <linux/pci.h>
  30 #include <linux/bpf.h>
  31
  32 /* Local includes */
  33 #include "i40e.h"
  34 #include "i40e_diag.h"
  35 #include <net/udp_tunnel.h>
  36 /* All i40e tracepoints are defined by the include below, which
  37  * must be included exactly once across the whole kernel with
  38  * CREATE_TRACE_POINTS defined
  39  */
  40 #define CREATE_TRACE_POINTS
  41 #include "i40e_trace.h"
  42
  43 const char i40e_driver_name[] = "i40e";
  44 static const char i40e_driver_string[] =
  45                         "Intel(R) Ethernet Connection XL710 Network Driver";
  46
  47 #define DRV_KERN "-k"
  48
  49 #define DRV_VERSION_MAJOR 2
  50 #define DRV_VERSION_MINOR 1
  51 #define DRV_VERSION_BUILD 14
  52 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  53              __stringify(DRV_VERSION_MINOR) "." \
  54              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  55 const char i40e_driver_version_str[] = DRV_VERSION;
  56 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  57
  58 /* a bit of forward declarations */
  59 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  60 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  61 static int i40e_add_vsi(struct i40e_vsi *vsi);
  62 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  63 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  64 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  65 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  66 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  67 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
  68 static int i40e_reset(struct i40e_pf *pf);
  69 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  70 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  71 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  72
  73 /* i40e_pci_tbl - PCI Device ID Table
  74  *
  75  * Last entry must be all 0s
  76  *
  77  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  78  *   Class, Class Mask, private data (not used) }
  79  */
  80 static const struct pci_device_id i40e_pci_tbl[] = {
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  86         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  87         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  88         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  89         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  90         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  91         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  92         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  93         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  94         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  95         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  96         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  97         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  98         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  99         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
 100         /* required last entry */
 101         {0, }
 102 };
 103 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
 104
 105 #define I40E_MAX_VF_COUNT 128
 106 static int debug = -1;
 107 module_param(debug, uint, 0);
 108 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
 109
 110 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 111 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
 112 MODULE_LICENSE("GPL");
 113 MODULE_VERSION(DRV_VERSION);
 114
 115 static struct workqueue_struct *i40e_wq;
 116
 117 /**
 118  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 119  * @hw:   pointer to the HW structure
 120  * @mem:  ptr to mem struct to fill out
 121  * @size: size of memory requested
 122  * @alignment: what to align the allocation to
 123  **/
 124 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 125                             u64 size, u32 alignment)
 126 {
 127         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 128
 129         mem->size = ALIGN(size, alignment);
 130         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 131                                       &mem->pa, GFP_KERNEL);
 132         if (!mem->va)
 133                 return -ENOMEM;
 134
 135         return 0;
 136 }
 137
 138 /**
 139  * i40e_free_dma_mem_d - OS specific memory free for shared code
 140  * @hw:   pointer to the HW structure
 141  * @mem:  ptr to mem struct to free
 142  **/
 143 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 144 {
 145         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 146
 147         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 148         mem->va = NULL;
 149         mem->pa = 0;
 150         mem->size = 0;
 151
 152         return 0;
 153 }
 154
 155 /**
 156  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 157  * @hw:   pointer to the HW structure
 158  * @mem:  ptr to mem struct to fill out
 159  * @size: size of memory requested
 160  **/
 161 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 162                              u32 size)
 163 {
 164         mem->size = size;
 165         mem->va = kzalloc(size, GFP_KERNEL);
 166
 167         if (!mem->va)
 168                 return -ENOMEM;
 169
 170         return 0;
 171 }
 172
 173 /**
 174  * i40e_free_virt_mem_d - OS specific memory free for shared code
 175  * @hw:   pointer to the HW structure
 176  * @mem:  ptr to mem struct to free
 177  **/
 178 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 179 {
 180         /* it's ok to kfree a NULL pointer */
 181         kfree(mem->va);
 182         mem->va = NULL;
 183         mem->size = 0;
 184
 185         return 0;
 186 }
 187
 188 /**
 189  * i40e_get_lump - find a lump of free generic resource
 190  * @pf: board private structure
 191  * @pile: the pile of resource to search
 192  * @needed: the number of items needed
 193  * @id: an owner id to stick on the items assigned
 194  *
 195  * Returns the base item index of the lump, or negative for error
 196  *
 197  * The search_hint trick and lack of advanced fit-finding only work
 198  * because we're highly likely to have all the same size lump requests.
 199  * Linear search time and any fragmentation should be minimal.
 200  **/
 201 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 202                          u16 needed, u16 id)
 203 {
 204         int ret = -ENOMEM;
 205         int i, j;
 206
 207         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 208                 dev_info(&pf->pdev->dev,
 209                          "param err: pile=%p needed=%d id=0x%04x\n",
 210                          pile, needed, id);
 211                 return -EINVAL;
 212         }
 213
 214         /* start the linear search with an imperfect hint */
 215         i = pile->search_hint;
 216         while (i < pile->num_entries) {
 217                 /* skip already allocated entries */
 218                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 219                         i++;
 220                         continue;
 221                 }
 222
 223                 /* do we have enough in this lump? */
 224                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 225                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 226                                 break;
 227                 }
 228
 229                 if (j == needed) {
 230                         /* there was enough, so assign it to the requestor */
 231                         for (j = 0; j < needed; j++)
 232                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 233                         ret = i;
 234                         pile->search_hint = i + j;
 235                         break;
 236                 }
 237
 238                 /* not enough, so skip over it and continue looking */
 239                 i += j;
 240         }
 241
 242         return ret;
 243 }
 244
 245 /**
 246  * i40e_put_lump - return a lump of generic resource
 247  * @pile: the pile of resource to search
 248  * @index: the base item index
 249  * @id: the owner id of the items assigned
 250  *
 251  * Returns the count of items in the lump
 252  **/
 253 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 254 {
 255         int valid_id = (id | I40E_PILE_VALID_BIT);
 256         int count = 0;
 257         int i;
 258
 259         if (!pile || index >= pile->num_entries)
 260                 return -EINVAL;
 261
 262         for (i = index;
 263              i < pile->num_entries && pile->list[i] == valid_id;
 264              i++) {
 265                 pile->list[i] = 0;
 266                 count++;
 267         }
 268
 269         if (count && index < pile->search_hint)
 270                 pile->search_hint = index;
 271
 272         return count;
 273 }
 274
 275 /**
 276  * i40e_find_vsi_from_id - searches for the vsi with the given id
 277  * @pf - the pf structure to search for the vsi
 278  * @id - id of the vsi it is searching for
 279  **/
 280 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 281 {
 282         int i;
 283
 284         for (i = 0; i < pf->num_alloc_vsi; i++)
 285                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 286                         return pf->vsi[i];
 287
 288         return NULL;
 289 }
 290
 291 /**
 292  * i40e_service_event_schedule - Schedule the service task to wake up
 293  * @pf: board private structure
 294  *
 295  * If not already scheduled, this puts the task into the work queue
 296  **/
 297 void i40e_service_event_schedule(struct i40e_pf *pf)
 298 {
 299         if (!test_bit(__I40E_DOWN, pf->state) &&
 300             !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
 301                 queue_work(i40e_wq, &pf->service_task);
 302 }
 303
 304 /**
 305  * i40e_tx_timeout - Respond to a Tx Hang
 306  * @netdev: network interface device structure
 307  *
 308  * If any port has noticed a Tx timeout, it is likely that the whole
 309  * device is munged, not just the one netdev port, so go for the full
 310  * reset.
 311  **/
 312 static void i40e_tx_timeout(struct net_device *netdev)
 313 {
 314         struct i40e_netdev_priv *np = netdev_priv(netdev);
 315         struct i40e_vsi *vsi = np->vsi;
 316         struct i40e_pf *pf = vsi->back;
 317         struct i40e_ring *tx_ring = NULL;
 318         unsigned int i, hung_queue = 0;
 319         u32 head, val;
 320
 321         pf->tx_timeout_count++;
 322
 323         /* find the stopped queue the same way the stack does */
 324         for (i = 0; i < netdev->num_tx_queues; i++) {
 325                 struct netdev_queue *q;
 326                 unsigned long trans_start;
 327
 328                 q = netdev_get_tx_queue(netdev, i);
 329                 trans_start = q->trans_start;
 330                 if (netif_xmit_stopped(q) &&
 331                     time_after(jiffies,
 332                                (trans_start + netdev->watchdog_timeo))) {
 333                         hung_queue = i;
 334                         break;
 335                 }
 336         }
 337
 338         if (i == netdev->num_tx_queues) {
 339                 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
 340         } else {
 341                 /* now that we have an index, find the tx_ring struct */
 342                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 343                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 344                                 if (hung_queue ==
 345                                     vsi->tx_rings[i]->queue_index) {
 346                                         tx_ring = vsi->tx_rings[i];
 347                                         break;
 348                                 }
 349                         }
 350                 }
 351         }
 352
 353         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 354                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 355         else if (time_before(jiffies,
 356                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 357                 return;   /* don't do any new action before the next timeout */
 358
 359         if (tx_ring) {
 360                 head = i40e_get_head(tx_ring);
 361                 /* Read interrupt register */
 362                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 363                         val = rd32(&pf->hw,
 364                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 365                                                 tx_ring->vsi->base_vector - 1));
 366                 else
 367                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 368
 369                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 370                             vsi->seid, hung_queue, tx_ring->next_to_clean,
 371                             head, tx_ring->next_to_use,
 372                             readl(tx_ring->tail), val);
 373         }
 374
 375         pf->tx_timeout_last_recovery = jiffies;
 376         netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
 377                     pf->tx_timeout_recovery_level, hung_queue);
 378
 379         switch (pf->tx_timeout_recovery_level) {
 380         case 1:
 381                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
 382                 break;
 383         case 2:
 384                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
 385                 break;
 386         case 3:
 387                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
 388                 break;
 389         default:
 390                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 391                 break;
 392         }
 393
 394         i40e_service_event_schedule(pf);
 395         pf->tx_timeout_recovery_level++;
 396 }
 397
 398 /**
 399  * i40e_get_vsi_stats_struct - Get System Network Statistics
 400  * @vsi: the VSI we care about
 401  *
 402  * Returns the address of the device statistics structure.
 403  * The statistics are actually updated from the service task.
 404  **/
 405 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 406 {
 407         return &vsi->net_stats;
 408 }
 409
 410 /**
 411  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
 412  * @ring: Tx ring to get statistics from
 413  * @stats: statistics entry to be updated
 414  **/
 415 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 416                                             struct rtnl_link_stats64 *stats)
 417 {
 418         u64 bytes, packets;
 419         unsigned int start;
 420
 421         do {
 422                 start = u64_stats_fetch_begin_irq(&ring->syncp);
 423                 packets = ring->stats.packets;
 424                 bytes   = ring->stats.bytes;
 425         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 426
 427         stats->tx_packets += packets;
 428         stats->tx_bytes   += bytes;
 429 }
 430
 431 /**
 432  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 433  * @netdev: network interface device structure
 434  *
 435  * Returns the address of the device statistics structure.
 436  * The statistics are actually updated from the service task.
 437  **/
 438 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 439                                   struct rtnl_link_stats64 *stats)
 440 {
 441         struct i40e_netdev_priv *np = netdev_priv(netdev);
 442         struct i40e_ring *tx_ring, *rx_ring;
 443         struct i40e_vsi *vsi = np->vsi;
 444         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 445         int i;
 446
 447         if (test_bit(__I40E_VSI_DOWN, vsi->state))
 448                 return;
 449
 450         if (!vsi->tx_rings)
 451                 return;
 452
 453         rcu_read_lock();
 454         for (i = 0; i < vsi->num_queue_pairs; i++) {
 455                 u64 bytes, packets;
 456                 unsigned int start;
 457
 458                 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
 459                 if (!tx_ring)
 460                         continue;
 461                 i40e_get_netdev_stats_struct_tx(tx_ring, stats);
 462
 463                 rx_ring = &tx_ring[1];
 464
 465                 do {
 466                         start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
 467                         packets = rx_ring->stats.packets;
 468                         bytes   = rx_ring->stats.bytes;
 469                 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
 470
 471                 stats->rx_packets += packets;
 472                 stats->rx_bytes   += bytes;
 473
 474                 if (i40e_enabled_xdp_vsi(vsi))
 475                         i40e_get_netdev_stats_struct_tx(&rx_ring[1], stats);
 476         }
 477         rcu_read_unlock();
 478
 479         /* following stats updated by i40e_watchdog_subtask() */
 480         stats->multicast        = vsi_stats->multicast;
 481         stats->tx_errors        = vsi_stats->tx_errors;
 482         stats->tx_dropped       = vsi_stats->tx_dropped;
 483         stats->rx_errors        = vsi_stats->rx_errors;
 484         stats->rx_dropped       = vsi_stats->rx_dropped;
 485         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 486         stats->rx_length_errors = vsi_stats->rx_length_errors;
 487 }
 488
 489 /**
 490  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 491  * @vsi: the VSI to have its stats reset
 492  **/
 493 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 494 {
 495         struct rtnl_link_stats64 *ns;
 496         int i;
 497
 498         if (!vsi)
 499                 return;
 500
 501         ns = i40e_get_vsi_stats_struct(vsi);
 502         memset(ns, 0, sizeof(*ns));
 503         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 504         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 505         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 506         if (vsi->rx_rings && vsi->rx_rings[0]) {
 507                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 508                         memset(&vsi->rx_rings[i]->stats, 0,
 509                                sizeof(vsi->rx_rings[i]->stats));
 510                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 511                                sizeof(vsi->rx_rings[i]->rx_stats));
 512                         memset(&vsi->tx_rings[i]->stats, 0,
 513                                sizeof(vsi->tx_rings[i]->stats));
 514                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 515                                sizeof(vsi->tx_rings[i]->tx_stats));
 516                 }
 517         }
 518         vsi->stat_offsets_loaded = false;
 519 }
 520
 521 /**
 522  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 523  * @pf: the PF to be reset
 524  **/
 525 void i40e_pf_reset_stats(struct i40e_pf *pf)
 526 {
 527         int i;
 528
 529         memset(&pf->stats, 0, sizeof(pf->stats));
 530         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 531         pf->stat_offsets_loaded = false;
 532
 533         for (i = 0; i < I40E_MAX_VEB; i++) {
 534                 if (pf->veb[i]) {
 535                         memset(&pf->veb[i]->stats, 0,
 536                                sizeof(pf->veb[i]->stats));
 537                         memset(&pf->veb[i]->stats_offsets, 0,
 538                                sizeof(pf->veb[i]->stats_offsets));
 539                         pf->veb[i]->stat_offsets_loaded = false;
 540                 }
 541         }
 542         pf->hw_csum_rx_error = 0;
 543 }
 544
 545 /**
 546  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 547  * @hw: ptr to the hardware info
 548  * @hireg: the high 32 bit reg to read
 549  * @loreg: the low 32 bit reg to read
 550  * @offset_loaded: has the initial offset been loaded yet
 551  * @offset: ptr to current offset value
 552  * @stat: ptr to the stat
 553  *
 554  * Since the device stats are not reset at PFReset, they likely will not
 555  * be zeroed when the driver starts.  We'll save the first values read
 556  * and use them as offsets to be subtracted from the raw values in order
 557  * to report stats that count from zero.  In the process, we also manage
 558  * the potential roll-over.
 559  **/
 560 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 561                                bool offset_loaded, u64 *offset, u64 *stat)
 562 {
 563         u64 new_data;
 564
 565         if (hw->device_id == I40E_DEV_ID_QEMU) {
 566                 new_data = rd32(hw, loreg);
 567                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 568         } else {
 569                 new_data = rd64(hw, loreg);
 570         }
 571         if (!offset_loaded)
 572                 *offset = new_data;
 573         if (likely(new_data >= *offset))
 574                 *stat = new_data - *offset;
 575         else
 576                 *stat = (new_data + BIT_ULL(48)) - *offset;
 577         *stat &= 0xFFFFFFFFFFFFULL;
 578 }
 579
 580 /**
 581  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 582  * @hw: ptr to the hardware info
 583  * @reg: the hw reg to read
 584  * @offset_loaded: has the initial offset been loaded yet
 585  * @offset: ptr to current offset value
 586  * @stat: ptr to the stat
 587  **/
 588 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 589                                bool offset_loaded, u64 *offset, u64 *stat)
 590 {
 591         u32 new_data;
 592
 593         new_data = rd32(hw, reg);
 594         if (!offset_loaded)
 595                 *offset = new_data;
 596         if (likely(new_data >= *offset))
 597                 *stat = (u32)(new_data - *offset);
 598         else
 599                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 600 }
 601
 602 /**
 603  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
 604  * @hw: ptr to the hardware info
 605  * @reg: the hw reg to read and clear
 606  * @stat: ptr to the stat
 607  **/
 608 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
 609 {
 610         u32 new_data = rd32(hw, reg);
 611
 612         wr32(hw, reg, 1); /* must write a nonzero value to clear register */
 613         *stat += new_data;
 614 }
 615
 616 /**
 617  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 618  * @vsi: the VSI to be updated
 619  **/
 620 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 621 {
 622         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 623         struct i40e_pf *pf = vsi->back;
 624         struct i40e_hw *hw = &pf->hw;
 625         struct i40e_eth_stats *oes;
 626         struct i40e_eth_stats *es;     /* device's eth stats */
 627
 628         es = &vsi->eth_stats;
 629         oes = &vsi->eth_stats_offsets;
 630
 631         /* Gather up the stats that the hw collects */
 632         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 633                            vsi->stat_offsets_loaded,
 634                            &oes->tx_errors, &es->tx_errors);
 635         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 636                            vsi->stat_offsets_loaded,
 637                            &oes->rx_discards, &es->rx_discards);
 638         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 639                            vsi->stat_offsets_loaded,
 640                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 641         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 642                            vsi->stat_offsets_loaded,
 643                            &oes->tx_errors, &es->tx_errors);
 644
 645         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 646                            I40E_GLV_GORCL(stat_idx),
 647                            vsi->stat_offsets_loaded,
 648                            &oes->rx_bytes, &es->rx_bytes);
 649         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 650                            I40E_GLV_UPRCL(stat_idx),
 651                            vsi->stat_offsets_loaded,
 652                            &oes->rx_unicast, &es->rx_unicast);
 653         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 654                            I40E_GLV_MPRCL(stat_idx),
 655                            vsi->stat_offsets_loaded,
 656                            &oes->rx_multicast, &es->rx_multicast);
 657         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 658                            I40E_GLV_BPRCL(stat_idx),
 659                            vsi->stat_offsets_loaded,
 660                            &oes->rx_broadcast, &es->rx_broadcast);
 661
 662         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 663                            I40E_GLV_GOTCL(stat_idx),
 664                            vsi->stat_offsets_loaded,
 665                            &oes->tx_bytes, &es->tx_bytes);
 666         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 667                            I40E_GLV_UPTCL(stat_idx),
 668                            vsi->stat_offsets_loaded,
 669                            &oes->tx_unicast, &es->tx_unicast);
 670         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 671                            I40E_GLV_MPTCL(stat_idx),
 672                            vsi->stat_offsets_loaded,
 673                            &oes->tx_multicast, &es->tx_multicast);
 674         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 675                            I40E_GLV_BPTCL(stat_idx),
 676                            vsi->stat_offsets_loaded,
 677                            &oes->tx_broadcast, &es->tx_broadcast);
 678         vsi->stat_offsets_loaded = true;
 679 }
 680
 681 /**
 682  * i40e_update_veb_stats - Update Switch component statistics
 683  * @veb: the VEB being updated
 684  **/
 685 static void i40e_update_veb_stats(struct i40e_veb *veb)
 686 {
 687         struct i40e_pf *pf = veb->pf;
 688         struct i40e_hw *hw = &pf->hw;
 689         struct i40e_eth_stats *oes;
 690         struct i40e_eth_stats *es;     /* device's eth stats */
 691         struct i40e_veb_tc_stats *veb_oes;
 692         struct i40e_veb_tc_stats *veb_es;
 693         int i, idx = 0;
 694
 695         idx = veb->stats_idx;
 696         es = &veb->stats;
 697         oes = &veb->stats_offsets;
 698         veb_es = &veb->tc_stats;
 699         veb_oes = &veb->tc_stats_offsets;
 700
 701         /* Gather up the stats that the hw collects */
 702         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 703                            veb->stat_offsets_loaded,
 704                            &oes->tx_discards, &es->tx_discards);
 705         if (hw->revision_id > 0)
 706                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 707                                    veb->stat_offsets_loaded,
 708                                    &oes->rx_unknown_protocol,
 709                                    &es->rx_unknown_protocol);
 710         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 711                            veb->stat_offsets_loaded,
 712                            &oes->rx_bytes, &es->rx_bytes);
 713         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 714                            veb->stat_offsets_loaded,
 715                            &oes->rx_unicast, &es->rx_unicast);
 716         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 717                            veb->stat_offsets_loaded,
 718                            &oes->rx_multicast, &es->rx_multicast);
 719         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 720                            veb->stat_offsets_loaded,
 721                            &oes->rx_broadcast, &es->rx_broadcast);
 722
 723         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 724                            veb->stat_offsets_loaded,
 725                            &oes->tx_bytes, &es->tx_bytes);
 726         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 727                            veb->stat_offsets_loaded,
 728                            &oes->tx_unicast, &es->tx_unicast);
 729         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 730                            veb->stat_offsets_loaded,
 731                            &oes->tx_multicast, &es->tx_multicast);
 732         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 733                            veb->stat_offsets_loaded,
 734                            &oes->tx_broadcast, &es->tx_broadcast);
 735         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 736                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 737                                    I40E_GLVEBTC_RPCL(i, idx),
 738                                    veb->stat_offsets_loaded,
 739                                    &veb_oes->tc_rx_packets[i],
 740                                    &veb_es->tc_rx_packets[i]);
 741                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 742                                    I40E_GLVEBTC_RBCL(i, idx),
 743                                    veb->stat_offsets_loaded,
 744                                    &veb_oes->tc_rx_bytes[i],
 745                                    &veb_es->tc_rx_bytes[i]);
 746                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 747                                    I40E_GLVEBTC_TPCL(i, idx),
 748                                    veb->stat_offsets_loaded,
 749                                    &veb_oes->tc_tx_packets[i],
 750                                    &veb_es->tc_tx_packets[i]);
 751                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 752                                    I40E_GLVEBTC_TBCL(i, idx),
 753                                    veb->stat_offsets_loaded,
 754                                    &veb_oes->tc_tx_bytes[i],
 755                                    &veb_es->tc_tx_bytes[i]);
 756         }
 757         veb->stat_offsets_loaded = true;
 758 }
 759
 760 /**
 761  * i40e_update_vsi_stats - Update the vsi statistics counters.
 762  * @vsi: the VSI to be updated
 763  *
 764  * There are a few instances where we store the same stat in a
 765  * couple of different structs.  This is partly because we have
 766  * the netdev stats that need to be filled out, which is slightly
 767  * different from the "eth_stats" defined by the chip and used in
 768  * VF communications.  We sort it out here.
 769  **/
 770 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 771 {
 772         struct i40e_pf *pf = vsi->back;
 773         struct rtnl_link_stats64 *ons;
 774         struct rtnl_link_stats64 *ns;   /* netdev stats */
 775         struct i40e_eth_stats *oes;
 776         struct i40e_eth_stats *es;     /* device's eth stats */
 777         u32 tx_restart, tx_busy;
 778         struct i40e_ring *p;
 779         u32 rx_page, rx_buf;
 780         u64 bytes, packets;
 781         unsigned int start;
 782         u64 tx_linearize;
 783         u64 tx_force_wb;
 784         u64 rx_p, rx_b;
 785         u64 tx_p, tx_b;
 786         u16 q;
 787
 788         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
 789             test_bit(__I40E_CONFIG_BUSY, pf->state))
 790                 return;
 791
 792         ns = i40e_get_vsi_stats_struct(vsi);
 793         ons = &vsi->net_stats_offsets;
 794         es = &vsi->eth_stats;
 795         oes = &vsi->eth_stats_offsets;
 796
 797         /* Gather up the netdev and vsi stats that the driver collects
 798          * on the fly during packet processing
 799          */
 800         rx_b = rx_p = 0;
 801         tx_b = tx_p = 0;
 802         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 803         rx_page = 0;
 804         rx_buf = 0;
 805         rcu_read_lock();
 806         for (q = 0; q < vsi->num_queue_pairs; q++) {
 807                 /* locate Tx ring */
 808                 p = ACCESS_ONCE(vsi->tx_rings[q]);
 809
 810                 do {
 811                         start = u64_stats_fetch_begin_irq(&p->syncp);
 812                         packets = p->stats.packets;
 813                         bytes = p->stats.bytes;
 814                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 815                 tx_b += bytes;
 816                 tx_p += packets;
 817                 tx_restart += p->tx_stats.restart_queue;
 818                 tx_busy += p->tx_stats.tx_busy;
 819                 tx_linearize += p->tx_stats.tx_linearize;
 820                 tx_force_wb += p->tx_stats.tx_force_wb;
 821
 822                 /* Rx queue is part of the same block as Tx queue */
 823                 p = &p[1];
 824                 do {
 825                         start = u64_stats_fetch_begin_irq(&p->syncp);
 826                         packets = p->stats.packets;
 827                         bytes = p->stats.bytes;
 828                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 829                 rx_b += bytes;
 830                 rx_p += packets;
 831                 rx_buf += p->rx_stats.alloc_buff_failed;
 832                 rx_page += p->rx_stats.alloc_page_failed;
 833         }
 834         rcu_read_unlock();
 835         vsi->tx_restart = tx_restart;
 836         vsi->tx_busy = tx_busy;
 837         vsi->tx_linearize = tx_linearize;
 838         vsi->tx_force_wb = tx_force_wb;
 839         vsi->rx_page_failed = rx_page;
 840         vsi->rx_buf_failed = rx_buf;
 841
 842         ns->rx_packets = rx_p;
 843         ns->rx_bytes = rx_b;
 844         ns->tx_packets = tx_p;
 845         ns->tx_bytes = tx_b;
 846
 847         /* update netdev stats from eth stats */
 848         i40e_update_eth_stats(vsi);
 849         ons->tx_errors = oes->tx_errors;
 850         ns->tx_errors = es->tx_errors;
 851         ons->multicast = oes->rx_multicast;
 852         ns->multicast = es->rx_multicast;
 853         ons->rx_dropped = oes->rx_discards;
 854         ns->rx_dropped = es->rx_discards;
 855         ons->tx_dropped = oes->tx_discards;
 856         ns->tx_dropped = es->tx_discards;
 857
 858         /* pull in a couple PF stats if this is the main vsi */
 859         if (vsi == pf->vsi[pf->lan_vsi]) {
 860                 ns->rx_crc_errors = pf->stats.crc_errors;
 861                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 862                 ns->rx_length_errors = pf->stats.rx_length_errors;
 863         }
 864 }
 865
 866 /**
 867  * i40e_update_pf_stats - Update the PF statistics counters.
 868  * @pf: the PF to be updated
 869  **/
 870 static void i40e_update_pf_stats(struct i40e_pf *pf)
 871 {
 872         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 873         struct i40e_hw_port_stats *nsd = &pf->stats;
 874         struct i40e_hw *hw = &pf->hw;
 875         u32 val;
 876         int i;
 877
 878         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 879                            I40E_GLPRT_GORCL(hw->port),
 880                            pf->stat_offsets_loaded,
 881                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 882         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 883                            I40E_GLPRT_GOTCL(hw->port),
 884                            pf->stat_offsets_loaded,
 885                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 886         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 887                            pf->stat_offsets_loaded,
 888                            &osd->eth.rx_discards,
 889                            &nsd->eth.rx_discards);
 890         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 891                            I40E_GLPRT_UPRCL(hw->port),
 892                            pf->stat_offsets_loaded,
 893                            &osd->eth.rx_unicast,
 894                            &nsd->eth.rx_unicast);
 895         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 896                            I40E_GLPRT_MPRCL(hw->port),
 897                            pf->stat_offsets_loaded,
 898                            &osd->eth.rx_multicast,
 899                            &nsd->eth.rx_multicast);
 900         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 901                            I40E_GLPRT_BPRCL(hw->port),
 902                            pf->stat_offsets_loaded,
 903                            &osd->eth.rx_broadcast,
 904                            &nsd->eth.rx_broadcast);
 905         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 906                            I40E_GLPRT_UPTCL(hw->port),
 907                            pf->stat_offsets_loaded,
 908                            &osd->eth.tx_unicast,
 909                            &nsd->eth.tx_unicast);
 910         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 911                            I40E_GLPRT_MPTCL(hw->port),
 912                            pf->stat_offsets_loaded,
 913                            &osd->eth.tx_multicast,
 914                            &nsd->eth.tx_multicast);
 915         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 916                            I40E_GLPRT_BPTCL(hw->port),
 917                            pf->stat_offsets_loaded,
 918                            &osd->eth.tx_broadcast,
 919                            &nsd->eth.tx_broadcast);
 920
 921         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 922                            pf->stat_offsets_loaded,
 923                            &osd->tx_dropped_link_down,
 924                            &nsd->tx_dropped_link_down);
 925
 926         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 927                            pf->stat_offsets_loaded,
 928                            &osd->crc_errors, &nsd->crc_errors);
 929
 930         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 931                            pf->stat_offsets_loaded,
 932                            &osd->illegal_bytes, &nsd->illegal_bytes);
 933
 934         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 935                            pf->stat_offsets_loaded,
 936                            &osd->mac_local_faults,
 937                            &nsd->mac_local_faults);
 938         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 939                            pf->stat_offsets_loaded,
 940                            &osd->mac_remote_faults,
 941                            &nsd->mac_remote_faults);
 942
 943         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 944                            pf->stat_offsets_loaded,
 945                            &osd->rx_length_errors,
 946                            &nsd->rx_length_errors);
 947
 948         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 949                            pf->stat_offsets_loaded,
 950                            &osd->link_xon_rx, &nsd->link_xon_rx);
 951         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 952                            pf->stat_offsets_loaded,
 953                            &osd->link_xon_tx, &nsd->link_xon_tx);
 954         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 955                            pf->stat_offsets_loaded,
 956                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 957         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 958                            pf->stat_offsets_loaded,
 959                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 960
 961         for (i = 0; i < 8; i++) {
 962                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 963                                    pf->stat_offsets_loaded,
 964                                    &osd->priority_xoff_rx[i],
 965                                    &nsd->priority_xoff_rx[i]);
 966                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 967                                    pf->stat_offsets_loaded,
 968                                    &osd->priority_xon_rx[i],
 969                                    &nsd->priority_xon_rx[i]);
 970                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 971                                    pf->stat_offsets_loaded,
 972                                    &osd->priority_xon_tx[i],
 973                                    &nsd->priority_xon_tx[i]);
 974                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 975                                    pf->stat_offsets_loaded,
 976                                    &osd->priority_xoff_tx[i],
 977                                    &nsd->priority_xoff_tx[i]);
 978                 i40e_stat_update32(hw,
 979                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 980                                    pf->stat_offsets_loaded,
 981                                    &osd->priority_xon_2_xoff[i],
 982                                    &nsd->priority_xon_2_xoff[i]);
 983         }
 984
 985         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 986                            I40E_GLPRT_PRC64L(hw->port),
 987                            pf->stat_offsets_loaded,
 988                            &osd->rx_size_64, &nsd->rx_size_64);
 989         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 990                            I40E_GLPRT_PRC127L(hw->port),
 991                            pf->stat_offsets_loaded,
 992                            &osd->rx_size_127, &nsd->rx_size_127);
 993         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 994                            I40E_GLPRT_PRC255L(hw->port),
 995                            pf->stat_offsets_loaded,
 996                            &osd->rx_size_255, &nsd->rx_size_255);
 997         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
 998                            I40E_GLPRT_PRC511L(hw->port),
 999                            pf->stat_offsets_loaded,
1000                            &osd->rx_size_511, &nsd->rx_size_511);
1001         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1002                            I40E_GLPRT_PRC1023L(hw->port),
1003                            pf->stat_offsets_loaded,
1004                            &osd->rx_size_1023, &nsd->rx_size_1023);
1005         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1006                            I40E_GLPRT_PRC1522L(hw->port),
1007                            pf->stat_offsets_loaded,
1008                            &osd->rx_size_1522, &nsd->rx_size_1522);
1009         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1010                            I40E_GLPRT_PRC9522L(hw->port),
1011                            pf->stat_offsets_loaded,
1012                            &osd->rx_size_big, &nsd->rx_size_big);
1013
1014         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1015                            I40E_GLPRT_PTC64L(hw->port),
1016                            pf->stat_offsets_loaded,
1017                            &osd->tx_size_64, &nsd->tx_size_64);
1018         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1019                            I40E_GLPRT_PTC127L(hw->port),
1020                            pf->stat_offsets_loaded,
1021                            &osd->tx_size_127, &nsd->tx_size_127);
1022         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1023                            I40E_GLPRT_PTC255L(hw->port),
1024                            pf->stat_offsets_loaded,
1025                            &osd->tx_size_255, &nsd->tx_size_255);
1026         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1027                            I40E_GLPRT_PTC511L(hw->port),
1028                            pf->stat_offsets_loaded,
1029                            &osd->tx_size_511, &nsd->tx_size_511);
1030         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1031                            I40E_GLPRT_PTC1023L(hw->port),
1032                            pf->stat_offsets_loaded,
1033                            &osd->tx_size_1023, &nsd->tx_size_1023);
1034         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1035                            I40E_GLPRT_PTC1522L(hw->port),
1036                            pf->stat_offsets_loaded,
1037                            &osd->tx_size_1522, &nsd->tx_size_1522);
1038         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1039                            I40E_GLPRT_PTC9522L(hw->port),
1040                            pf->stat_offsets_loaded,
1041                            &osd->tx_size_big, &nsd->tx_size_big);
1042
1043         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1044                            pf->stat_offsets_loaded,
1045                            &osd->rx_undersize, &nsd->rx_undersize);
1046         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1047                            pf->stat_offsets_loaded,
1048                            &osd->rx_fragments, &nsd->rx_fragments);
1049         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1050                            pf->stat_offsets_loaded,
1051                            &osd->rx_oversize, &nsd->rx_oversize);
1052         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1053                            pf->stat_offsets_loaded,
1054                            &osd->rx_jabber, &nsd->rx_jabber);
1055
1056         /* FDIR stats */
1057         i40e_stat_update_and_clear32(hw,
1058                         I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1059                         &nsd->fd_atr_match);
1060         i40e_stat_update_and_clear32(hw,
1061                         I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1062                         &nsd->fd_sb_match);
1063         i40e_stat_update_and_clear32(hw,
1064                         I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1065                         &nsd->fd_atr_tunnel_match);
1066
1067         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1068         nsd->tx_lpi_status =
1069                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1070                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1071         nsd->rx_lpi_status =
1072                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1073                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1074         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1075                            pf->stat_offsets_loaded,
1076                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1077         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1078                            pf->stat_offsets_loaded,
1079                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1080
1081         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1082             !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED))
1083                 nsd->fd_sb_status = true;
1084         else
1085                 nsd->fd_sb_status = false;
1086
1087         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1088             !(pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED))
1089                 nsd->fd_atr_status = true;
1090         else
1091                 nsd->fd_atr_status = false;
1092
1093         pf->stat_offsets_loaded = true;
1094 }
1095
1096 /**
1097  * i40e_update_stats - Update the various statistics counters.
1098  * @vsi: the VSI to be updated
1099  *
1100  * Update the various stats for this VSI and its related entities.
1101  **/
1102 void i40e_update_stats(struct i40e_vsi *vsi)
1103 {
1104         struct i40e_pf *pf = vsi->back;
1105
1106         if (vsi == pf->vsi[pf->lan_vsi])
1107                 i40e_update_pf_stats(pf);
1108
1109         i40e_update_vsi_stats(vsi);
1110 }
1111
1112 /**
1113  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1114  * @vsi: the VSI to be searched
1115  * @macaddr: the MAC address
1116  * @vlan: the vlan
1117  *
1118  * Returns ptr to the filter object or NULL
1119  **/
1120 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1121                                                 const u8 *macaddr, s16 vlan)
1122 {
1123         struct i40e_mac_filter *f;
1124         u64 key;
1125
1126         if (!vsi || !macaddr)
1127                 return NULL;
1128
1129         key = i40e_addr_to_hkey(macaddr);
1130         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1131                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1132                     (vlan == f->vlan))
1133                         return f;
1134         }
1135         return NULL;
1136 }
1137
1138 /**
1139  * i40e_find_mac - Find a mac addr in the macvlan filters list
1140  * @vsi: the VSI to be searched
1141  * @macaddr: the MAC address we are searching for
1142  *
1143  * Returns the first filter with the provided MAC address or NULL if
1144  * MAC address was not found
1145  **/
1146 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1147 {
1148         struct i40e_mac_filter *f;
1149         u64 key;
1150
1151         if (!vsi || !macaddr)
1152                 return NULL;
1153
1154         key = i40e_addr_to_hkey(macaddr);
1155         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1156                 if ((ether_addr_equal(macaddr, f->macaddr)))
1157                         return f;
1158         }
1159         return NULL;
1160 }
1161
1162 /**
1163  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1164  * @vsi: the VSI to be searched
1165  *
1166  * Returns true if VSI is in vlan mode or false otherwise
1167  **/
1168 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1169 {
1170         /* If we have a PVID, always operate in VLAN mode */
1171         if (vsi->info.pvid)
1172                 return true;
1173
1174         /* We need to operate in VLAN mode whenever we have any filters with
1175          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1176          * time, incurring search cost repeatedly. However, we can notice two
1177          * things:
1178          *
1179          * 1) the only place where we can gain a VLAN filter is in
1180          *    i40e_add_filter.
1181          *
1182          * 2) the only place where filters are actually removed is in
1183          *    i40e_sync_filters_subtask.
1184          *
1185          * Thus, we can simply use a boolean value, has_vlan_filters which we
1186          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1187          * we have to perform the full search after deleting filters in
1188          * i40e_sync_filters_subtask, but we already have to search
1189          * filters here and can perform the check at the same time. This
1190          * results in avoiding embedding a loop for VLAN mode inside another
1191          * loop over all the filters, and should maintain correctness as noted
1192          * above.
1193          */
1194         return vsi->has_vlan_filter;
1195 }
1196
1197 /**
1198  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1199  * @vsi: the VSI to configure
1200  * @tmp_add_list: list of filters ready to be added
1201  * @tmp_del_list: list of filters ready to be deleted
1202  * @vlan_filters: the number of active VLAN filters
1203  *
1204  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1205  * behave as expected. If we have any active VLAN filters remaining or about
1206  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1207  * so that they only match against untagged traffic. If we no longer have any
1208  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1209  * so that they match against both tagged and untagged traffic. In this way,
1210  * we ensure that we correctly receive the desired traffic. This ensures that
1211  * when we have an active VLAN we will receive only untagged traffic and
1212  * traffic matching active VLANs. If we have no active VLANs then we will
1213  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1214  *
1215  * Finally, in a similar fashion, this function also corrects filters when
1216  * there is an active PVID assigned to this VSI.
1217  *
1218  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1219  *
1220  * This function is only expected to be called from within
1221  * i40e_sync_vsi_filters.
1222  *
1223  * NOTE: This function expects to be called while under the
1224  * mac_filter_hash_lock
1225  */
1226 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1227                                          struct hlist_head *tmp_add_list,
1228                                          struct hlist_head *tmp_del_list,
1229                                          int vlan_filters)
1230 {
1231         s16 pvid = le16_to_cpu(vsi->info.pvid);
1232         struct i40e_mac_filter *f, *add_head;
1233         struct i40e_new_mac_filter *new;
1234         struct hlist_node *h;
1235         int bkt, new_vlan;
1236
1237         /* To determine if a particular filter needs to be replaced we
1238          * have the three following conditions:
1239          *
1240          * a) if we have a PVID assigned, then all filters which are
1241          *    not marked as VLAN=PVID must be replaced with filters that
1242          *    are.
1243          * b) otherwise, if we have any active VLANS, all filters
1244          *    which are marked as VLAN=-1 must be replaced with
1245          *    filters marked as VLAN=0
1246          * c) finally, if we do not have any active VLANS, all filters
1247          *    which are marked as VLAN=0 must be replaced with filters
1248          *    marked as VLAN=-1
1249          */
1250
1251         /* Update the filters about to be added in place */
1252         hlist_for_each_entry(new, tmp_add_list, hlist) {
1253                 if (pvid && new->f->vlan != pvid)
1254                         new->f->vlan = pvid;
1255                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1256                         new->f->vlan = 0;
1257                 else if (!vlan_filters && new->f->vlan == 0)
1258                         new->f->vlan = I40E_VLAN_ANY;
1259         }
1260
1261         /* Update the remaining active filters */
1262         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1263                 /* Combine the checks for whether a filter needs to be changed
1264                  * and then determine the new VLAN inside the if block, in
1265                  * order to avoid duplicating code for adding the new filter
1266                  * then deleting the old filter.
1267                  */
1268                 if ((pvid && f->vlan != pvid) ||
1269                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1270                     (!vlan_filters && f->vlan == 0)) {
1271                         /* Determine the new vlan we will be adding */
1272                         if (pvid)
1273                                 new_vlan = pvid;
1274                         else if (vlan_filters)
1275                                 new_vlan = 0;
1276                         else
1277                                 new_vlan = I40E_VLAN_ANY;
1278
1279                         /* Create the new filter */
1280                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1281                         if (!add_head)
1282                                 return -ENOMEM;
1283
1284                         /* Create a temporary i40e_new_mac_filter */
1285                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1286                         if (!new)
1287                                 return -ENOMEM;
1288
1289                         new->f = add_head;
1290                         new->state = add_head->state;
1291
1292                         /* Add the new filter to the tmp list */
1293                         hlist_add_head(&new->hlist, tmp_add_list);
1294
1295                         /* Put the original filter into the delete list */
1296                         f->state = I40E_FILTER_REMOVE;
1297                         hash_del(&f->hlist);
1298                         hlist_add_head(&f->hlist, tmp_del_list);
1299                 }
1300         }
1301
1302         vsi->has_vlan_filter = !!vlan_filters;
1303
1304         return 0;
1305 }
1306
1307 /**
1308  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1309  * @vsi: the PF Main VSI - inappropriate for any other VSI
1310  * @macaddr: the MAC address
1311  *
1312  * Remove whatever filter the firmware set up so the driver can manage
1313  * its own filtering intelligently.
1314  **/
1315 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1316 {
1317         struct i40e_aqc_remove_macvlan_element_data element;
1318         struct i40e_pf *pf = vsi->back;
1319
1320         /* Only appropriate for the PF main VSI */
1321         if (vsi->type != I40E_VSI_MAIN)
1322                 return;
1323
1324         memset(&element, 0, sizeof(element));
1325         ether_addr_copy(element.mac_addr, macaddr);
1326         element.vlan_tag = 0;
1327         /* Ignore error returns, some firmware does it this way... */
1328         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1329         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1330
1331         memset(&element, 0, sizeof(element));
1332         ether_addr_copy(element.mac_addr, macaddr);
1333         element.vlan_tag = 0;
1334         /* ...and some firmware does it this way. */
1335         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1336                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1337         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1338 }
1339
1340 /**
1341  * i40e_add_filter - Add a mac/vlan filter to the VSI
1342  * @vsi: the VSI to be searched
1343  * @macaddr: the MAC address
1344  * @vlan: the vlan
1345  *
1346  * Returns ptr to the filter object or NULL when no memory available.
1347  *
1348  * NOTE: This function is expected to be called with mac_filter_hash_lock
1349  * being held.
1350  **/
1351 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1352                                         const u8 *macaddr, s16 vlan)
1353 {
1354         struct i40e_mac_filter *f;
1355         u64 key;
1356
1357         if (!vsi || !macaddr)
1358                 return NULL;
1359
1360         f = i40e_find_filter(vsi, macaddr, vlan);
1361         if (!f) {
1362                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1363                 if (!f)
1364                         return NULL;
1365
1366                 /* Update the boolean indicating if we need to function in
1367                  * VLAN mode.
1368                  */
1369                 if (vlan >= 0)
1370                         vsi->has_vlan_filter = true;
1371
1372                 ether_addr_copy(f->macaddr, macaddr);
1373                 f->vlan = vlan;
1374                 /* If we're in overflow promisc mode, set the state directly
1375                  * to failed, so we don't bother to try sending the filter
1376                  * to the hardware.
1377                  */
1378                 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state))
1379                         f->state = I40E_FILTER_FAILED;
1380                 else
1381                         f->state = I40E_FILTER_NEW;
1382                 INIT_HLIST_NODE(&f->hlist);
1383
1384                 key = i40e_addr_to_hkey(macaddr);
1385                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1386
1387                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1388                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1389         }
1390
1391         /* If we're asked to add a filter that has been marked for removal, it
1392          * is safe to simply restore it to active state. __i40e_del_filter
1393          * will have simply deleted any filters which were previously marked
1394          * NEW or FAILED, so if it is currently marked REMOVE it must have
1395          * previously been ACTIVE. Since we haven't yet run the sync filters
1396          * task, just restore this filter to the ACTIVE state so that the
1397          * sync task leaves it in place
1398          */
1399         if (f->state == I40E_FILTER_REMOVE)
1400                 f->state = I40E_FILTER_ACTIVE;
1401
1402         return f;
1403 }
1404
1405 /**
1406  * __i40e_del_filter - Remove a specific filter from the VSI
1407  * @vsi: VSI to remove from
1408  * @f: the filter to remove from the list
1409  *
1410  * This function should be called instead of i40e_del_filter only if you know
1411  * the exact filter you will remove already, such as via i40e_find_filter or
1412  * i40e_find_mac.
1413  *
1414  * NOTE: This function is expected to be called with mac_filter_hash_lock
1415  * being held.
1416  * ANOTHER NOTE: This function MUST be called from within the context of
1417  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1418  * instead of list_for_each_entry().
1419  **/
1420 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1421 {
1422         if (!f)
1423                 return;
1424
1425         /* If the filter was never added to firmware then we can just delete it
1426          * directly and we don't want to set the status to remove or else an
1427          * admin queue command will unnecessarily fire.
1428          */
1429         if ((f->state == I40E_FILTER_FAILED) ||
1430             (f->state == I40E_FILTER_NEW)) {
1431                 hash_del(&f->hlist);
1432                 kfree(f);
1433         } else {
1434                 f->state = I40E_FILTER_REMOVE;
1435         }
1436
1437         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1438         vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1439 }
1440
1441 /**
1442  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1443  * @vsi: the VSI to be searched
1444  * @macaddr: the MAC address
1445  * @vlan: the VLAN
1446  *
1447  * NOTE: This function is expected to be called with mac_filter_hash_lock
1448  * being held.
1449  * ANOTHER NOTE: This function MUST be called from within the context of
1450  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1451  * instead of list_for_each_entry().
1452  **/
1453 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1454 {
1455         struct i40e_mac_filter *f;
1456
1457         if (!vsi || !macaddr)
1458                 return;
1459
1460         f = i40e_find_filter(vsi, macaddr, vlan);
1461         __i40e_del_filter(vsi, f);
1462 }
1463
1464 /**
1465  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1466  * @vsi: the VSI to be searched
1467  * @macaddr: the mac address to be filtered
1468  *
1469  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1470  * go through all the macvlan filters and add a macvlan filter for each
1471  * unique vlan that already exists. If a PVID has been assigned, instead only
1472  * add the macaddr to that VLAN.
1473  *
1474  * Returns last filter added on success, else NULL
1475  **/
1476 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1477                                             const u8 *macaddr)
1478 {
1479         struct i40e_mac_filter *f, *add = NULL;
1480         struct hlist_node *h;
1481         int bkt;
1482
1483         if (vsi->info.pvid)
1484                 return i40e_add_filter(vsi, macaddr,
1485                                        le16_to_cpu(vsi->info.pvid));
1486
1487         if (!i40e_is_vsi_in_vlan(vsi))
1488                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1489
1490         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1491                 if (f->state == I40E_FILTER_REMOVE)
1492                         continue;
1493                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1494                 if (!add)
1495                         return NULL;
1496         }
1497
1498         return add;
1499 }
1500
1501 /**
1502  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1503  * @vsi: the VSI to be searched
1504  * @macaddr: the mac address to be removed
1505  *
1506  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1507  * associated with.
1508  *
1509  * Returns 0 for success, or error
1510  **/
1511 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1512 {
1513         struct i40e_mac_filter *f;
1514         struct hlist_node *h;
1515         bool found = false;
1516         int bkt;
1517
1518         WARN(!spin_is_locked(&vsi->mac_filter_hash_lock),
1519              "Missing mac_filter_hash_lock\n");
1520         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1521                 if (ether_addr_equal(macaddr, f->macaddr)) {
1522                         __i40e_del_filter(vsi, f);
1523                         found = true;
1524                 }
1525         }
1526
1527         if (found)
1528                 return 0;
1529         else
1530                 return -ENOENT;
1531 }
1532
1533 /**
1534  * i40e_set_mac - NDO callback to set mac address
1535  * @netdev: network interface device structure
1536  * @p: pointer to an address structure
1537  *
1538  * Returns 0 on success, negative on failure
1539  **/
1540 static int i40e_set_mac(struct net_device *netdev, void *p)
1541 {
1542         struct i40e_netdev_priv *np = netdev_priv(netdev);
1543         struct i40e_vsi *vsi = np->vsi;
1544         struct i40e_pf *pf = vsi->back;
1545         struct i40e_hw *hw = &pf->hw;
1546         struct sockaddr *addr = p;
1547
1548         if (!is_valid_ether_addr(addr->sa_data))
1549                 return -EADDRNOTAVAIL;
1550
1551         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1552                 netdev_info(netdev, "already using mac address %pM\n",
1553                             addr->sa_data);
1554                 return 0;
1555         }
1556
1557         if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
1558             test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
1559                 return -EADDRNOTAVAIL;
1560
1561         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1562                 netdev_info(netdev, "returning to hw mac address %pM\n",
1563                             hw->mac.addr);
1564         else
1565                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1566
1567         spin_lock_bh(&vsi->mac_filter_hash_lock);
1568         i40e_del_mac_filter(vsi, netdev->dev_addr);
1569         i40e_add_mac_filter(vsi, addr->sa_data);
1570         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1571         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1572         if (vsi->type == I40E_VSI_MAIN) {
1573                 i40e_status ret;
1574
1575                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1576                                                 I40E_AQC_WRITE_TYPE_LAA_WOL,
1577                                                 addr->sa_data, NULL);
1578                 if (ret)
1579                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1580                                     i40e_stat_str(hw, ret),
1581                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1582         }
1583
1584         /* schedule our worker thread which will take care of
1585          * applying the new filter changes
1586          */
1587         i40e_service_event_schedule(vsi->back);
1588         return 0;
1589 }
1590
1591 /**
1592  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1593  * @vsi: the VSI being setup
1594  * @ctxt: VSI context structure
1595  * @enabled_tc: Enabled TCs bitmap
1596  * @is_add: True if called before Add VSI
1597  *
1598  * Setup VSI queue mapping for enabled traffic classes.
1599  **/
1600 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1601                                      struct i40e_vsi_context *ctxt,
1602                                      u8 enabled_tc,
1603                                      bool is_add)
1604 {
1605         struct i40e_pf *pf = vsi->back;
1606         u16 sections = 0;
1607         u8 netdev_tc = 0;
1608         u16 numtc = 0;
1609         u16 qcount;
1610         u8 offset;
1611         u16 qmap;
1612         int i;
1613         u16 num_tc_qps = 0;
1614
1615         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1616         offset = 0;
1617
1618         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1619                 /* Find numtc from enabled TC bitmap */
1620                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1621                         if (enabled_tc & BIT(i)) /* TC is enabled */
1622                                 numtc++;
1623                 }
1624                 if (!numtc) {
1625                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1626                         numtc = 1;
1627                 }
1628         } else {
1629                 /* At least TC0 is enabled in case of non-DCB case */
1630                 numtc = 1;
1631         }
1632
1633         vsi->tc_config.numtc = numtc;
1634         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1635         /* Number of queues per enabled TC */
1636         qcount = vsi->alloc_queue_pairs;
1637
1638         num_tc_qps = qcount / numtc;
1639         num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf));
1640
1641         /* Setup queue offset/count for all TCs for given VSI */
1642         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1643                 /* See if the given TC is enabled for the given VSI */
1644                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1645                         /* TC is enabled */
1646                         int pow, num_qps;
1647
1648                         switch (vsi->type) {
1649                         case I40E_VSI_MAIN:
1650                                 qcount = min_t(int, pf->alloc_rss_size,
1651                                                num_tc_qps);
1652                                 break;
1653                         case I40E_VSI_FDIR:
1654                         case I40E_VSI_SRIOV:
1655                         case I40E_VSI_VMDQ2:
1656                         default:
1657                                 qcount = num_tc_qps;
1658                                 WARN_ON(i != 0);
1659                                 break;
1660                         }
1661                         vsi->tc_config.tc_info[i].qoffset = offset;
1662                         vsi->tc_config.tc_info[i].qcount = qcount;
1663
1664                         /* find the next higher power-of-2 of num queue pairs */
1665                         num_qps = qcount;
1666                         pow = 0;
1667                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1668                                 pow++;
1669                                 num_qps >>= 1;
1670                         }
1671
1672                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1673                         qmap =
1674                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1675                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1676
1677                         offset += qcount;
1678                 } else {
1679                         /* TC is not enabled so set the offset to
1680                          * default queue and allocate one queue
1681                          * for the given TC.
1682                          */
1683                         vsi->tc_config.tc_info[i].qoffset = 0;
1684                         vsi->tc_config.tc_info[i].qcount = 1;
1685                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1686
1687                         qmap = 0;
1688                 }
1689                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1690         }
1691
1692         /* Set actual Tx/Rx queue pairs */
1693         vsi->num_queue_pairs = offset;
1694         if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1695                 if (vsi->req_queue_pairs > 0)
1696                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1697                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1698                         vsi->num_queue_pairs = pf->num_lan_msix;
1699         }
1700
1701         /* Scheduler section valid can only be set for ADD VSI */
1702         if (is_add) {
1703                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1704
1705                 ctxt->info.up_enable_bits = enabled_tc;
1706         }
1707         if (vsi->type == I40E_VSI_SRIOV) {
1708                 ctxt->info.mapping_flags |=
1709                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1710                 for (i = 0; i < vsi->num_queue_pairs; i++)
1711                         ctxt->info.queue_mapping[i] =
1712                                                cpu_to_le16(vsi->base_queue + i);
1713         } else {
1714                 ctxt->info.mapping_flags |=
1715                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1716                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1717         }
1718         ctxt->info.valid_sections |= cpu_to_le16(sections);
1719 }
1720
1721 /**
1722  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1723  * @netdev: the netdevice
1724  * @addr: address to add
1725  *
1726  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1727  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1728  */
1729 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1730 {
1731         struct i40e_netdev_priv *np = netdev_priv(netdev);
1732         struct i40e_vsi *vsi = np->vsi;
1733
1734         if (i40e_add_mac_filter(vsi, addr))
1735                 return 0;
1736         else
1737                 return -ENOMEM;
1738 }
1739
1740 /**
1741  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1742  * @netdev: the netdevice
1743  * @addr: address to add
1744  *
1745  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1746  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1747  */
1748 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1749 {
1750         struct i40e_netdev_priv *np = netdev_priv(netdev);
1751         struct i40e_vsi *vsi = np->vsi;
1752
1753         i40e_del_mac_filter(vsi, addr);
1754
1755         return 0;
1756 }
1757
1758 /**
1759  * i40e_set_rx_mode - NDO callback to set the netdev filters
1760  * @netdev: network interface device structure
1761  **/
1762 static void i40e_set_rx_mode(struct net_device *netdev)
1763 {
1764         struct i40e_netdev_priv *np = netdev_priv(netdev);
1765         struct i40e_vsi *vsi = np->vsi;
1766
1767         spin_lock_bh(&vsi->mac_filter_hash_lock);
1768
1769         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1770         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1771
1772         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1773
1774         /* check for other flag changes */
1775         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1776                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1777                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1778         }
1779 }
1780
1781 /**
1782  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
1783  * @vsi: Pointer to VSI struct
1784  * @from: Pointer to list which contains MAC filter entries - changes to
1785  *        those entries needs to be undone.
1786  *
1787  * MAC filter entries from this list were slated for deletion.
1788  **/
1789 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
1790                                          struct hlist_head *from)
1791 {
1792         struct i40e_mac_filter *f;
1793         struct hlist_node *h;
1794
1795         hlist_for_each_entry_safe(f, h, from, hlist) {
1796                 u64 key = i40e_addr_to_hkey(f->macaddr);
1797
1798                 /* Move the element back into MAC filter list*/
1799                 hlist_del(&f->hlist);
1800                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1801         }
1802 }
1803
1804 /**
1805  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
1806  * @vsi: Pointer to vsi struct
1807  * @from: Pointer to list which contains MAC filter entries - changes to
1808  *        those entries needs to be undone.
1809  *
1810  * MAC filter entries from this list were slated for addition.
1811  **/
1812 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
1813                                          struct hlist_head *from)
1814 {
1815         struct i40e_new_mac_filter *new;
1816         struct hlist_node *h;
1817
1818         hlist_for_each_entry_safe(new, h, from, hlist) {
1819                 /* We can simply free the wrapper structure */
1820                 hlist_del(&new->hlist);
1821                 kfree(new);
1822         }
1823 }
1824
1825 /**
1826  * i40e_next_entry - Get the next non-broadcast filter from a list
1827  * @next: pointer to filter in list
1828  *
1829  * Returns the next non-broadcast filter in the list. Required so that we
1830  * ignore broadcast filters within the list, since these are not handled via
1831  * the normal firmware update path.
1832  */
1833 static
1834 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
1835 {
1836         hlist_for_each_entry_continue(next, hlist) {
1837                 if (!is_broadcast_ether_addr(next->f->macaddr))
1838                         return next;
1839         }
1840
1841         return NULL;
1842 }
1843
1844 /**
1845  * i40e_update_filter_state - Update filter state based on return data
1846  * from firmware
1847  * @count: Number of filters added
1848  * @add_list: return data from fw
1849  * @head: pointer to first filter in current batch
1850  *
1851  * MAC filter entries from list were slated to be added to device. Returns
1852  * number of successful filters. Note that 0 does NOT mean success!
1853  **/
1854 static int
1855 i40e_update_filter_state(int count,
1856                          struct i40e_aqc_add_macvlan_element_data *add_list,
1857                          struct i40e_new_mac_filter *add_head)
1858 {
1859         int retval = 0;
1860         int i;
1861
1862         for (i = 0; i < count; i++) {
1863                 /* Always check status of each filter. We don't need to check
1864                  * the firmware return status because we pre-set the filter
1865                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
1866                  * request to the adminq. Thus, if it no longer matches then
1867                  * we know the filter is active.
1868                  */
1869                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
1870                         add_head->state = I40E_FILTER_FAILED;
1871                 } else {
1872                         add_head->state = I40E_FILTER_ACTIVE;
1873                         retval++;
1874                 }
1875
1876                 add_head = i40e_next_filter(add_head);
1877                 if (!add_head)
1878                         break;
1879         }
1880
1881         return retval;
1882 }
1883
1884 /**
1885  * i40e_aqc_del_filters - Request firmware to delete a set of filters
1886  * @vsi: ptr to the VSI
1887  * @vsi_name: name to display in messages
1888  * @list: the list of filters to send to firmware
1889  * @num_del: the number of filters to delete
1890  * @retval: Set to -EIO on failure to delete
1891  *
1892  * Send a request to firmware via AdminQ to delete a set of filters. Uses
1893  * *retval instead of a return value so that success does not force ret_val to
1894  * be set to 0. This ensures that a sequence of calls to this function
1895  * preserve the previous value of *retval on successful delete.
1896  */
1897 static
1898 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
1899                           struct i40e_aqc_remove_macvlan_element_data *list,
1900                           int num_del, int *retval)
1901 {
1902         struct i40e_hw *hw = &vsi->back->hw;
1903         i40e_status aq_ret;
1904         int aq_err;
1905
1906         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
1907         aq_err = hw->aq.asq_last_status;
1908
1909         /* Explicitly ignore and do not report when firmware returns ENOENT */
1910         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
1911                 *retval = -EIO;
1912                 dev_info(&vsi->back->pdev->dev,
1913                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
1914                          vsi_name, i40e_stat_str(hw, aq_ret),
1915                          i40e_aq_str(hw, aq_err));
1916         }
1917 }
1918
1919 /**
1920  * i40e_aqc_add_filters - Request firmware to add a set of filters
1921  * @vsi: ptr to the VSI
1922  * @vsi_name: name to display in messages
1923  * @list: the list of filters to send to firmware
1924  * @add_head: Position in the add hlist
1925  * @num_add: the number of filters to add
1926  * @promisc_change: set to true on exit if promiscuous mode was forced on
1927  *
1928  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
1929  * promisc_changed to true if the firmware has run out of space for more
1930  * filters.
1931  */
1932 static
1933 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
1934                           struct i40e_aqc_add_macvlan_element_data *list,
1935                           struct i40e_new_mac_filter *add_head,
1936                           int num_add, bool *promisc_changed)
1937 {
1938         struct i40e_hw *hw = &vsi->back->hw;
1939         int aq_err, fcnt;
1940
1941         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
1942         aq_err = hw->aq.asq_last_status;
1943         fcnt = i40e_update_filter_state(num_add, list, add_head);
1944
1945         if (fcnt != num_add) {
1946                 *promisc_changed = true;
1947                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
1948                 dev_warn(&vsi->back->pdev->dev,
1949                          "Error %s adding RX filters on %s, promiscuous mode forced on\n",
1950                          i40e_aq_str(hw, aq_err),
1951                          vsi_name);
1952         }
1953 }
1954
1955 /**
1956  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
1957  * @vsi: pointer to the VSI
1958  * @f: filter data
1959  *
1960  * This function sets or clears the promiscuous broadcast flags for VLAN
1961  * filters in order to properly receive broadcast frames. Assumes that only
1962  * broadcast filters are passed.
1963  *
1964  * Returns status indicating success or failure;
1965  **/
1966 static i40e_status
1967 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
1968                           struct i40e_mac_filter *f)
1969 {
1970         bool enable = f->state == I40E_FILTER_NEW;
1971         struct i40e_hw *hw = &vsi->back->hw;
1972         i40e_status aq_ret;
1973
1974         if (f->vlan == I40E_VLAN_ANY) {
1975                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
1976                                                    vsi->seid,
1977                                                    enable,
1978                                                    NULL);
1979         } else {
1980                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
1981                                                             vsi->seid,
1982                                                             enable,
1983                                                             f->vlan,
1984                                                             NULL);
1985         }
1986
1987         if (aq_ret)
1988                 dev_warn(&vsi->back->pdev->dev,
1989                          "Error %s setting broadcast promiscuous mode on %s\n",
1990                          i40e_aq_str(hw, hw->aq.asq_last_status),
1991                          vsi_name);
1992
1993         return aq_ret;
1994 }
1995
1996 /**
1997  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
1998  * @vsi: ptr to the VSI
1999  *
2000  * Push any outstanding VSI filter changes through the AdminQ.
2001  *
2002  * Returns 0 or error value
2003  **/
2004 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2005 {
2006         struct hlist_head tmp_add_list, tmp_del_list;
2007         struct i40e_mac_filter *f;
2008         struct i40e_new_mac_filter *new, *add_head = NULL;
2009         struct i40e_hw *hw = &vsi->back->hw;
2010         unsigned int failed_filters = 0;
2011         unsigned int vlan_filters = 0;
2012         bool promisc_changed = false;
2013         char vsi_name[16] = "PF";
2014         int filter_list_len = 0;
2015         i40e_status aq_ret = 0;
2016         u32 changed_flags = 0;
2017         struct hlist_node *h;
2018         struct i40e_pf *pf;
2019         int num_add = 0;
2020         int num_del = 0;
2021         int retval = 0;
2022         u16 cmd_flags;
2023         int list_size;
2024         int bkt;
2025
2026         /* empty array typed pointers, kcalloc later */
2027         struct i40e_aqc_add_macvlan_element_data *add_list;
2028         struct i40e_aqc_remove_macvlan_element_data *del_list;
2029
2030         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2031                 usleep_range(1000, 2000);
2032         pf = vsi->back;
2033
2034         if (vsi->netdev) {
2035                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2036                 vsi->current_netdev_flags = vsi->netdev->flags;
2037         }
2038
2039         INIT_HLIST_HEAD(&tmp_add_list);
2040         INIT_HLIST_HEAD(&tmp_del_list);
2041
2042         if (vsi->type == I40E_VSI_SRIOV)
2043                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2044         else if (vsi->type != I40E_VSI_MAIN)
2045                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2046
2047         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2048                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2049
2050                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2051                 /* Create a list of filters to delete. */
2052                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2053                         if (f->state == I40E_FILTER_REMOVE) {
2054                                 /* Move the element into temporary del_list */
2055                                 hash_del(&f->hlist);
2056                                 hlist_add_head(&f->hlist, &tmp_del_list);
2057
2058                                 /* Avoid counting removed filters */
2059                                 continue;
2060                         }
2061                         if (f->state == I40E_FILTER_NEW) {
2062                                 /* Create a temporary i40e_new_mac_filter */
2063                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2064                                 if (!new)
2065                                         goto err_no_memory_locked;
2066
2067                                 /* Store pointer to the real filter */
2068                                 new->f = f;
2069                                 new->state = f->state;
2070
2071                                 /* Add it to the hash list */
2072                                 hlist_add_head(&new->hlist, &tmp_add_list);
2073                         }
2074
2075                         /* Count the number of active (current and new) VLAN
2076                          * filters we have now. Does not count filters which
2077                          * are marked for deletion.
2078                          */
2079                         if (f->vlan > 0)
2080                                 vlan_filters++;
2081                 }
2082
2083                 retval = i40e_correct_mac_vlan_filters(vsi,
2084                                                        &tmp_add_list,
2085                                                        &tmp_del_list,
2086                                                        vlan_filters);
2087                 if (retval)
2088                         goto err_no_memory_locked;
2089
2090                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2091         }
2092
2093         /* Now process 'del_list' outside the lock */
2094         if (!hlist_empty(&tmp_del_list)) {
2095                 filter_list_len = hw->aq.asq_buf_size /
2096                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2097                 list_size = filter_list_len *
2098                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2099                 del_list = kzalloc(list_size, GFP_ATOMIC);
2100                 if (!del_list)
2101                         goto err_no_memory;
2102
2103                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2104                         cmd_flags = 0;
2105
2106                         /* handle broadcast filters by updating the broadcast
2107                          * promiscuous flag and release filter list.
2108                          */
2109                         if (is_broadcast_ether_addr(f->macaddr)) {
2110                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2111
2112                                 hlist_del(&f->hlist);
2113                                 kfree(f);
2114                                 continue;
2115                         }
2116
2117                         /* add to delete list */
2118                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2119                         if (f->vlan == I40E_VLAN_ANY) {
2120                                 del_list[num_del].vlan_tag = 0;
2121                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2122                         } else {
2123                                 del_list[num_del].vlan_tag =
2124                                         cpu_to_le16((u16)(f->vlan));
2125                         }
2126
2127                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2128                         del_list[num_del].flags = cmd_flags;
2129                         num_del++;
2130
2131                         /* flush a full buffer */
2132                         if (num_del == filter_list_len) {
2133                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2134                                                      num_del, &retval);
2135                                 memset(del_list, 0, list_size);
2136                                 num_del = 0;
2137                         }
2138                         /* Release memory for MAC filter entries which were
2139                          * synced up with HW.
2140                          */
2141                         hlist_del(&f->hlist);
2142                         kfree(f);
2143                 }
2144
2145                 if (num_del) {
2146                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2147                                              num_del, &retval);
2148                 }
2149
2150                 kfree(del_list);
2151                 del_list = NULL;
2152         }
2153
2154         if (!hlist_empty(&tmp_add_list)) {
2155                 /* Do all the adds now. */
2156                 filter_list_len = hw->aq.asq_buf_size /
2157                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2158                 list_size = filter_list_len *
2159                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2160                 add_list = kzalloc(list_size, GFP_ATOMIC);
2161                 if (!add_list)
2162                         goto err_no_memory;
2163
2164                 num_add = 0;
2165                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2166                         if (test_bit(__I40E_VSI_OVERFLOW_PROMISC,
2167                                      vsi->state)) {
2168                                 new->state = I40E_FILTER_FAILED;
2169                                 continue;
2170                         }
2171
2172                         /* handle broadcast filters by updating the broadcast
2173                          * promiscuous flag instead of adding a MAC filter.
2174                          */
2175                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2176                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2177                                                               new->f))
2178                                         new->state = I40E_FILTER_FAILED;
2179                                 else
2180                                         new->state = I40E_FILTER_ACTIVE;
2181                                 continue;
2182                         }
2183
2184                         /* add to add array */
2185                         if (num_add == 0)
2186                                 add_head = new;
2187                         cmd_flags = 0;
2188                         ether_addr_copy(add_list[num_add].mac_addr,
2189                                         new->f->macaddr);
2190                         if (new->f->vlan == I40E_VLAN_ANY) {
2191                                 add_list[num_add].vlan_tag = 0;
2192                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2193                         } else {
2194                                 add_list[num_add].vlan_tag =
2195                                         cpu_to_le16((u16)(new->f->vlan));
2196                         }
2197                         add_list[num_add].queue_number = 0;
2198                         /* set invalid match method for later detection */
2199                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2200                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2201                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2202                         num_add++;
2203
2204                         /* flush a full buffer */
2205                         if (num_add == filter_list_len) {
2206                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2207                                                      add_head, num_add,
2208                                                      &promisc_changed);
2209                                 memset(add_list, 0, list_size);
2210                                 num_add = 0;
2211                         }
2212                 }
2213                 if (num_add) {
2214                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2215                                              num_add, &promisc_changed);
2216                 }
2217                 /* Now move all of the filters from the temp add list back to
2218                  * the VSI's list.
2219                  */
2220                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2221                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2222                         /* Only update the state if we're still NEW */
2223                         if (new->f->state == I40E_FILTER_NEW)
2224                                 new->f->state = new->state;
2225                         hlist_del(&new->hlist);
2226                         kfree(new);
2227                 }
2228                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2229                 kfree(add_list);
2230                 add_list = NULL;
2231         }
2232
2233         /* Determine the number of active and failed filters. */
2234         spin_lock_bh(&vsi->mac_filter_hash_lock);
2235         vsi->active_filters = 0;
2236         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2237                 if (f->state == I40E_FILTER_ACTIVE)
2238                         vsi->active_filters++;
2239                 else if (f->state == I40E_FILTER_FAILED)
2240                         failed_filters++;
2241         }
2242         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2243
2244         /* If promiscuous mode has changed, we need to calculate a new
2245          * threshold for when we are safe to exit
2246          */
2247         if (promisc_changed)
2248                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2249
2250         /* Check if we are able to exit overflow promiscuous mode. We can
2251          * safely exit if we didn't just enter, we no longer have any failed
2252          * filters, and we have reduced filters below the threshold value.
2253          */
2254         if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state) &&
2255             !promisc_changed && !failed_filters &&
2256             (vsi->active_filters < vsi->promisc_threshold)) {
2257                 dev_info(&pf->pdev->dev,
2258                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2259                          vsi_name);
2260                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2261                 promisc_changed = true;
2262                 vsi->promisc_threshold = 0;
2263         }
2264
2265         /* if the VF is not trusted do not do promisc */
2266         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2267                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2268                 goto out;
2269         }
2270
2271         /* check for changes in promiscuous modes */
2272         if (changed_flags & IFF_ALLMULTI) {
2273                 bool cur_multipromisc;
2274
2275                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2276                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2277                                                                vsi->seid,
2278                                                                cur_multipromisc,
2279                                                                NULL);
2280                 if (aq_ret) {
2281                         retval = i40e_aq_rc_to_posix(aq_ret,
2282                                                      hw->aq.asq_last_status);
2283                         dev_info(&pf->pdev->dev,
2284                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2285                                  vsi_name,
2286                                  i40e_stat_str(hw, aq_ret),
2287                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2288                 }
2289         }
2290
2291         if ((changed_flags & IFF_PROMISC) || promisc_changed) {
2292                 bool cur_promisc;
2293
2294                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2295                                test_bit(__I40E_VSI_OVERFLOW_PROMISC,
2296                                         vsi->state));
2297                 if ((vsi->type == I40E_VSI_MAIN) &&
2298                     (pf->lan_veb != I40E_NO_VEB) &&
2299                     !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2300                         /* set defport ON for Main VSI instead of true promisc
2301                          * this way we will get all unicast/multicast and VLAN
2302                          * promisc behavior but will not get VF or VMDq traffic
2303                          * replicated on the Main VSI.
2304                          */
2305                         if (pf->cur_promisc != cur_promisc) {
2306                                 pf->cur_promisc = cur_promisc;
2307                                 if (cur_promisc)
2308                                         aq_ret =
2309                                               i40e_aq_set_default_vsi(hw,
2310                                                                       vsi->seid,
2311                                                                       NULL);
2312                                 else
2313                                         aq_ret =
2314                                             i40e_aq_clear_default_vsi(hw,
2315                                                                       vsi->seid,
2316                                                                       NULL);
2317                                 if (aq_ret) {
2318                                         retval = i40e_aq_rc_to_posix(aq_ret,
2319                                                         hw->aq.asq_last_status);
2320                                         dev_info(&pf->pdev->dev,
2321                                                  "Set default VSI failed on %s, err %s, aq_err %s\n",
2322                                                  vsi_name,
2323                                                  i40e_stat_str(hw, aq_ret),
2324                                                  i40e_aq_str(hw,
2325                                                      hw->aq.asq_last_status));
2326                                 }
2327                         }
2328                 } else {
2329                         aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2330                                                           hw,
2331                                                           vsi->seid,
2332                                                           cur_promisc, NULL,
2333                                                           true);
2334                         if (aq_ret) {
2335                                 retval =
2336                                 i40e_aq_rc_to_posix(aq_ret,
2337                                                     hw->aq.asq_last_status);
2338                                 dev_info(&pf->pdev->dev,
2339                                          "set unicast promisc failed on %s, err %s, aq_err %s\n",
2340                                          vsi_name,
2341                                          i40e_stat_str(hw, aq_ret),
2342                                          i40e_aq_str(hw,
2343                                                      hw->aq.asq_last_status));
2344                         }
2345                         aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2346                                                           hw,
2347                                                           vsi->seid,
2348                                                           cur_promisc, NULL);
2349                         if (aq_ret) {
2350                                 retval =
2351                                 i40e_aq_rc_to_posix(aq_ret,
2352                                                     hw->aq.asq_last_status);
2353                                 dev_info(&pf->pdev->dev,
2354                                          "set multicast promisc failed on %s, err %s, aq_err %s\n",
2355                                          vsi_name,
2356                                          i40e_stat_str(hw, aq_ret),
2357                                          i40e_aq_str(hw,
2358                                                      hw->aq.asq_last_status));
2359                         }
2360                 }
2361                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
2362                                                    vsi->seid,
2363                                                    cur_promisc, NULL);
2364                 if (aq_ret) {
2365                         retval = i40e_aq_rc_to_posix(aq_ret,
2366                                                      pf->hw.aq.asq_last_status);
2367                         dev_info(&pf->pdev->dev,
2368                                  "set brdcast promisc failed, err %s, aq_err %s\n",
2369                                          i40e_stat_str(hw, aq_ret),
2370                                          i40e_aq_str(hw,
2371                                                      hw->aq.asq_last_status));
2372                 }
2373         }
2374 out:
2375         /* if something went wrong then set the changed flag so we try again */
2376         if (retval)
2377                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2378
2379         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2380         return retval;
2381
2382 err_no_memory:
2383         /* Restore elements on the temporary add and delete lists */
2384         spin_lock_bh(&vsi->mac_filter_hash_lock);
2385 err_no_memory_locked:
2386         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2387         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2388         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2389
2390         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2391         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2392         return -ENOMEM;
2393 }
2394
2395 /**
2396  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2397  * @pf: board private structure
2398  **/
2399 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2400 {
2401         int v;
2402
2403         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
2404                 return;
2405         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
2406
2407         for (v = 0; v < pf->num_alloc_vsi; v++) {
2408                 if (pf->vsi[v] &&
2409                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2410                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2411
2412                         if (ret) {
2413                                 /* come back and try again later */
2414                                 pf->flags |= I40E_FLAG_FILTER_SYNC;
2415                                 break;
2416                         }
2417                 }
2418         }
2419 }
2420
2421 /**
2422  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2423  * @vsi: the vsi
2424  **/
2425 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2426 {
2427         if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2428                 return I40E_RXBUFFER_2048;
2429         else
2430                 return I40E_RXBUFFER_3072;
2431 }
2432
2433 /**
2434  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2435  * @netdev: network interface device structure
2436  * @new_mtu: new value for maximum frame size
2437  *
2438  * Returns 0 on success, negative on failure
2439  **/
2440 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2441 {
2442         struct i40e_netdev_priv *np = netdev_priv(netdev);
2443         struct i40e_vsi *vsi = np->vsi;
2444         struct i40e_pf *pf = vsi->back;
2445
2446         if (i40e_enabled_xdp_vsi(vsi)) {
2447                 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2448
2449                 if (frame_size > i40e_max_xdp_frame_size(vsi))
2450                         return -EINVAL;
2451         }
2452
2453         netdev_info(netdev, "changing MTU from %d to %d\n",
2454                     netdev->mtu, new_mtu);
2455         netdev->mtu = new_mtu;
2456         if (netif_running(netdev))
2457                 i40e_vsi_reinit_locked(vsi);
2458         pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
2459                       I40E_FLAG_CLIENT_L2_CHANGE);
2460         return 0;
2461 }
2462
2463 /**
2464  * i40e_ioctl - Access the hwtstamp interface
2465  * @netdev: network interface device structure
2466  * @ifr: interface request data
2467  * @cmd: ioctl command
2468  **/
2469 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2470 {
2471         struct i40e_netdev_priv *np = netdev_priv(netdev);
2472         struct i40e_pf *pf = np->vsi->back;
2473
2474         switch (cmd) {
2475         case SIOCGHWTSTAMP:
2476                 return i40e_ptp_get_ts_config(pf, ifr);
2477         case SIOCSHWTSTAMP:
2478                 return i40e_ptp_set_ts_config(pf, ifr);
2479         default:
2480                 return -EOPNOTSUPP;
2481         }
2482 }
2483
2484 /**
2485  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2486  * @vsi: the vsi being adjusted
2487  **/
2488 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2489 {
2490         struct i40e_vsi_context ctxt;
2491         i40e_status ret;
2492
2493         if ((vsi->info.valid_sections &
2494              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2495             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2496                 return;  /* already enabled */
2497
2498         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2499         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2500                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2501
2502         ctxt.seid = vsi->seid;
2503         ctxt.info = vsi->info;
2504         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2505         if (ret) {
2506                 dev_info(&vsi->back->pdev->dev,
2507                          "update vlan stripping failed, err %s aq_err %s\n",
2508                          i40e_stat_str(&vsi->back->hw, ret),
2509                          i40e_aq_str(&vsi->back->hw,
2510                                      vsi->back->hw.aq.asq_last_status));
2511         }
2512 }
2513
2514 /**
2515  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2516  * @vsi: the vsi being adjusted
2517  **/
2518 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2519 {
2520         struct i40e_vsi_context ctxt;
2521         i40e_status ret;
2522
2523         if ((vsi->info.valid_sections &
2524              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2525             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2526              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2527                 return;  /* already disabled */
2528
2529         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2530         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2531                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2532
2533         ctxt.seid = vsi->seid;
2534         ctxt.info = vsi->info;
2535         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2536         if (ret) {
2537                 dev_info(&vsi->back->pdev->dev,
2538                          "update vlan stripping failed, err %s aq_err %s\n",
2539                          i40e_stat_str(&vsi->back->hw, ret),
2540                          i40e_aq_str(&vsi->back->hw,
2541                                      vsi->back->hw.aq.asq_last_status));
2542         }
2543 }
2544
2545 /**
2546  * i40e_vlan_rx_register - Setup or shutdown vlan offload
2547  * @netdev: network interface to be adjusted
2548  * @features: netdev features to test if VLAN offload is enabled or not
2549  **/
2550 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
2551 {
2552         struct i40e_netdev_priv *np = netdev_priv(netdev);
2553         struct i40e_vsi *vsi = np->vsi;
2554
2555         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2556                 i40e_vlan_stripping_enable(vsi);
2557         else
2558                 i40e_vlan_stripping_disable(vsi);
2559 }
2560
2561 /**
2562  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2563  * @vsi: the vsi being configured
2564  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2565  *
2566  * This is a helper function for adding a new MAC/VLAN filter with the
2567  * specified VLAN for each existing MAC address already in the hash table.
2568  * This function does *not* perform any accounting to update filters based on
2569  * VLAN mode.
2570  *
2571  * NOTE: this function expects to be called while under the
2572  * mac_filter_hash_lock
2573  **/
2574 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2575 {
2576         struct i40e_mac_filter *f, *add_f;
2577         struct hlist_node *h;
2578         int bkt;
2579
2580         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2581                 if (f->state == I40E_FILTER_REMOVE)
2582                         continue;
2583                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2584                 if (!add_f) {
2585                         dev_info(&vsi->back->pdev->dev,
2586                                  "Could not add vlan filter %d for %pM\n",
2587                                  vid, f->macaddr);
2588                         return -ENOMEM;
2589                 }
2590         }
2591
2592         return 0;
2593 }
2594
2595 /**
2596  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2597  * @vsi: the VSI being configured
2598  * @vid: VLAN id to be added
2599  **/
2600 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2601 {
2602         int err;
2603
2604         if (vsi->info.pvid)
2605                 return -EINVAL;
2606
2607         /* The network stack will attempt to add VID=0, with the intention to
2608          * receive priority tagged packets with a VLAN of 0. Our HW receives
2609          * these packets by default when configured to receive untagged
2610          * packets, so we don't need to add a filter for this case.
2611          * Additionally, HW interprets adding a VID=0 filter as meaning to
2612          * receive *only* tagged traffic and stops receiving untagged traffic.
2613          * Thus, we do not want to actually add a filter for VID=0
2614          */
2615         if (!vid)
2616                 return 0;
2617
2618         /* Locked once because all functions invoked below iterates list*/
2619         spin_lock_bh(&vsi->mac_filter_hash_lock);
2620         err = i40e_add_vlan_all_mac(vsi, vid);
2621         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2622         if (err)
2623                 return err;
2624
2625         /* schedule our worker thread which will take care of
2626          * applying the new filter changes
2627          */
2628         i40e_service_event_schedule(vsi->back);
2629         return 0;
2630 }
2631
2632 /**
2633  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2634  * @vsi: the vsi being configured
2635  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2636  *
2637  * This function should be used to remove all VLAN filters which match the
2638  * given VID. It does not schedule the service event and does not take the
2639  * mac_filter_hash_lock so it may be combined with other operations under
2640  * a single invocation of the mac_filter_hash_lock.
2641  *
2642  * NOTE: this function expects to be called while under the
2643  * mac_filter_hash_lock
2644  */
2645 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2646 {
2647         struct i40e_mac_filter *f;
2648         struct hlist_node *h;
2649         int bkt;
2650
2651         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2652                 if (f->vlan == vid)
2653                         __i40e_del_filter(vsi, f);
2654         }
2655 }
2656
2657 /**
2658  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2659  * @vsi: the VSI being configured
2660  * @vid: VLAN id to be removed
2661  **/
2662 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2663 {
2664         if (!vid || vsi->info.pvid)
2665                 return;
2666
2667         spin_lock_bh(&vsi->mac_filter_hash_lock);
2668         i40e_rm_vlan_all_mac(vsi, vid);
2669         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2670
2671         /* schedule our worker thread which will take care of
2672          * applying the new filter changes
2673          */
2674         i40e_service_event_schedule(vsi->back);
2675 }
2676
2677 /**
2678  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2679  * @netdev: network interface to be adjusted
2680  * @vid: vlan id to be added
2681  *
2682  * net_device_ops implementation for adding vlan ids
2683  **/
2684 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2685                                 __always_unused __be16 proto, u16 vid)
2686 {
2687         struct i40e_netdev_priv *np = netdev_priv(netdev);
2688         struct i40e_vsi *vsi = np->vsi;
2689         int ret = 0;
2690
2691         if (vid >= VLAN_N_VID)
2692                 return -EINVAL;
2693
2694         ret = i40e_vsi_add_vlan(vsi, vid);
2695         if (!ret)
2696                 set_bit(vid, vsi->active_vlans);
2697
2698         return ret;
2699 }
2700
2701 /**
2702  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2703  * @netdev: network interface to be adjusted
2704  * @vid: vlan id to be removed
2705  *
2706  * net_device_ops implementation for removing vlan ids
2707  **/
2708 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2709                                  __always_unused __be16 proto, u16 vid)
2710 {
2711         struct i40e_netdev_priv *np = netdev_priv(netdev);
2712         struct i40e_vsi *vsi = np->vsi;
2713
2714         /* return code is ignored as there is nothing a user
2715          * can do about failure to remove and a log message was
2716          * already printed from the other function
2717          */
2718         i40e_vsi_kill_vlan(vsi, vid);
2719
2720         clear_bit(vid, vsi->active_vlans);
2721
2722         return 0;
2723 }
2724
2725 /**
2726  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2727  * @vsi: the vsi being brought back up
2728  **/
2729 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2730 {
2731         u16 vid;
2732
2733         if (!vsi->netdev)
2734                 return;
2735
2736         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2737
2738         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2739                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2740                                      vid);
2741 }
2742
2743 /**
2744  * i40e_vsi_add_pvid - Add pvid for the VSI
2745  * @vsi: the vsi being adjusted
2746  * @vid: the vlan id to set as a PVID
2747  **/
2748 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2749 {
2750         struct i40e_vsi_context ctxt;
2751         i40e_status ret;
2752
2753         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2754         vsi->info.pvid = cpu_to_le16(vid);
2755         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2756                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2757                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2758
2759         ctxt.seid = vsi->seid;
2760         ctxt.info = vsi->info;
2761         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2762         if (ret) {
2763                 dev_info(&vsi->back->pdev->dev,
2764                          "add pvid failed, err %s aq_err %s\n",
2765                          i40e_stat_str(&vsi->back->hw, ret),
2766                          i40e_aq_str(&vsi->back->hw,
2767                                      vsi->back->hw.aq.asq_last_status));
2768                 return -ENOENT;
2769         }
2770
2771         return 0;
2772 }
2773
2774 /**
2775  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2776  * @vsi: the vsi being adjusted
2777  *
2778  * Just use the vlan_rx_register() service to put it back to normal
2779  **/
2780 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2781 {
2782         i40e_vlan_stripping_disable(vsi);
2783
2784         vsi->info.pvid = 0;
2785 }
2786
2787 /**
2788  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2789  * @vsi: ptr to the VSI
2790  *
2791  * If this function returns with an error, then it's possible one or
2792  * more of the rings is populated (while the rest are not).  It is the
2793  * callers duty to clean those orphaned rings.
2794  *
2795  * Return 0 on success, negative on failure
2796  **/
2797 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2798 {
2799         int i, err = 0;
2800
2801         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2802                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2803
2804         if (!i40e_enabled_xdp_vsi(vsi))
2805                 return err;
2806
2807         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2808                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
2809
2810         return err;
2811 }
2812
2813 /**
2814  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2815  * @vsi: ptr to the VSI
2816  *
2817  * Free VSI's transmit software resources
2818  **/
2819 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2820 {
2821         int i;
2822
2823         if (vsi->tx_rings) {
2824                 for (i = 0; i < vsi->num_queue_pairs; i++)
2825                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2826                                 i40e_free_tx_resources(vsi->tx_rings[i]);
2827         }
2828
2829         if (vsi->xdp_rings) {
2830                 for (i = 0; i < vsi->num_queue_pairs; i++)
2831                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
2832                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
2833         }
2834 }
2835
2836 /**
2837  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2838  * @vsi: ptr to the VSI
2839  *
2840  * If this function returns with an error, then it's possible one or
2841  * more of the rings is populated (while the rest are not).  It is the
2842  * callers duty to clean those orphaned rings.
2843  *
2844  * Return 0 on success, negative on failure
2845  **/
2846 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2847 {
2848         int i, err = 0;
2849
2850         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2851                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2852         return err;
2853 }
2854
2855 /**
2856  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2857  * @vsi: ptr to the VSI
2858  *
2859  * Free all receive software resources
2860  **/
2861 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2862 {
2863         int i;
2864
2865         if (!vsi->rx_rings)
2866                 return;
2867
2868         for (i = 0; i < vsi->num_queue_pairs; i++)
2869                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2870                         i40e_free_rx_resources(vsi->rx_rings[i]);
2871 }
2872
2873 /**
2874  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
2875  * @ring: The Tx ring to configure
2876  *
2877  * This enables/disables XPS for a given Tx descriptor ring
2878  * based on the TCs enabled for the VSI that ring belongs to.
2879  **/
2880 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
2881 {
2882         struct i40e_vsi *vsi = ring->vsi;
2883         int cpu;
2884
2885         if (!ring->q_vector || !ring->netdev)
2886                 return;
2887
2888         if ((vsi->tc_config.numtc <= 1) &&
2889             !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) {
2890                 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
2891                 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
2892                                     ring->queue_index);
2893         }
2894
2895         /* schedule our worker thread which will take care of
2896          * applying the new filter changes
2897          */
2898         i40e_service_event_schedule(vsi->back);
2899 }
2900
2901 /**
2902  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2903  * @ring: The Tx ring to configure
2904  *
2905  * Configure the Tx descriptor ring in the HMC context.
2906  **/
2907 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2908 {
2909         struct i40e_vsi *vsi = ring->vsi;
2910         u16 pf_q = vsi->base_queue + ring->queue_index;
2911         struct i40e_hw *hw = &vsi->back->hw;
2912         struct i40e_hmc_obj_txq tx_ctx;
2913         i40e_status err = 0;
2914         u32 qtx_ctl = 0;
2915
2916         /* some ATR related tx ring init */
2917         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
2918                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2919                 ring->atr_count = 0;
2920         } else {
2921                 ring->atr_sample_rate = 0;
2922         }
2923
2924         /* configure XPS */
2925         i40e_config_xps_tx_ring(ring);
2926
2927         /* clear the context structure first */
2928         memset(&tx_ctx, 0, sizeof(tx_ctx));
2929
2930         tx_ctx.new_context = 1;
2931         tx_ctx.base = (ring->dma / 128);
2932         tx_ctx.qlen = ring->count;
2933         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
2934                                                I40E_FLAG_FD_ATR_ENABLED));
2935         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2936         /* FDIR VSI tx ring can still use RS bit and writebacks */
2937         if (vsi->type != I40E_VSI_FDIR)
2938                 tx_ctx.head_wb_ena = 1;
2939         tx_ctx.head_wb_addr = ring->dma +
2940                               (ring->count * sizeof(struct i40e_tx_desc));
2941
2942         /* As part of VSI creation/update, FW allocates certain
2943          * Tx arbitration queue sets for each TC enabled for
2944          * the VSI. The FW returns the handles to these queue
2945          * sets as part of the response buffer to Add VSI,
2946          * Update VSI, etc. AQ commands. It is expected that
2947          * these queue set handles be associated with the Tx
2948          * queues by the driver as part of the TX queue context
2949          * initialization. This has to be done regardless of
2950          * DCB as by default everything is mapped to TC0.
2951          */
2952         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2953         tx_ctx.rdylist_act = 0;
2954
2955         /* clear the context in the HMC */
2956         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2957         if (err) {
2958                 dev_info(&vsi->back->pdev->dev,
2959                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2960                          ring->queue_index, pf_q, err);
2961                 return -ENOMEM;
2962         }
2963
2964         /* set the context in the HMC */
2965         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2966         if (err) {
2967                 dev_info(&vsi->back->pdev->dev,
2968                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2969                          ring->queue_index, pf_q, err);
2970                 return -ENOMEM;
2971         }
2972
2973         /* Now associate this queue with this PCI function */
2974         if (vsi->type == I40E_VSI_VMDQ2) {
2975                 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
2976                 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
2977                            I40E_QTX_CTL_VFVM_INDX_MASK;
2978         } else {
2979                 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2980         }
2981
2982         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2983                     I40E_QTX_CTL_PF_INDX_MASK);
2984         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2985         i40e_flush(hw);
2986
2987         /* cache tail off for easier writes later */
2988         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2989
2990         return 0;
2991 }
2992
2993 /**
2994  * i40e_configure_rx_ring - Configure a receive ring context
2995  * @ring: The Rx ring to configure
2996  *
2997  * Configure the Rx descriptor ring in the HMC context.
2998  **/
2999 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3000 {
3001         struct i40e_vsi *vsi = ring->vsi;
3002         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3003         u16 pf_q = vsi->base_queue + ring->queue_index;
3004         struct i40e_hw *hw = &vsi->back->hw;
3005         struct i40e_hmc_obj_rxq rx_ctx;
3006         i40e_status err = 0;
3007
3008         bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3009
3010         /* clear the context structure first */
3011         memset(&rx_ctx, 0, sizeof(rx_ctx));
3012
3013         ring->rx_buf_len = vsi->rx_buf_len;
3014
3015         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3016                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3017
3018         rx_ctx.base = (ring->dma / 128);
3019         rx_ctx.qlen = ring->count;
3020
3021         /* use 32 byte descriptors */
3022         rx_ctx.dsize = 1;
3023
3024         /* descriptor type is always zero
3025          * rx_ctx.dtype = 0;
3026          */
3027         rx_ctx.hsplit_0 = 0;
3028
3029         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3030         if (hw->revision_id == 0)
3031                 rx_ctx.lrxqthresh = 0;
3032         else
3033                 rx_ctx.lrxqthresh = 2;
3034         rx_ctx.crcstrip = 1;
3035         rx_ctx.l2tsel = 1;
3036         /* this controls whether VLAN is stripped from inner headers */
3037         rx_ctx.showiv = 0;
3038         /* set the prefena field to 1 because the manual says to */
3039         rx_ctx.prefena = 1;
3040
3041         /* clear the context in the HMC */
3042         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3043         if (err) {
3044                 dev_info(&vsi->back->pdev->dev,
3045                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3046                          ring->queue_index, pf_q, err);
3047                 return -ENOMEM;
3048         }
3049
3050         /* set the context in the HMC */
3051         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3052         if (err) {
3053                 dev_info(&vsi->back->pdev->dev,
3054                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3055                          ring->queue_index, pf_q, err);
3056                 return -ENOMEM;
3057         }
3058
3059         /* configure Rx buffer alignment */
3060         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3061                 clear_ring_build_skb_enabled(ring);
3062         else
3063                 set_ring_build_skb_enabled(ring);
3064
3065         /* cache tail for quicker writes, and clear the reg before use */
3066         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3067         writel(0, ring->tail);
3068
3069         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3070
3071         return 0;
3072 }
3073
3074 /**
3075  * i40e_vsi_configure_tx - Configure the VSI for Tx
3076  * @vsi: VSI structure describing this set of rings and resources
3077  *
3078  * Configure the Tx VSI for operation.
3079  **/
3080 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3081 {
3082         int err = 0;
3083         u16 i;
3084
3085         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3086                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3087
3088         if (!i40e_enabled_xdp_vsi(vsi))
3089                 return err;
3090
3091         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3092                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3093
3094         return err;
3095 }
3096
3097 /**
3098  * i40e_vsi_configure_rx - Configure the VSI for Rx
3099  * @vsi: the VSI being configured
3100  *
3101  * Configure the Rx VSI for operation.
3102  **/
3103 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3104 {
3105         int err = 0;
3106         u16 i;
3107
3108         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3109                 vsi->max_frame = I40E_MAX_RXBUFFER;
3110                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3111 #if (PAGE_SIZE < 8192)
3112         } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3113                    (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3114                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3115                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3116 #endif
3117         } else {
3118                 vsi->max_frame = I40E_MAX_RXBUFFER;
3119                 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3120                                                        I40E_RXBUFFER_2048;
3121         }
3122
3123         /* set up individual rings */
3124         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3125                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3126
3127         return err;
3128 }
3129
3130 /**
3131  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3132  * @vsi: ptr to the VSI
3133  **/
3134 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3135 {
3136         struct i40e_ring *tx_ring, *rx_ring;
3137         u16 qoffset, qcount;
3138         int i, n;
3139
3140         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3141                 /* Reset the TC information */
3142                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3143                         rx_ring = vsi->rx_rings[i];
3144                         tx_ring = vsi->tx_rings[i];
3145                         rx_ring->dcb_tc = 0;
3146                         tx_ring->dcb_tc = 0;
3147                 }
3148         }
3149
3150         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3151                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3152                         continue;
3153
3154                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3155                 qcount = vsi->tc_config.tc_info[n].qcount;
3156                 for (i = qoffset; i < (qoffset + qcount); i++) {
3157                         rx_ring = vsi->rx_rings[i];
3158                         tx_ring = vsi->tx_rings[i];
3159                         rx_ring->dcb_tc = n;
3160                         tx_ring->dcb_tc = n;
3161                 }
3162         }
3163 }
3164
3165 /**
3166  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3167  * @vsi: ptr to the VSI
3168  **/
3169 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3170 {
3171         if (vsi->netdev)
3172                 i40e_set_rx_mode(vsi->netdev);
3173 }
3174
3175 /**
3176  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3177  * @vsi: Pointer to the targeted VSI
3178  *
3179  * This function replays the hlist on the hw where all the SB Flow Director
3180  * filters were saved.
3181  **/
3182 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3183 {
3184         struct i40e_fdir_filter *filter;
3185         struct i40e_pf *pf = vsi->back;
3186         struct hlist_node *node;
3187
3188         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3189                 return;
3190
3191         /* Reset FDir counters as we're replaying all existing filters */
3192         pf->fd_tcp4_filter_cnt = 0;
3193         pf->fd_udp4_filter_cnt = 0;
3194         pf->fd_sctp4_filter_cnt = 0;
3195         pf->fd_ip4_filter_cnt = 0;
3196
3197         hlist_for_each_entry_safe(filter, node,
3198                                   &pf->fdir_filter_list, fdir_node) {
3199                 i40e_add_del_fdir(vsi, filter, true);
3200         }
3201 }
3202
3203 /**
3204  * i40e_vsi_configure - Set up the VSI for action
3205  * @vsi: the VSI being configured
3206  **/
3207 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3208 {
3209         int err;
3210
3211         i40e_set_vsi_rx_mode(vsi);
3212         i40e_restore_vlan(vsi);
3213         i40e_vsi_config_dcb_rings(vsi);
3214         err = i40e_vsi_configure_tx(vsi);
3215         if (!err)
3216                 err = i40e_vsi_configure_rx(vsi);
3217
3218         return err;
3219 }
3220
3221 /**
3222  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3223  * @vsi: the VSI being configured
3224  **/
3225 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3226 {
3227         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3228         struct i40e_pf *pf = vsi->back;
3229         struct i40e_hw *hw = &pf->hw;
3230         u16 vector;
3231         int i, q;
3232         u32 qp;
3233
3234         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3235          * and PFINT_LNKLSTn registers, e.g.:
3236          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3237          */
3238         qp = vsi->base_queue;
3239         vector = vsi->base_vector;
3240         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3241                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3242
3243                 q_vector->itr_countdown = ITR_COUNTDOWN_START;
3244                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting);
3245                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
3246                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3247                      q_vector->rx.itr);
3248                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting);
3249                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
3250                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3251                      q_vector->tx.itr);
3252                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3253                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3254
3255                 /* Linked list for the queuepairs assigned to this vector */
3256                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3257                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3258                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3259                         u32 val;
3260
3261                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3262                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3263                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3264                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3265                               (I40E_QUEUE_TYPE_TX <<
3266                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3267
3268                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3269
3270                         if (has_xdp) {
3271                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3272                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3273                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3274                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3275                                       (I40E_QUEUE_TYPE_TX <<
3276                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3277
3278                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3279                         }
3280
3281                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3282                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3283                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3284                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3285                               (I40E_QUEUE_TYPE_RX <<
3286                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3287
3288                         /* Terminate the linked list */
3289                         if (q == (q_vector->num_ringpairs - 1))
3290                                 val |= (I40E_QUEUE_END_OF_LIST <<
3291                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3292
3293                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3294                         qp++;
3295                 }
3296         }
3297
3298         i40e_flush(hw);
3299 }
3300
3301 /**
3302  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3303  * @hw: ptr to the hardware info
3304  **/
3305 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3306 {
3307         struct i40e_hw *hw = &pf->hw;
3308         u32 val;
3309
3310         /* clear things first */
3311         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3312         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3313
3314         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3315               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3316               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3317               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3318               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3319               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3320               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3321               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3322
3323         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3324                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3325
3326         if (pf->flags & I40E_FLAG_PTP)
3327                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3328
3329         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3330
3331         /* SW_ITR_IDX = 0, but don't change INTENA */
3332         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3333                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3334
3335         /* OTHER_ITR_IDX = 0 */
3336         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3337 }
3338
3339 /**
3340  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3341  * @vsi: the VSI being configured
3342  **/
3343 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3344 {
3345         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3346         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3347         struct i40e_pf *pf = vsi->back;
3348         struct i40e_hw *hw = &pf->hw;
3349         u32 val;
3350
3351         /* set the ITR configuration */
3352         q_vector->itr_countdown = ITR_COUNTDOWN_START;
3353         q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting);
3354         q_vector->rx.latency_range = I40E_LOW_LATENCY;
3355         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
3356         q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting);
3357         q_vector->tx.latency_range = I40E_LOW_LATENCY;
3358         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
3359
3360         i40e_enable_misc_int_causes(pf);
3361
3362         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3363         wr32(hw, I40E_PFINT_LNKLST0, 0);
3364
3365         /* Associate the queue pair to the vector and enable the queue int */
3366         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                   |
3367               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3368               (nextqp      << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3369               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3370
3371         wr32(hw, I40E_QINT_RQCTL(0), val);
3372
3373         if (i40e_enabled_xdp_vsi(vsi)) {
3374                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                 |
3375                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3376                       (I40E_QUEUE_TYPE_TX
3377                        << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3378
3379                wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3380         }
3381
3382         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3383               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3384               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3385
3386         wr32(hw, I40E_QINT_TQCTL(0), val);
3387         i40e_flush(hw);
3388 }
3389
3390 /**
3391  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3392  * @pf: board private structure
3393  **/
3394 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3395 {
3396         struct i40e_hw *hw = &pf->hw;
3397
3398         wr32(hw, I40E_PFINT_DYN_CTL0,
3399              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3400         i40e_flush(hw);
3401 }
3402
3403 /**
3404  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3405  * @pf: board private structure
3406  * @clearpba: true when all pending interrupt events should be cleared
3407  **/
3408 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba)
3409 {
3410         struct i40e_hw *hw = &pf->hw;
3411         u32 val;
3412
3413         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3414               (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) |
3415               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3416
3417         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3418         i40e_flush(hw);
3419 }
3420
3421 /**
3422  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3423  * @irq: interrupt number
3424  * @data: pointer to a q_vector
3425  **/
3426 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3427 {
3428         struct i40e_q_vector *q_vector = data;
3429
3430         if (!q_vector->tx.ring && !q_vector->rx.ring)
3431                 return IRQ_HANDLED;
3432
3433         napi_schedule_irqoff(&q_vector->napi);
3434
3435         return IRQ_HANDLED;
3436 }
3437
3438 /**
3439  * i40e_irq_affinity_notify - Callback for affinity changes
3440  * @notify: context as to what irq was changed
3441  * @mask: the new affinity mask
3442  *
3443  * This is a callback function used by the irq_set_affinity_notifier function
3444  * so that we may register to receive changes to the irq affinity masks.
3445  **/
3446 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3447                                      const cpumask_t *mask)
3448 {
3449         struct i40e_q_vector *q_vector =
3450                 container_of(notify, struct i40e_q_vector, affinity_notify);
3451
3452         cpumask_copy(&q_vector->affinity_mask, mask);
3453 }
3454
3455 /**
3456  * i40e_irq_affinity_release - Callback for affinity notifier release
3457  * @ref: internal core kernel usage
3458  *
3459  * This is a callback function used by the irq_set_affinity_notifier function
3460  * to inform the current notification subscriber that they will no longer
3461  * receive notifications.
3462  **/
3463 static void i40e_irq_affinity_release(struct kref *ref) {}
3464
3465 /**
3466  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3467  * @vsi: the VSI being configured
3468  * @basename: name for the vector
3469  *
3470  * Allocates MSI-X vectors and requests interrupts from the kernel.
3471  **/
3472 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3473 {
3474         int q_vectors = vsi->num_q_vectors;
3475         struct i40e_pf *pf = vsi->back;
3476         int base = vsi->base_vector;
3477         int rx_int_idx = 0;
3478         int tx_int_idx = 0;
3479         int vector, err;
3480         int irq_num;
3481         int cpu;
3482
3483         for (vector = 0; vector < q_vectors; vector++) {
3484                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3485
3486                 irq_num = pf->msix_entries[base + vector].vector;
3487
3488                 if (q_vector->tx.ring && q_vector->rx.ring) {
3489                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3490                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3491                         tx_int_idx++;
3492                 } else if (q_vector->rx.ring) {
3493                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3494                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3495                 } else if (q_vector->tx.ring) {
3496                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3497                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3498                 } else {
3499                         /* skip this unused q_vector */
3500                         continue;
3501                 }
3502                 err = request_irq(irq_num,
3503                                   vsi->irq_handler,
3504                                   0,
3505                                   q_vector->name,
3506                                   q_vector);
3507                 if (err) {
3508                         dev_info(&pf->pdev->dev,
3509                                  "MSIX request_irq failed, error: %d\n", err);
3510                         goto free_queue_irqs;
3511                 }
3512
3513                 /* register for affinity change notifications */
3514                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3515                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3516                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3517                 /* Spread affinity hints out across online CPUs.
3518                  *
3519                  * get_cpu_mask returns a static constant mask with
3520                  * a permanent lifetime so it's ok to pass to
3521                  * irq_set_affinity_hint without making a copy.
3522                  */
3523                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
3524                 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
3525         }
3526
3527         vsi->irqs_ready = true;
3528         return 0;
3529
3530 free_queue_irqs:
3531         while (vector) {
3532                 vector--;
3533                 irq_num = pf->msix_entries[base + vector].vector;
3534                 irq_set_affinity_notifier(irq_num, NULL);
3535                 irq_set_affinity_hint(irq_num, NULL);
3536                 free_irq(irq_num, &vsi->q_vectors[vector]);
3537         }
3538         return err;
3539 }
3540
3541 /**
3542  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3543  * @vsi: the VSI being un-configured
3544  **/
3545 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3546 {
3547         struct i40e_pf *pf = vsi->back;
3548         struct i40e_hw *hw = &pf->hw;
3549         int base = vsi->base_vector;
3550         int i;
3551
3552         /* disable interrupt causation from each queue */
3553         for (i = 0; i < vsi->num_queue_pairs; i++) {
3554                 u32 val;
3555
3556                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
3557                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3558                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
3559
3560                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
3561                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3562                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
3563
3564                 if (!i40e_enabled_xdp_vsi(vsi))
3565                         continue;
3566                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3567         }
3568
3569         /* disable each interrupt */
3570         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3571                 for (i = vsi->base_vector;
3572                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3573                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3574
3575                 i40e_flush(hw);
3576                 for (i = 0; i < vsi->num_q_vectors; i++)
3577                         synchronize_irq(pf->msix_entries[i + base].vector);
3578         } else {
3579                 /* Legacy and MSI mode - this stops all interrupt handling */
3580                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3581                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3582                 i40e_flush(hw);
3583                 synchronize_irq(pf->pdev->irq);
3584         }
3585 }
3586
3587 /**
3588  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3589  * @vsi: the VSI being configured
3590  **/
3591 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3592 {
3593         struct i40e_pf *pf = vsi->back;
3594         int i;
3595
3596         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3597                 for (i = 0; i < vsi->num_q_vectors; i++)
3598                         i40e_irq_dynamic_enable(vsi, i);
3599         } else {
3600                 i40e_irq_dynamic_enable_icr0(pf, true);
3601         }
3602
3603         i40e_flush(&pf->hw);
3604         return 0;
3605 }
3606
3607 /**
3608  * i40e_free_misc_vector - Free the vector that handles non-queue events
3609  * @pf: board private structure
3610  **/
3611 static void i40e_free_misc_vector(struct i40e_pf *pf)
3612 {
3613         /* Disable ICR 0 */
3614         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3615         i40e_flush(&pf->hw);
3616
3617         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
3618                 synchronize_irq(pf->msix_entries[0].vector);
3619                 free_irq(pf->msix_entries[0].vector, pf);
3620                 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
3621         }
3622 }
3623
3624 /**
3625  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3626  * @irq: interrupt number
3627  * @data: pointer to a q_vector
3628  *
3629  * This is the handler used for all MSI/Legacy interrupts, and deals
3630  * with both queue and non-queue interrupts.  This is also used in
3631  * MSIX mode to handle the non-queue interrupts.
3632  **/
3633 static irqreturn_t i40e_intr(int irq, void *data)
3634 {
3635         struct i40e_pf *pf = (struct i40e_pf *)data;
3636         struct i40e_hw *hw = &pf->hw;
3637         irqreturn_t ret = IRQ_NONE;
3638         u32 icr0, icr0_remaining;
3639         u32 val, ena_mask;
3640
3641         icr0 = rd32(hw, I40E_PFINT_ICR0);
3642         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3643
3644         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3645         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3646                 goto enable_intr;
3647
3648         /* if interrupt but no bits showing, must be SWINT */
3649         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3650             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3651                 pf->sw_int_count++;
3652
3653         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3654             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3655                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3656                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3657                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
3658         }
3659
3660         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3661         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3662                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3663                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3664
3665                 /* We do not have a way to disarm Queue causes while leaving
3666                  * interrupt enabled for all other causes, ideally
3667                  * interrupt should be disabled while we are in NAPI but
3668                  * this is not a performance path and napi_schedule()
3669                  * can deal with rescheduling.
3670                  */
3671                 if (!test_bit(__I40E_DOWN, pf->state))
3672                         napi_schedule_irqoff(&q_vector->napi);
3673         }
3674
3675         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3676                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3677                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
3678                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3679         }
3680
3681         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3682                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3683                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
3684         }
3685
3686         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3687                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3688                 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
3689         }
3690
3691         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3692                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
3693                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
3694                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3695                 val = rd32(hw, I40E_GLGEN_RSTAT);
3696                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3697                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3698                 if (val == I40E_RESET_CORER) {
3699                         pf->corer_count++;
3700                 } else if (val == I40E_RESET_GLOBR) {
3701                         pf->globr_count++;
3702                 } else if (val == I40E_RESET_EMPR) {
3703                         pf->empr_count++;
3704                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
3705                 }
3706         }
3707
3708         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3709                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3710                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3711                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
3712                          rd32(hw, I40E_PFHMC_ERRORINFO),
3713                          rd32(hw, I40E_PFHMC_ERRORDATA));
3714         }
3715
3716         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3717                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3718
3719                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
3720                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3721                         i40e_ptp_tx_hwtstamp(pf);
3722                 }
3723         }
3724
3725         /* If a critical error is pending we have no choice but to reset the
3726          * device.
3727          * Report and mask out any remaining unexpected interrupts.
3728          */
3729         icr0_remaining = icr0 & ena_mask;
3730         if (icr0_remaining) {
3731                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
3732                          icr0_remaining);
3733                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
3734                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
3735                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
3736                         dev_info(&pf->pdev->dev, "device will be reset\n");
3737                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
3738                         i40e_service_event_schedule(pf);
3739                 }
3740                 ena_mask &= ~icr0_remaining;
3741         }
3742         ret = IRQ_HANDLED;
3743
3744 enable_intr:
3745         /* re-enable interrupt causes */
3746         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
3747         if (!test_bit(__I40E_DOWN, pf->state)) {
3748                 i40e_service_event_schedule(pf);
3749                 i40e_irq_dynamic_enable_icr0(pf, false);
3750         }
3751
3752         return ret;
3753 }
3754
3755 /**
3756  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
3757  * @tx_ring:  tx ring to clean
3758  * @budget:   how many cleans we're allowed
3759  *
3760  * Returns true if there's any budget left (e.g. the clean is finished)
3761  **/
3762 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
3763 {
3764         struct i40e_vsi *vsi = tx_ring->vsi;
3765         u16 i = tx_ring->next_to_clean;
3766         struct i40e_tx_buffer *tx_buf;
3767         struct i40e_tx_desc *tx_desc;
3768
3769         tx_buf = &tx_ring->tx_bi[i];
3770         tx_desc = I40E_TX_DESC(tx_ring, i);
3771         i -= tx_ring->count;
3772
3773         do {
3774                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
3775
3776                 /* if next_to_watch is not set then there is no work pending */
3777                 if (!eop_desc)
3778                         break;
3779
3780                 /* prevent any other reads prior to eop_desc */
3781                 read_barrier_depends();
3782
3783                 /* if the descriptor isn't done, no work yet to do */
3784                 if (!(eop_desc->cmd_type_offset_bsz &
3785                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
3786                         break;
3787
3788                 /* clear next_to_watch to prevent false hangs */
3789                 tx_buf->next_to_watch = NULL;
3790
3791                 tx_desc->buffer_addr = 0;
3792                 tx_desc->cmd_type_offset_bsz = 0;
3793                 /* move past filter desc */
3794                 tx_buf++;
3795                 tx_desc++;
3796                 i++;
3797                 if (unlikely(!i)) {
3798                         i -= tx_ring->count;
3799                         tx_buf = tx_ring->tx_bi;
3800                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3801                 }
3802                 /* unmap skb header data */
3803                 dma_unmap_single(tx_ring->dev,
3804                                  dma_unmap_addr(tx_buf, dma),
3805                                  dma_unmap_len(tx_buf, len),
3806                                  DMA_TO_DEVICE);
3807                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
3808                         kfree(tx_buf->raw_buf);
3809
3810                 tx_buf->raw_buf = NULL;
3811                 tx_buf->tx_flags = 0;
3812                 tx_buf->next_to_watch = NULL;
3813                 dma_unmap_len_set(tx_buf, len, 0);
3814                 tx_desc->buffer_addr = 0;
3815                 tx_desc->cmd_type_offset_bsz = 0;
3816
3817                 /* move us past the eop_desc for start of next FD desc */
3818                 tx_buf++;
3819                 tx_desc++;
3820                 i++;
3821                 if (unlikely(!i)) {
3822                         i -= tx_ring->count;
3823                         tx_buf = tx_ring->tx_bi;
3824                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3825                 }
3826
3827                 /* update budget accounting */
3828                 budget--;
3829         } while (likely(budget));
3830
3831         i += tx_ring->count;
3832         tx_ring->next_to_clean = i;
3833
3834         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
3835                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
3836
3837         return budget > 0;
3838 }
3839
3840 /**
3841  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
3842  * @irq: interrupt number
3843  * @data: pointer to a q_vector
3844  **/
3845 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
3846 {
3847         struct i40e_q_vector *q_vector = data;
3848         struct i40e_vsi *vsi;
3849
3850         if (!q_vector->tx.ring)
3851                 return IRQ_HANDLED;
3852
3853         vsi = q_vector->tx.ring->vsi;
3854         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
3855
3856         return IRQ_HANDLED;
3857 }
3858
3859 /**
3860  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
3861  * @vsi: the VSI being configured
3862  * @v_idx: vector index
3863  * @qp_idx: queue pair index
3864  **/
3865 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
3866 {
3867         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3868         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
3869         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
3870
3871         tx_ring->q_vector = q_vector;
3872         tx_ring->next = q_vector->tx.ring;
3873         q_vector->tx.ring = tx_ring;
3874         q_vector->tx.count++;
3875
3876         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
3877         if (i40e_enabled_xdp_vsi(vsi)) {
3878                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
3879
3880                 xdp_ring->q_vector = q_vector;
3881                 xdp_ring->next = q_vector->tx.ring;
3882                 q_vector->tx.ring = xdp_ring;
3883                 q_vector->tx.count++;
3884         }
3885
3886         rx_ring->q_vector = q_vector;
3887         rx_ring->next = q_vector->rx.ring;
3888         q_vector->rx.ring = rx_ring;
3889         q_vector->rx.count++;
3890 }
3891
3892 /**
3893  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
3894  * @vsi: the VSI being configured
3895  *
3896  * This function maps descriptor rings to the queue-specific vectors
3897  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
3898  * one vector per queue pair, but on a constrained vector budget, we
3899  * group the queue pairs as "efficiently" as possible.
3900  **/
3901 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
3902 {
3903         int qp_remaining = vsi->num_queue_pairs;
3904         int q_vectors = vsi->num_q_vectors;
3905         int num_ringpairs;
3906         int v_start = 0;
3907         int qp_idx = 0;
3908
3909         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
3910          * group them so there are multiple queues per vector.
3911          * It is also important to go through all the vectors available to be
3912          * sure that if we don't use all the vectors, that the remaining vectors
3913          * are cleared. This is especially important when decreasing the
3914          * number of queues in use.
3915          */
3916         for (; v_start < q_vectors; v_start++) {
3917                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
3918
3919                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
3920
3921                 q_vector->num_ringpairs = num_ringpairs;
3922
3923                 q_vector->rx.count = 0;
3924                 q_vector->tx.count = 0;
3925                 q_vector->rx.ring = NULL;
3926                 q_vector->tx.ring = NULL;
3927
3928                 while (num_ringpairs--) {
3929                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
3930                         qp_idx++;
3931                         qp_remaining--;
3932                 }
3933         }
3934 }
3935
3936 /**
3937  * i40e_vsi_request_irq - Request IRQ from the OS
3938  * @vsi: the VSI being configured
3939  * @basename: name for the vector
3940  **/
3941 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
3942 {
3943         struct i40e_pf *pf = vsi->back;
3944         int err;
3945
3946         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3947                 err = i40e_vsi_request_irq_msix(vsi, basename);
3948         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
3949                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
3950                                   pf->int_name, pf);
3951         else
3952                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
3953                                   pf->int_name, pf);
3954
3955         if (err)
3956                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
3957
3958         return err;
3959 }
3960
3961 #ifdef CONFIG_NET_POLL_CONTROLLER
3962 /**
3963  * i40e_netpoll - A Polling 'interrupt' handler
3964  * @netdev: network interface device structure
3965  *
3966  * This is used by netconsole to send skbs without having to re-enable
3967  * interrupts.  It's not called while the normal interrupt routine is executing.
3968  **/
3969 static void i40e_netpoll(struct net_device *netdev)
3970 {
3971         struct i40e_netdev_priv *np = netdev_priv(netdev);
3972         struct i40e_vsi *vsi = np->vsi;
3973         struct i40e_pf *pf = vsi->back;
3974         int i;
3975
3976         /* if interface is down do nothing */
3977         if (test_bit(__I40E_VSI_DOWN, vsi->state))
3978                 return;
3979
3980         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3981                 for (i = 0; i < vsi->num_q_vectors; i++)
3982                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
3983         } else {
3984                 i40e_intr(pf->pdev->irq, netdev);
3985         }
3986 }
3987 #endif
3988
3989 #define I40E_QTX_ENA_WAIT_COUNT 50
3990
3991 /**
3992  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
3993  * @pf: the PF being configured
3994  * @pf_q: the PF queue
3995  * @enable: enable or disable state of the queue
3996  *
3997  * This routine will wait for the given Tx queue of the PF to reach the
3998  * enabled or disabled state.
3999  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4000  * multiple retries; else will return 0 in case of success.
4001  **/
4002 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4003 {
4004         int i;
4005         u32 tx_reg;
4006
4007         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4008                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4009                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4010                         break;
4011
4012                 usleep_range(10, 20);
4013         }
4014         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4015                 return -ETIMEDOUT;
4016
4017         return 0;
4018 }
4019
4020 /**
4021  * i40e_control_tx_q - Start or stop a particular Tx queue
4022  * @pf: the PF structure
4023  * @pf_q: the PF queue to configure
4024  * @enable: start or stop the queue
4025  *
4026  * This function enables or disables a single queue. Note that any delay
4027  * required after the operation is expected to be handled by the caller of
4028  * this function.
4029  **/
4030 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4031 {
4032         struct i40e_hw *hw = &pf->hw;
4033         u32 tx_reg;
4034         int i;
4035
4036         /* warn the TX unit of coming changes */
4037         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4038         if (!enable)
4039                 usleep_range(10, 20);
4040
4041         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4042                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4043                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4044                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4045                         break;
4046                 usleep_range(1000, 2000);
4047         }
4048
4049         /* Skip if the queue is already in the requested state */
4050         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4051                 return;
4052
4053         /* turn on/off the queue */
4054         if (enable) {
4055                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4056                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4057         } else {
4058                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4059         }
4060
4061         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4062 }
4063
4064 /**
4065  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4066  * @seid: VSI SEID
4067  * @pf: the PF structure
4068  * @pf_q: the PF queue to configure
4069  * @is_xdp: true if the queue is used for XDP
4070  * @enable: start or stop the queue
4071  **/
4072 static int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4073                                   bool is_xdp, bool enable)
4074 {
4075         int ret;
4076
4077         i40e_control_tx_q(pf, pf_q, enable);
4078
4079         /* wait for the change to finish */
4080         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4081         if (ret) {
4082                 dev_info(&pf->pdev->dev,
4083                          "VSI seid %d %sTx ring %d %sable timeout\n",
4084                          seid, (is_xdp ? "XDP " : ""), pf_q,
4085                          (enable ? "en" : "dis"));
4086         }
4087
4088         return ret;
4089 }
4090
4091 /**
4092  * i40e_vsi_control_tx - Start or stop a VSI's rings
4093  * @vsi: the VSI being configured
4094  * @enable: start or stop the rings
4095  **/
4096 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
4097 {
4098         struct i40e_pf *pf = vsi->back;
4099         int i, pf_q, ret = 0;
4100
4101         pf_q = vsi->base_queue;
4102         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4103                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4104                                              pf_q,
4105                                              false /*is xdp*/, enable);
4106                 if (ret)
4107                         break;
4108
4109                 if (!i40e_enabled_xdp_vsi(vsi))
4110                         continue;
4111
4112                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4113                                              pf_q + vsi->alloc_queue_pairs,
4114                                              true /*is xdp*/, enable);
4115                 if (ret)
4116                         break;
4117         }
4118
4119         return ret;
4120 }
4121
4122 /**
4123  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4124  * @pf: the PF being configured
4125  * @pf_q: the PF queue
4126  * @enable: enable or disable state of the queue
4127  *
4128  * This routine will wait for the given Rx queue of the PF to reach the
4129  * enabled or disabled state.
4130  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4131  * multiple retries; else will return 0 in case of success.
4132  **/
4133 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4134 {
4135         int i;
4136         u32 rx_reg;
4137
4138         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4139                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4140                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4141                         break;
4142
4143                 usleep_range(10, 20);
4144         }
4145         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4146                 return -ETIMEDOUT;
4147
4148         return 0;
4149 }
4150
4151 /**
4152  * i40e_control_rx_q - Start or stop a particular Rx queue
4153  * @pf: the PF structure
4154  * @pf_q: the PF queue to configure
4155  * @enable: start or stop the queue
4156  *
4157  * This function enables or disables a single queue. Note that any delay
4158  * required after the operation is expected to be handled by the caller of
4159  * this function.
4160  **/
4161 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4162 {
4163         struct i40e_hw *hw = &pf->hw;
4164         u32 rx_reg;
4165         int i;
4166
4167         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4168                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4169                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4170                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4171                         break;
4172                 usleep_range(1000, 2000);
4173         }
4174
4175         /* Skip if the queue is already in the requested state */
4176         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4177                 return;
4178
4179         /* turn on/off the queue */
4180         if (enable)
4181                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4182         else
4183                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4184
4185         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4186 }
4187
4188 /**
4189  * i40e_vsi_control_rx - Start or stop a VSI's rings
4190  * @vsi: the VSI being configured
4191  * @enable: start or stop the rings
4192  **/
4193 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
4194 {
4195         struct i40e_pf *pf = vsi->back;
4196         int i, pf_q, ret = 0;
4197
4198         pf_q = vsi->base_queue;
4199         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4200                 i40e_control_rx_q(pf, pf_q, enable);
4201
4202                 /* wait for the change to finish */
4203                 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4204                 if (ret) {
4205                         dev_info(&pf->pdev->dev,
4206                                  "VSI seid %d Rx ring %d %sable timeout\n",
4207                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
4208                         break;
4209                 }
4210         }
4211
4212         /* Due to HW errata, on Rx disable only, the register can indicate done
4213          * before it really is. Needs 50ms to be sure
4214          */
4215         if (!enable)
4216                 mdelay(50);
4217
4218         return ret;
4219 }
4220
4221 /**
4222  * i40e_vsi_start_rings - Start a VSI's rings
4223  * @vsi: the VSI being configured
4224  **/
4225 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4226 {
4227         int ret = 0;
4228
4229         /* do rx first for enable and last for disable */
4230         ret = i40e_vsi_control_rx(vsi, true);
4231         if (ret)
4232                 return ret;
4233         ret = i40e_vsi_control_tx(vsi, true);
4234
4235         return ret;
4236 }
4237
4238 /**
4239  * i40e_vsi_stop_rings - Stop a VSI's rings
4240  * @vsi: the VSI being configured
4241  **/
4242 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4243 {
4244         /* When port TX is suspended, don't wait */
4245         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4246                 return i40e_vsi_stop_rings_no_wait(vsi);
4247
4248         /* do rx first for enable and last for disable
4249          * Ignore return value, we need to shutdown whatever we can
4250          */
4251         i40e_vsi_control_tx(vsi, false);
4252         i40e_vsi_control_rx(vsi, false);
4253 }
4254
4255 /**
4256  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4257  * @vsi: the VSI being shutdown
4258  *
4259  * This function stops all the rings for a VSI but does not delay to verify
4260  * that rings have been disabled. It is expected that the caller is shutting
4261  * down multiple VSIs at once and will delay together for all the VSIs after
4262  * initiating the shutdown. This is particularly useful for shutting down lots
4263  * of VFs together. Otherwise, a large delay can be incurred while configuring
4264  * each VSI in serial.
4265  **/
4266 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4267 {
4268         struct i40e_pf *pf = vsi->back;
4269         int i, pf_q;
4270
4271         pf_q = vsi->base_queue;
4272         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4273                 i40e_control_tx_q(pf, pf_q, false);
4274                 i40e_control_rx_q(pf, pf_q, false);
4275         }
4276 }
4277
4278 /**
4279  * i40e_vsi_free_irq - Free the irq association with the OS
4280  * @vsi: the VSI being configured
4281  **/
4282 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4283 {
4284         struct i40e_pf *pf = vsi->back;
4285         struct i40e_hw *hw = &pf->hw;
4286         int base = vsi->base_vector;
4287         u32 val, qp;
4288         int i;
4289
4290         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4291                 if (!vsi->q_vectors)
4292                         return;
4293
4294                 if (!vsi->irqs_ready)
4295                         return;
4296
4297                 vsi->irqs_ready = false;
4298                 for (i = 0; i < vsi->num_q_vectors; i++) {
4299                         int irq_num;
4300                         u16 vector;
4301
4302                         vector = i + base;
4303                         irq_num = pf->msix_entries[vector].vector;
4304
4305                         /* free only the irqs that were actually requested */
4306                         if (!vsi->q_vectors[i] ||
4307                             !vsi->q_vectors[i]->num_ringpairs)
4308                                 continue;
4309
4310                         /* clear the affinity notifier in the IRQ descriptor */
4311                         irq_set_affinity_notifier(irq_num, NULL);
4312                         /* remove our suggested affinity mask for this IRQ */
4313                         irq_set_affinity_hint(irq_num, NULL);
4314                         synchronize_irq(irq_num);
4315                         free_irq(irq_num, vsi->q_vectors[i]);
4316
4317                         /* Tear down the interrupt queue link list
4318                          *
4319                          * We know that they come in pairs and always
4320                          * the Rx first, then the Tx.  To clear the
4321                          * link list, stick the EOL value into the
4322                          * next_q field of the registers.
4323                          */
4324                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4325                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4326                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4327                         val |= I40E_QUEUE_END_OF_LIST
4328                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4329                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4330
4331                         while (qp != I40E_QUEUE_END_OF_LIST) {
4332                                 u32 next;
4333
4334                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4335
4336                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4337                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4338                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4339                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4340
4341                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4342                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4343
4344                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4345
4346                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4347
4348                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4349                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4350
4351                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4352                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4353                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4354                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4355
4356                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4357                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4358
4359                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4360                                 qp = next;
4361                         }
4362                 }
4363         } else {
4364                 free_irq(pf->pdev->irq, pf);
4365
4366                 val = rd32(hw, I40E_PFINT_LNKLST0);
4367                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4368                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4369                 val |= I40E_QUEUE_END_OF_LIST
4370                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4371                 wr32(hw, I40E_PFINT_LNKLST0, val);
4372
4373                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4374                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4375                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4376                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4377                          I40E_QINT_RQCTL_INTEVENT_MASK);
4378
4379                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4380                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4381
4382                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4383
4384                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4385
4386                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4387                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4388                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4389                          I40E_QINT_TQCTL_INTEVENT_MASK);
4390
4391                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4392                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4393
4394                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4395         }
4396 }
4397
4398 /**
4399  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4400  * @vsi: the VSI being configured
4401  * @v_idx: Index of vector to be freed
4402  *
4403  * This function frees the memory allocated to the q_vector.  In addition if
4404  * NAPI is enabled it will delete any references to the NAPI struct prior
4405  * to freeing the q_vector.
4406  **/
4407 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4408 {
4409         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4410         struct i40e_ring *ring;
4411
4412         if (!q_vector)
4413                 return;
4414
4415         /* disassociate q_vector from rings */
4416         i40e_for_each_ring(ring, q_vector->tx)
4417                 ring->q_vector = NULL;
4418
4419         i40e_for_each_ring(ring, q_vector->rx)
4420                 ring->q_vector = NULL;
4421
4422         /* only VSI w/ an associated netdev is set up w/ NAPI */
4423         if (vsi->netdev)
4424                 netif_napi_del(&q_vector->napi);
4425
4426         vsi->q_vectors[v_idx] = NULL;
4427
4428         kfree_rcu(q_vector, rcu);
4429 }
4430
4431 /**
4432  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4433  * @vsi: the VSI being un-configured
4434  *
4435  * This frees the memory allocated to the q_vectors and
4436  * deletes references to the NAPI struct.
4437  **/
4438 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4439 {
4440         int v_idx;
4441
4442         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4443                 i40e_free_q_vector(vsi, v_idx);
4444 }
4445
4446 /**
4447  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4448  * @pf: board private structure
4449  **/
4450 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4451 {
4452         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4453         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4454                 pci_disable_msix(pf->pdev);
4455                 kfree(pf->msix_entries);
4456                 pf->msix_entries = NULL;
4457                 kfree(pf->irq_pile);
4458                 pf->irq_pile = NULL;
4459         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4460                 pci_disable_msi(pf->pdev);
4461         }
4462         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4463 }
4464
4465 /**
4466  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4467  * @pf: board private structure
4468  *
4469  * We go through and clear interrupt specific resources and reset the structure
4470  * to pre-load conditions
4471  **/
4472 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4473 {
4474         int i;
4475
4476         i40e_free_misc_vector(pf);
4477
4478         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4479                       I40E_IWARP_IRQ_PILE_ID);
4480
4481         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4482         for (i = 0; i < pf->num_alloc_vsi; i++)
4483                 if (pf->vsi[i])
4484                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4485         i40e_reset_interrupt_capability(pf);
4486 }
4487
4488 /**
4489  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4490  * @vsi: the VSI being configured
4491  **/
4492 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4493 {
4494         int q_idx;
4495
4496         if (!vsi->netdev)
4497                 return;
4498
4499         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4500                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4501
4502                 if (q_vector->rx.ring || q_vector->tx.ring)
4503                         napi_enable(&q_vector->napi);
4504         }
4505 }
4506
4507 /**
4508  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4509  * @vsi: the VSI being configured
4510  **/
4511 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4512 {
4513         int q_idx;
4514
4515         if (!vsi->netdev)
4516                 return;
4517
4518         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4519                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4520
4521                 if (q_vector->rx.ring || q_vector->tx.ring)
4522                         napi_disable(&q_vector->napi);
4523         }
4524 }
4525
4526 /**
4527  * i40e_vsi_close - Shut down a VSI
4528  * @vsi: the vsi to be quelled
4529  **/
4530 static void i40e_vsi_close(struct i40e_vsi *vsi)
4531 {
4532         struct i40e_pf *pf = vsi->back;
4533         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4534                 i40e_down(vsi);
4535         i40e_vsi_free_irq(vsi);
4536         i40e_vsi_free_tx_resources(vsi);
4537         i40e_vsi_free_rx_resources(vsi);
4538         vsi->current_netdev_flags = 0;
4539         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
4540         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4541                 pf->flags |=  I40E_FLAG_CLIENT_RESET;
4542 }
4543
4544 /**
4545  * i40e_quiesce_vsi - Pause a given VSI
4546  * @vsi: the VSI being paused
4547  **/
4548 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4549 {
4550         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4551                 return;
4552
4553         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4554         if (vsi->netdev && netif_running(vsi->netdev))
4555                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4556         else
4557                 i40e_vsi_close(vsi);
4558 }
4559
4560 /**
4561  * i40e_unquiesce_vsi - Resume a given VSI
4562  * @vsi: the VSI being resumed
4563  **/
4564 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4565 {
4566         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
4567                 return;
4568
4569         if (vsi->netdev && netif_running(vsi->netdev))
4570                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4571         else
4572                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4573 }
4574
4575 /**
4576  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4577  * @pf: the PF
4578  **/
4579 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4580 {
4581         int v;
4582
4583         for (v = 0; v < pf->num_alloc_vsi; v++) {
4584                 if (pf->vsi[v])
4585                         i40e_quiesce_vsi(pf->vsi[v]);
4586         }
4587 }
4588
4589 /**
4590  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4591  * @pf: the PF
4592  **/
4593 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4594 {
4595         int v;
4596
4597         for (v = 0; v < pf->num_alloc_vsi; v++) {
4598                 if (pf->vsi[v])
4599                         i40e_unquiesce_vsi(pf->vsi[v]);
4600         }
4601 }
4602
4603 /**
4604  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4605  * @vsi: the VSI being configured
4606  *
4607  * Wait until all queues on a given VSI have been disabled.
4608  **/
4609 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4610 {
4611         struct i40e_pf *pf = vsi->back;
4612         int i, pf_q, ret;
4613
4614         pf_q = vsi->base_queue;
4615         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4616                 /* Check and wait for the Tx queue */
4617                 ret = i40e_pf_txq_wait(pf, pf_q, false);
4618                 if (ret) {
4619                         dev_info(&pf->pdev->dev,
4620                                  "VSI seid %d Tx ring %d disable timeout\n",
4621                                  vsi->seid, pf_q);
4622                         return ret;
4623                 }
4624
4625                 if (!i40e_enabled_xdp_vsi(vsi))
4626                         goto wait_rx;
4627
4628                 /* Check and wait for the XDP Tx queue */
4629                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
4630                                        false);
4631                 if (ret) {
4632                         dev_info(&pf->pdev->dev,
4633                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
4634                                  vsi->seid, pf_q);
4635                         return ret;
4636                 }
4637 wait_rx:
4638                 /* Check and wait for the Rx queue */
4639                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4640                 if (ret) {
4641                         dev_info(&pf->pdev->dev,
4642                                  "VSI seid %d Rx ring %d disable timeout\n",
4643                                  vsi->seid, pf_q);
4644                         return ret;
4645                 }
4646         }
4647
4648         return 0;
4649 }
4650
4651 #ifdef CONFIG_I40E_DCB
4652 /**
4653  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
4654  * @pf: the PF
4655  *
4656  * This function waits for the queues to be in disabled state for all the
4657  * VSIs that are managed by this PF.
4658  **/
4659 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4660 {
4661         int v, ret = 0;
4662
4663         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4664                 if (pf->vsi[v]) {
4665                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
4666                         if (ret)
4667                                 break;
4668                 }
4669         }
4670
4671         return ret;
4672 }
4673
4674 #endif
4675
4676 /**
4677  * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue
4678  * @q_idx: TX queue number
4679  * @vsi: Pointer to VSI struct
4680  *
4681  * This function checks specified queue for given VSI. Detects hung condition.
4682  * We proactively detect hung TX queues by checking if interrupts are disabled
4683  * but there are pending descriptors.  If it appears hung, attempt to recover
4684  * by triggering a SW interrupt.
4685  **/
4686 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi)
4687 {
4688         struct i40e_ring *tx_ring = NULL;
4689         struct i40e_pf  *pf;
4690         u32 val, tx_pending;
4691         int i;
4692
4693         pf = vsi->back;
4694
4695         /* now that we have an index, find the tx_ring struct */
4696         for (i = 0; i < vsi->num_queue_pairs; i++) {
4697                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
4698                         if (q_idx == vsi->tx_rings[i]->queue_index) {
4699                                 tx_ring = vsi->tx_rings[i];
4700                                 break;
4701                         }
4702                 }
4703         }
4704
4705         if (!tx_ring)
4706                 return;
4707
4708         /* Read interrupt register */
4709         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4710                 val = rd32(&pf->hw,
4711                            I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
4712                                                tx_ring->vsi->base_vector - 1));
4713         else
4714                 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
4715
4716         tx_pending = i40e_get_tx_pending(tx_ring);
4717
4718         /* Interrupts are disabled and TX pending is non-zero,
4719          * trigger the SW interrupt (don't wait). Worst case
4720          * there will be one extra interrupt which may result
4721          * into not cleaning any queues because queues are cleaned.
4722          */
4723         if (tx_pending && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK)))
4724                 i40e_force_wb(vsi, tx_ring->q_vector);
4725 }
4726
4727 /**
4728  * i40e_detect_recover_hung - Function to detect and recover hung_queues
4729  * @pf:  pointer to PF struct
4730  *
4731  * LAN VSI has netdev and netdev has TX queues. This function is to check
4732  * each of those TX queues if they are hung, trigger recovery by issuing
4733  * SW interrupt.
4734  **/
4735 static void i40e_detect_recover_hung(struct i40e_pf *pf)
4736 {
4737         struct net_device *netdev;
4738         struct i40e_vsi *vsi;
4739         unsigned int i;
4740
4741         /* Only for LAN VSI */
4742         vsi = pf->vsi[pf->lan_vsi];
4743
4744         if (!vsi)
4745                 return;
4746
4747         /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */
4748         if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
4749             test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
4750                 return;
4751
4752         /* Make sure type is MAIN VSI */
4753         if (vsi->type != I40E_VSI_MAIN)
4754                 return;
4755
4756         netdev = vsi->netdev;
4757         if (!netdev)
4758                 return;
4759
4760         /* Bail out if netif_carrier is not OK */
4761         if (!netif_carrier_ok(netdev))
4762                 return;
4763
4764         /* Go thru' TX queues for netdev */
4765         for (i = 0; i < netdev->num_tx_queues; i++) {
4766                 struct netdev_queue *q;
4767
4768                 q = netdev_get_tx_queue(netdev, i);
4769                 if (q)
4770                         i40e_detect_recover_hung_queue(i, vsi);
4771         }
4772 }
4773
4774 /**
4775  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
4776  * @pf: pointer to PF
4777  *
4778  * Get TC map for ISCSI PF type that will include iSCSI TC
4779  * and LAN TC.
4780  **/
4781 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
4782 {
4783         struct i40e_dcb_app_priority_table app;
4784         struct i40e_hw *hw = &pf->hw;
4785         u8 enabled_tc = 1; /* TC0 is always enabled */
4786         u8 tc, i;
4787         /* Get the iSCSI APP TLV */
4788         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4789
4790         for (i = 0; i < dcbcfg->numapps; i++) {
4791                 app = dcbcfg->app[i];
4792                 if (app.selector == I40E_APP_SEL_TCPIP &&
4793                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
4794                         tc = dcbcfg->etscfg.prioritytable[app.priority];
4795                         enabled_tc |= BIT(tc);
4796                         break;
4797                 }
4798         }
4799
4800         return enabled_tc;
4801 }
4802
4803 /**
4804  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
4805  * @dcbcfg: the corresponding DCBx configuration structure
4806  *
4807  * Return the number of TCs from given DCBx configuration
4808  **/
4809 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
4810 {
4811         int i, tc_unused = 0;
4812         u8 num_tc = 0;
4813         u8 ret = 0;
4814
4815         /* Scan the ETS Config Priority Table to find
4816          * traffic class enabled for a given priority
4817          * and create a bitmask of enabled TCs
4818          */
4819         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
4820                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
4821
4822         /* Now scan the bitmask to check for
4823          * contiguous TCs starting with TC0
4824          */
4825         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4826                 if (num_tc & BIT(i)) {
4827                         if (!tc_unused) {
4828                                 ret++;
4829                         } else {
4830                                 pr_err("Non-contiguous TC - Disabling DCB\n");
4831                                 return 1;
4832                         }
4833                 } else {
4834                         tc_unused = 1;
4835                 }
4836         }
4837
4838         /* There is always at least TC0 */
4839         if (!ret)
4840                 ret = 1;
4841
4842         return ret;
4843 }
4844
4845 /**
4846  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
4847  * @dcbcfg: the corresponding DCBx configuration structure
4848  *
4849  * Query the current DCB configuration and return the number of
4850  * traffic classes enabled from the given DCBX config
4851  **/
4852 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
4853 {
4854         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
4855         u8 enabled_tc = 1;
4856         u8 i;
4857
4858         for (i = 0; i < num_tc; i++)
4859                 enabled_tc |= BIT(i);
4860
4861         return enabled_tc;
4862 }
4863
4864 /**
4865  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
4866  * @pf: PF being queried
4867  *
4868  * Return number of traffic classes enabled for the given PF
4869  **/
4870 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
4871 {
4872         struct i40e_hw *hw = &pf->hw;
4873         u8 i, enabled_tc = 1;
4874         u8 num_tc = 0;
4875         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4876
4877         /* If DCB is not enabled then always in single TC */
4878         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4879                 return 1;
4880
4881         /* SFP mode will be enabled for all TCs on port */
4882         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4883                 return i40e_dcb_get_num_tc(dcbcfg);
4884
4885         /* MFP mode return count of enabled TCs for this PF */
4886         if (pf->hw.func_caps.iscsi)
4887                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
4888         else
4889                 return 1; /* Only TC0 */
4890
4891         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4892                 if (enabled_tc & BIT(i))
4893                         num_tc++;
4894         }
4895         return num_tc;
4896 }
4897
4898 /**
4899  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
4900  * @pf: PF being queried
4901  *
4902  * Return a bitmap for enabled traffic classes for this PF.
4903  **/
4904 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
4905 {
4906         /* If DCB is not enabled for this PF then just return default TC */
4907         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4908                 return I40E_DEFAULT_TRAFFIC_CLASS;
4909
4910         /* SFP mode we want PF to be enabled for all TCs */
4911         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4912                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
4913
4914         /* MFP enabled and iSCSI PF type */
4915         if (pf->hw.func_caps.iscsi)
4916                 return i40e_get_iscsi_tc_map(pf);
4917         else
4918                 return I40E_DEFAULT_TRAFFIC_CLASS;
4919 }
4920
4921 /**
4922  * i40e_vsi_get_bw_info - Query VSI BW Information
4923  * @vsi: the VSI being queried
4924  *
4925  * Returns 0 on success, negative value on failure
4926  **/
4927 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
4928 {
4929         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
4930         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
4931         struct i40e_pf *pf = vsi->back;
4932         struct i40e_hw *hw = &pf->hw;
4933         i40e_status ret;
4934         u32 tc_bw_max;
4935         int i;
4936
4937         /* Get the VSI level BW configuration */
4938         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4939         if (ret) {
4940                 dev_info(&pf->pdev->dev,
4941                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
4942                          i40e_stat_str(&pf->hw, ret),
4943                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4944                 return -EINVAL;
4945         }
4946
4947         /* Get the VSI level BW configuration per TC */
4948         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
4949                                                NULL);
4950         if (ret) {
4951                 dev_info(&pf->pdev->dev,
4952                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
4953                          i40e_stat_str(&pf->hw, ret),
4954                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4955                 return -EINVAL;
4956         }
4957
4958         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
4959                 dev_info(&pf->pdev->dev,
4960                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
4961                          bw_config.tc_valid_bits,
4962                          bw_ets_config.tc_valid_bits);
4963                 /* Still continuing */
4964         }
4965
4966         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
4967         vsi->bw_max_quanta = bw_config.max_bw;
4968         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
4969                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
4970         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4971                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
4972                 vsi->bw_ets_limit_credits[i] =
4973                                         le16_to_cpu(bw_ets_config.credits[i]);
4974                 /* 3 bits out of 4 for each TC */
4975                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
4976         }
4977
4978         return 0;
4979 }
4980
4981 /**
4982  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
4983  * @vsi: the VSI being configured
4984  * @enabled_tc: TC bitmap
4985  * @bw_credits: BW shared credits per TC
4986  *
4987  * Returns 0 on success, negative value on failure
4988  **/
4989 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
4990                                        u8 *bw_share)
4991 {
4992         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
4993         i40e_status ret;
4994         int i;
4995
4996         bw_data.tc_valid_bits = enabled_tc;
4997         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4998                 bw_data.tc_bw_credits[i] = bw_share[i];
4999
5000         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
5001                                        NULL);
5002         if (ret) {
5003                 dev_info(&vsi->back->pdev->dev,
5004                          "AQ command Config VSI BW allocation per TC failed = %d\n",
5005                          vsi->back->hw.aq.asq_last_status);
5006                 return -EINVAL;
5007         }
5008
5009         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5010                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5011
5012         return 0;
5013 }
5014
5015 /**
5016  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5017  * @vsi: the VSI being configured
5018  * @enabled_tc: TC map to be enabled
5019  *
5020  **/
5021 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5022 {
5023         struct net_device *netdev = vsi->netdev;
5024         struct i40e_pf *pf = vsi->back;
5025         struct i40e_hw *hw = &pf->hw;
5026         u8 netdev_tc = 0;
5027         int i;
5028         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5029
5030         if (!netdev)
5031                 return;
5032
5033         if (!enabled_tc) {
5034                 netdev_reset_tc(netdev);
5035                 return;
5036         }
5037
5038         /* Set up actual enabled TCs on the VSI */
5039         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5040                 return;
5041
5042         /* set per TC queues for the VSI */
5043         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5044                 /* Only set TC queues for enabled tcs
5045                  *
5046                  * e.g. For a VSI that has TC0 and TC3 enabled the
5047                  * enabled_tc bitmap would be 0x00001001; the driver
5048                  * will set the numtc for netdev as 2 that will be
5049                  * referenced by the netdev layer as TC 0 and 1.
5050                  */
5051                 if (vsi->tc_config.enabled_tc & BIT(i))
5052                         netdev_set_tc_queue(netdev,
5053                                         vsi->tc_config.tc_info[i].netdev_tc,
5054                                         vsi->tc_config.tc_info[i].qcount,
5055                                         vsi->tc_config.tc_info[i].qoffset);
5056         }
5057
5058         /* Assign UP2TC map for the VSI */
5059         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5060                 /* Get the actual TC# for the UP */
5061                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5062                 /* Get the mapped netdev TC# for the UP */
5063                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5064                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5065         }
5066 }
5067
5068 /**
5069  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5070  * @vsi: the VSI being configured
5071  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5072  **/
5073 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5074                                       struct i40e_vsi_context *ctxt)
5075 {
5076         /* copy just the sections touched not the entire info
5077          * since not all sections are valid as returned by
5078          * update vsi params
5079          */
5080         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5081         memcpy(&vsi->info.queue_mapping,
5082                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5083         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5084                sizeof(vsi->info.tc_mapping));
5085 }
5086
5087 /**
5088  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5089  * @vsi: VSI to be configured
5090  * @enabled_tc: TC bitmap
5091  *
5092  * This configures a particular VSI for TCs that are mapped to the
5093  * given TC bitmap. It uses default bandwidth share for TCs across
5094  * VSIs to configure TC for a particular VSI.
5095  *
5096  * NOTE:
5097  * It is expected that the VSI queues have been quisced before calling
5098  * this function.
5099  **/
5100 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5101 {
5102         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5103         struct i40e_vsi_context ctxt;
5104         int ret = 0;
5105         int i;
5106
5107         /* Check if enabled_tc is same as existing or new TCs */
5108         if (vsi->tc_config.enabled_tc == enabled_tc)
5109                 return ret;
5110
5111         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5112         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5113                 if (enabled_tc & BIT(i))
5114                         bw_share[i] = 1;
5115         }
5116
5117         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5118         if (ret) {
5119                 dev_info(&vsi->back->pdev->dev,
5120                          "Failed configuring TC map %d for VSI %d\n",
5121                          enabled_tc, vsi->seid);
5122                 goto out;
5123         }
5124
5125         /* Update Queue Pairs Mapping for currently enabled UPs */
5126         ctxt.seid = vsi->seid;
5127         ctxt.pf_num = vsi->back->hw.pf_id;
5128         ctxt.vf_num = 0;
5129         ctxt.uplink_seid = vsi->uplink_seid;
5130         ctxt.info = vsi->info;
5131         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5132
5133         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5134                 ctxt.info.valid_sections |=
5135                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5136                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5137         }
5138
5139         /* Update the VSI after updating the VSI queue-mapping information */
5140         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
5141         if (ret) {
5142                 dev_info(&vsi->back->pdev->dev,
5143                          "Update vsi tc config failed, err %s aq_err %s\n",
5144                          i40e_stat_str(&vsi->back->hw, ret),
5145                          i40e_aq_str(&vsi->back->hw,
5146                                      vsi->back->hw.aq.asq_last_status));
5147                 goto out;
5148         }
5149         /* update the local VSI info with updated queue map */
5150         i40e_vsi_update_queue_map(vsi, &ctxt);
5151         vsi->info.valid_sections = 0;
5152
5153         /* Update current VSI BW information */
5154         ret = i40e_vsi_get_bw_info(vsi);
5155         if (ret) {
5156                 dev_info(&vsi->back->pdev->dev,
5157                          "Failed updating vsi bw info, err %s aq_err %s\n",
5158                          i40e_stat_str(&vsi->back->hw, ret),
5159                          i40e_aq_str(&vsi->back->hw,
5160                                      vsi->back->hw.aq.asq_last_status));
5161                 goto out;
5162         }
5163
5164         /* Update the netdev TC setup */
5165         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5166 out:
5167         return ret;
5168 }
5169
5170 /**
5171  * i40e_veb_config_tc - Configure TCs for given VEB
5172  * @veb: given VEB
5173  * @enabled_tc: TC bitmap
5174  *
5175  * Configures given TC bitmap for VEB (switching) element
5176  **/
5177 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
5178 {
5179         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
5180         struct i40e_pf *pf = veb->pf;
5181         int ret = 0;
5182         int i;
5183
5184         /* No TCs or already enabled TCs just return */
5185         if (!enabled_tc || veb->enabled_tc == enabled_tc)
5186                 return ret;
5187
5188         bw_data.tc_valid_bits = enabled_tc;
5189         /* bw_data.absolute_credits is not set (relative) */
5190
5191         /* Enable ETS TCs with equal BW Share for now */
5192         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5193                 if (enabled_tc & BIT(i))
5194                         bw_data.tc_bw_share_credits[i] = 1;
5195         }
5196
5197         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
5198                                                    &bw_data, NULL);
5199         if (ret) {
5200                 dev_info(&pf->pdev->dev,
5201                          "VEB bw config failed, err %s aq_err %s\n",
5202                          i40e_stat_str(&pf->hw, ret),
5203                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5204                 goto out;
5205         }
5206
5207         /* Update the BW information */
5208         ret = i40e_veb_get_bw_info(veb);
5209         if (ret) {
5210                 dev_info(&pf->pdev->dev,
5211                          "Failed getting veb bw config, err %s aq_err %s\n",
5212                          i40e_stat_str(&pf->hw, ret),
5213                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5214         }
5215
5216 out:
5217         return ret;
5218 }
5219
5220 #ifdef CONFIG_I40E_DCB
5221 /**
5222  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
5223  * @pf: PF struct
5224  *
5225  * Reconfigure VEB/VSIs on a given PF; it is assumed that
5226  * the caller would've quiesce all the VSIs before calling
5227  * this function
5228  **/
5229 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
5230 {
5231         u8 tc_map = 0;
5232         int ret;
5233         u8 v;
5234
5235         /* Enable the TCs available on PF to all VEBs */
5236         tc_map = i40e_pf_get_tc_map(pf);
5237         for (v = 0; v < I40E_MAX_VEB; v++) {
5238                 if (!pf->veb[v])
5239                         continue;
5240                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
5241                 if (ret) {
5242                         dev_info(&pf->pdev->dev,
5243                                  "Failed configuring TC for VEB seid=%d\n",
5244                                  pf->veb[v]->seid);
5245                         /* Will try to configure as many components */
5246                 }
5247         }
5248
5249         /* Update each VSI */
5250         for (v = 0; v < pf->num_alloc_vsi; v++) {
5251                 if (!pf->vsi[v])
5252                         continue;
5253
5254                 /* - Enable all TCs for the LAN VSI
5255                  * - For all others keep them at TC0 for now
5256                  */
5257                 if (v == pf->lan_vsi)
5258                         tc_map = i40e_pf_get_tc_map(pf);
5259                 else
5260                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
5261
5262                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
5263                 if (ret) {
5264                         dev_info(&pf->pdev->dev,
5265                                  "Failed configuring TC for VSI seid=%d\n",
5266                                  pf->vsi[v]->seid);
5267                         /* Will try to configure as many components */
5268                 } else {
5269                         /* Re-configure VSI vectors based on updated TC map */
5270                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
5271                         if (pf->vsi[v]->netdev)
5272                                 i40e_dcbnl_set_all(pf->vsi[v]);
5273                 }
5274         }
5275 }
5276
5277 /**
5278  * i40e_resume_port_tx - Resume port Tx
5279  * @pf: PF struct
5280  *
5281  * Resume a port's Tx and issue a PF reset in case of failure to
5282  * resume.
5283  **/
5284 static int i40e_resume_port_tx(struct i40e_pf *pf)
5285 {
5286         struct i40e_hw *hw = &pf->hw;
5287         int ret;
5288
5289         ret = i40e_aq_resume_port_tx(hw, NULL);
5290         if (ret) {
5291                 dev_info(&pf->pdev->dev,
5292                          "Resume Port Tx failed, err %s aq_err %s\n",
5293                           i40e_stat_str(&pf->hw, ret),
5294                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5295                 /* Schedule PF reset to recover */
5296                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
5297                 i40e_service_event_schedule(pf);
5298         }
5299
5300         return ret;
5301 }
5302
5303 /**
5304  * i40e_init_pf_dcb - Initialize DCB configuration
5305  * @pf: PF being configured
5306  *
5307  * Query the current DCB configuration and cache it
5308  * in the hardware structure
5309  **/
5310 static int i40e_init_pf_dcb(struct i40e_pf *pf)
5311 {
5312         struct i40e_hw *hw = &pf->hw;
5313         int err = 0;
5314
5315         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */
5316         if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT)
5317                 goto out;
5318
5319         /* Get the initial DCB configuration */
5320         err = i40e_init_dcb(hw);
5321         if (!err) {
5322                 /* Device/Function is not DCBX capable */
5323                 if ((!hw->func_caps.dcb) ||
5324                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
5325                         dev_info(&pf->pdev->dev,
5326                                  "DCBX offload is not supported or is disabled for this PF.\n");
5327                 } else {
5328                         /* When status is not DISABLED then DCBX in FW */
5329                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
5330                                        DCB_CAP_DCBX_VER_IEEE;
5331
5332                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
5333                         /* Enable DCB tagging only when more than one TC
5334                          * or explicitly disable if only one TC
5335                          */
5336                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
5337                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
5338                         else
5339                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
5340                         dev_dbg(&pf->pdev->dev,
5341                                 "DCBX offload is supported for this PF.\n");
5342                 }
5343         } else {
5344                 dev_info(&pf->pdev->dev,
5345                          "Query for DCB configuration failed, err %s aq_err %s\n",
5346                          i40e_stat_str(&pf->hw, err),
5347                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5348         }
5349
5350 out:
5351         return err;
5352 }
5353 #endif /* CONFIG_I40E_DCB */
5354 #define SPEED_SIZE 14
5355 #define FC_SIZE 8
5356 /**
5357  * i40e_print_link_message - print link up or down
5358  * @vsi: the VSI for which link needs a message
5359  */
5360 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
5361 {
5362         enum i40e_aq_link_speed new_speed;
5363         struct i40e_pf *pf = vsi->back;
5364         char *speed = "Unknown";
5365         char *fc = "Unknown";
5366         char *fec = "";
5367         char *req_fec = "";
5368         char *an = "";
5369
5370         new_speed = pf->hw.phy.link_info.link_speed;
5371
5372         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
5373                 return;
5374         vsi->current_isup = isup;
5375         vsi->current_speed = new_speed;
5376         if (!isup) {
5377                 netdev_info(vsi->netdev, "NIC Link is Down\n");
5378                 return;
5379         }
5380
5381         /* Warn user if link speed on NPAR enabled partition is not at
5382          * least 10GB
5383          */
5384         if (pf->hw.func_caps.npar_enable &&
5385             (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
5386              pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
5387                 netdev_warn(vsi->netdev,
5388                             "The partition detected link speed that is less than 10Gbps\n");
5389
5390         switch (pf->hw.phy.link_info.link_speed) {
5391         case I40E_LINK_SPEED_40GB:
5392                 speed = "40 G";
5393                 break;
5394         case I40E_LINK_SPEED_20GB:
5395                 speed = "20 G";
5396                 break;
5397         case I40E_LINK_SPEED_25GB:
5398                 speed = "25 G";
5399                 break;
5400         case I40E_LINK_SPEED_10GB:
5401                 speed = "10 G";
5402                 break;
5403         case I40E_LINK_SPEED_1GB:
5404                 speed = "1000 M";
5405                 break;
5406         case I40E_LINK_SPEED_100MB:
5407                 speed = "100 M";
5408                 break;
5409         default:
5410                 break;
5411         }
5412
5413         switch (pf->hw.fc.current_mode) {
5414         case I40E_FC_FULL:
5415                 fc = "RX/TX";
5416                 break;
5417         case I40E_FC_TX_PAUSE:
5418                 fc = "TX";
5419                 break;
5420         case I40E_FC_RX_PAUSE:
5421                 fc = "RX";
5422                 break;
5423         default:
5424                 fc = "None";
5425                 break;
5426         }
5427
5428         if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
5429                 req_fec = ", Requested FEC: None";
5430                 fec = ", FEC: None";
5431                 an = ", Autoneg: False";
5432
5433                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
5434                         an = ", Autoneg: True";
5435
5436                 if (pf->hw.phy.link_info.fec_info &
5437                     I40E_AQ_CONFIG_FEC_KR_ENA)
5438                         fec = ", FEC: CL74 FC-FEC/BASE-R";
5439                 else if (pf->hw.phy.link_info.fec_info &
5440                          I40E_AQ_CONFIG_FEC_RS_ENA)
5441                         fec = ", FEC: CL108 RS-FEC";
5442
5443                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
5444                  * both RS and FC are requested
5445                  */
5446                 if (vsi->back->hw.phy.link_info.req_fec_info &
5447                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
5448                         if (vsi->back->hw.phy.link_info.req_fec_info &
5449                             I40E_AQ_REQUEST_FEC_RS)
5450                                 req_fec = ", Requested FEC: CL108 RS-FEC";
5451                         else
5452                                 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R";
5453                 }
5454         }
5455
5456         netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n",
5457                     speed, req_fec, fec, an, fc);
5458 }
5459
5460 /**
5461  * i40e_up_complete - Finish the last steps of bringing up a connection
5462  * @vsi: the VSI being configured
5463  **/
5464 static int i40e_up_complete(struct i40e_vsi *vsi)
5465 {
5466         struct i40e_pf *pf = vsi->back;
5467         int err;
5468
5469         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5470                 i40e_vsi_configure_msix(vsi);
5471         else
5472                 i40e_configure_msi_and_legacy(vsi);
5473
5474         /* start rings */
5475         err = i40e_vsi_start_rings(vsi);
5476         if (err)
5477                 return err;
5478
5479         clear_bit(__I40E_VSI_DOWN, vsi->state);
5480         i40e_napi_enable_all(vsi);
5481         i40e_vsi_enable_irq(vsi);
5482
5483         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
5484             (vsi->netdev)) {
5485                 i40e_print_link_message(vsi, true);
5486                 netif_tx_start_all_queues(vsi->netdev);
5487                 netif_carrier_on(vsi->netdev);
5488         }
5489
5490         /* replay FDIR SB filters */
5491         if (vsi->type == I40E_VSI_FDIR) {
5492                 /* reset fd counters */
5493                 pf->fd_add_err = 0;
5494                 pf->fd_atr_cnt = 0;
5495                 i40e_fdir_filter_restore(vsi);
5496         }
5497
5498         /* On the next run of the service_task, notify any clients of the new
5499          * opened netdev
5500          */
5501         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
5502         i40e_service_event_schedule(pf);
5503
5504         return 0;
5505 }
5506
5507 /**
5508  * i40e_vsi_reinit_locked - Reset the VSI
5509  * @vsi: the VSI being configured
5510  *
5511  * Rebuild the ring structs after some configuration
5512  * has changed, e.g. MTU size.
5513  **/
5514 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
5515 {
5516         struct i40e_pf *pf = vsi->back;
5517
5518         WARN_ON(in_interrupt());
5519         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
5520                 usleep_range(1000, 2000);
5521         i40e_down(vsi);
5522
5523         i40e_up(vsi);
5524         clear_bit(__I40E_CONFIG_BUSY, pf->state);
5525 }
5526
5527 /**
5528  * i40e_up - Bring the connection back up after being down
5529  * @vsi: the VSI being configured
5530  **/
5531 int i40e_up(struct i40e_vsi *vsi)
5532 {
5533         int err;
5534
5535         err = i40e_vsi_configure(vsi);
5536         if (!err)
5537                 err = i40e_up_complete(vsi);
5538
5539         return err;
5540 }
5541
5542 /**
5543  * i40e_down - Shutdown the connection processing
5544  * @vsi: the VSI being stopped
5545  **/
5546 void i40e_down(struct i40e_vsi *vsi)
5547 {
5548         int i;
5549
5550         /* It is assumed that the caller of this function
5551          * sets the vsi->state __I40E_VSI_DOWN bit.
5552          */
5553         if (vsi->netdev) {
5554                 netif_carrier_off(vsi->netdev);
5555                 netif_tx_disable(vsi->netdev);
5556         }
5557         i40e_vsi_disable_irq(vsi);
5558         i40e_vsi_stop_rings(vsi);
5559         i40e_napi_disable_all(vsi);
5560
5561         for (i = 0; i < vsi->num_queue_pairs; i++) {
5562                 i40e_clean_tx_ring(vsi->tx_rings[i]);
5563                 if (i40e_enabled_xdp_vsi(vsi))
5564                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
5565                 i40e_clean_rx_ring(vsi->rx_rings[i]);
5566         }
5567
5568 }
5569
5570 /**
5571  * i40e_setup_tc - configure multiple traffic classes
5572  * @netdev: net device to configure
5573  * @tc: number of traffic classes to enable
5574  **/
5575 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
5576 {
5577         struct i40e_netdev_priv *np = netdev_priv(netdev);
5578         struct i40e_vsi *vsi = np->vsi;
5579         struct i40e_pf *pf = vsi->back;
5580         u8 enabled_tc = 0;
5581         int ret = -EINVAL;
5582         int i;
5583
5584         /* Check if DCB enabled to continue */
5585         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5586                 netdev_info(netdev, "DCB is not enabled for adapter\n");
5587                 goto exit;
5588         }
5589
5590         /* Check if MFP enabled */
5591         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
5592                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
5593                 goto exit;
5594         }
5595
5596         /* Check whether tc count is within enabled limit */
5597         if (tc > i40e_pf_get_num_tc(pf)) {
5598                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
5599                 goto exit;
5600         }
5601
5602         /* Generate TC map for number of tc requested */
5603         for (i = 0; i < tc; i++)
5604                 enabled_tc |= BIT(i);
5605
5606         /* Requesting same TC configuration as already enabled */
5607         if (enabled_tc == vsi->tc_config.enabled_tc)
5608                 return 0;
5609
5610         /* Quiesce VSI queues */
5611         i40e_quiesce_vsi(vsi);
5612
5613         /* Configure VSI for enabled TCs */
5614         ret = i40e_vsi_config_tc(vsi, enabled_tc);
5615         if (ret) {
5616                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
5617                             vsi->seid);
5618                 goto exit;
5619         }
5620
5621         /* Unquiesce VSI */
5622         i40e_unquiesce_vsi(vsi);
5623
5624 exit:
5625         return ret;
5626 }
5627
5628 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
5629                            void *type_data)
5630 {
5631         struct tc_mqprio_qopt *mqprio = type_data;
5632
5633         if (type != TC_SETUP_MQPRIO)
5634                 return -EOPNOTSUPP;
5635
5636         mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
5637
5638         return i40e_setup_tc(netdev, mqprio->num_tc);
5639 }
5640
5641 /**
5642  * i40e_open - Called when a network interface is made active
5643  * @netdev: network interface device structure
5644  *
5645  * The open entry point is called when a network interface is made
5646  * active by the system (IFF_UP).  At this point all resources needed
5647  * for transmit and receive operations are allocated, the interrupt
5648  * handler is registered with the OS, the netdev watchdog subtask is
5649  * enabled, and the stack is notified that the interface is ready.
5650  *
5651  * Returns 0 on success, negative value on failure
5652  **/
5653 int i40e_open(struct net_device *netdev)
5654 {
5655         struct i40e_netdev_priv *np = netdev_priv(netdev);
5656         struct i40e_vsi *vsi = np->vsi;
5657         struct i40e_pf *pf = vsi->back;
5658         int err;
5659
5660         /* disallow open during test or if eeprom is broken */
5661         if (test_bit(__I40E_TESTING, pf->state) ||
5662             test_bit(__I40E_BAD_EEPROM, pf->state))
5663                 return -EBUSY;
5664
5665         netif_carrier_off(netdev);
5666
5667         err = i40e_vsi_open(vsi);
5668         if (err)
5669                 return err;
5670
5671         /* configure global TSO hardware offload settings */
5672         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
5673                                                        TCP_FLAG_FIN) >> 16);
5674         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
5675                                                        TCP_FLAG_FIN |
5676                                                        TCP_FLAG_CWR) >> 16);
5677         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
5678
5679         udp_tunnel_get_rx_info(netdev);
5680
5681         return 0;
5682 }
5683
5684 /**
5685  * i40e_vsi_open -
5686  * @vsi: the VSI to open
5687  *
5688  * Finish initialization of the VSI.
5689  *
5690  * Returns 0 on success, negative value on failure
5691  *
5692  * Note: expects to be called while under rtnl_lock()
5693  **/
5694 int i40e_vsi_open(struct i40e_vsi *vsi)
5695 {
5696         struct i40e_pf *pf = vsi->back;
5697         char int_name[I40E_INT_NAME_STR_LEN];
5698         int err;
5699
5700         /* allocate descriptors */
5701         err = i40e_vsi_setup_tx_resources(vsi);
5702         if (err)
5703                 goto err_setup_tx;
5704         err = i40e_vsi_setup_rx_resources(vsi);
5705         if (err)
5706                 goto err_setup_rx;
5707
5708         err = i40e_vsi_configure(vsi);
5709         if (err)
5710                 goto err_setup_rx;
5711
5712         if (vsi->netdev) {
5713                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
5714                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
5715                 err = i40e_vsi_request_irq(vsi, int_name);
5716                 if (err)
5717                         goto err_setup_rx;
5718
5719                 /* Notify the stack of the actual queue counts. */
5720                 err = netif_set_real_num_tx_queues(vsi->netdev,
5721                                                    vsi->num_queue_pairs);
5722                 if (err)
5723                         goto err_set_queues;
5724
5725                 err = netif_set_real_num_rx_queues(vsi->netdev,
5726                                                    vsi->num_queue_pairs);
5727                 if (err)
5728                         goto err_set_queues;
5729
5730         } else if (vsi->type == I40E_VSI_FDIR) {
5731                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
5732                          dev_driver_string(&pf->pdev->dev),
5733                          dev_name(&pf->pdev->dev));
5734                 err = i40e_vsi_request_irq(vsi, int_name);
5735
5736         } else {
5737                 err = -EINVAL;
5738                 goto err_setup_rx;
5739         }
5740
5741         err = i40e_up_complete(vsi);
5742         if (err)
5743                 goto err_up_complete;
5744
5745         return 0;
5746
5747 err_up_complete:
5748         i40e_down(vsi);
5749 err_set_queues:
5750         i40e_vsi_free_irq(vsi);
5751 err_setup_rx:
5752         i40e_vsi_free_rx_resources(vsi);
5753 err_setup_tx:
5754         i40e_vsi_free_tx_resources(vsi);
5755         if (vsi == pf->vsi[pf->lan_vsi])
5756                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true);
5757
5758         return err;
5759 }
5760
5761 /**
5762  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
5763  * @pf: Pointer to PF
5764  *
5765  * This function destroys the hlist where all the Flow Director
5766  * filters were saved.
5767  **/
5768 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
5769 {
5770         struct i40e_fdir_filter *filter;
5771         struct i40e_flex_pit *pit_entry, *tmp;
5772         struct hlist_node *node2;
5773
5774         hlist_for_each_entry_safe(filter, node2,
5775                                   &pf->fdir_filter_list, fdir_node) {
5776                 hlist_del(&filter->fdir_node);
5777                 kfree(filter);
5778         }
5779
5780         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
5781                 list_del(&pit_entry->list);
5782                 kfree(pit_entry);
5783         }
5784         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
5785
5786         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
5787                 list_del(&pit_entry->list);
5788                 kfree(pit_entry);
5789         }
5790         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
5791
5792         pf->fdir_pf_active_filters = 0;
5793         pf->fd_tcp4_filter_cnt = 0;
5794         pf->fd_udp4_filter_cnt = 0;
5795         pf->fd_sctp4_filter_cnt = 0;
5796         pf->fd_ip4_filter_cnt = 0;
5797
5798         /* Reprogram the default input set for TCP/IPv4 */
5799         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
5800                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5801                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5802
5803         /* Reprogram the default input set for UDP/IPv4 */
5804         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
5805                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5806                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5807
5808         /* Reprogram the default input set for SCTP/IPv4 */
5809         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
5810                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5811                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5812
5813         /* Reprogram the default input set for Other/IPv4 */
5814         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
5815                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
5816 }
5817
5818 /**
5819  * i40e_close - Disables a network interface
5820  * @netdev: network interface device structure
5821  *
5822  * The close entry point is called when an interface is de-activated
5823  * by the OS.  The hardware is still under the driver's control, but
5824  * this netdev interface is disabled.
5825  *
5826  * Returns 0, this is not allowed to fail
5827  **/
5828 int i40e_close(struct net_device *netdev)
5829 {
5830         struct i40e_netdev_priv *np = netdev_priv(netdev);
5831         struct i40e_vsi *vsi = np->vsi;
5832
5833         i40e_vsi_close(vsi);
5834
5835         return 0;
5836 }
5837
5838 /**
5839  * i40e_do_reset - Start a PF or Core Reset sequence
5840  * @pf: board private structure
5841  * @reset_flags: which reset is requested
5842  * @lock_acquired: indicates whether or not the lock has been acquired
5843  * before this function was called.
5844  *
5845  * The essential difference in resets is that the PF Reset
5846  * doesn't clear the packet buffers, doesn't reset the PE
5847  * firmware, and doesn't bother the other PFs on the chip.
5848  **/
5849 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
5850 {
5851         u32 val;
5852
5853         WARN_ON(in_interrupt());
5854
5855
5856         /* do the biggest reset indicated */
5857         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
5858
5859                 /* Request a Global Reset
5860                  *
5861                  * This will start the chip's countdown to the actual full
5862                  * chip reset event, and a warning interrupt to be sent
5863                  * to all PFs, including the requestor.  Our handler
5864                  * for the warning interrupt will deal with the shutdown
5865                  * and recovery of the switch setup.
5866                  */
5867                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
5868                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5869                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
5870                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5871
5872         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
5873
5874                 /* Request a Core Reset
5875                  *
5876                  * Same as Global Reset, except does *not* include the MAC/PHY
5877                  */
5878                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
5879                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5880                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
5881                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5882                 i40e_flush(&pf->hw);
5883
5884         } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) {
5885
5886                 /* Request a PF Reset
5887                  *
5888                  * Resets only the PF-specific registers
5889                  *
5890                  * This goes directly to the tear-down and rebuild of
5891                  * the switch, since we need to do all the recovery as
5892                  * for the Core Reset.
5893                  */
5894                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
5895                 i40e_handle_reset_warning(pf, lock_acquired);
5896
5897         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
5898                 int v;
5899
5900                 /* Find the VSI(s) that requested a re-init */
5901                 dev_info(&pf->pdev->dev,
5902                          "VSI reinit requested\n");
5903                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5904                         struct i40e_vsi *vsi = pf->vsi[v];
5905
5906                         if (vsi != NULL &&
5907                             test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
5908                                                vsi->state))
5909                                 i40e_vsi_reinit_locked(pf->vsi[v]);
5910                 }
5911         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
5912                 int v;
5913
5914                 /* Find the VSI(s) that needs to be brought down */
5915                 dev_info(&pf->pdev->dev, "VSI down requested\n");
5916                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5917                         struct i40e_vsi *vsi = pf->vsi[v];
5918
5919                         if (vsi != NULL &&
5920                             test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
5921                                                vsi->state)) {
5922                                 set_bit(__I40E_VSI_DOWN, vsi->state);
5923                                 i40e_down(vsi);
5924                         }
5925                 }
5926         } else {
5927                 dev_info(&pf->pdev->dev,
5928                          "bad reset request 0x%08x\n", reset_flags);
5929         }
5930 }
5931
5932 #ifdef CONFIG_I40E_DCB
5933 /**
5934  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
5935  * @pf: board private structure
5936  * @old_cfg: current DCB config
5937  * @new_cfg: new DCB config
5938  **/
5939 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
5940                             struct i40e_dcbx_config *old_cfg,
5941                             struct i40e_dcbx_config *new_cfg)
5942 {
5943         bool need_reconfig = false;
5944
5945         /* Check if ETS configuration has changed */
5946         if (memcmp(&new_cfg->etscfg,
5947                    &old_cfg->etscfg,
5948                    sizeof(new_cfg->etscfg))) {
5949                 /* If Priority Table has changed reconfig is needed */
5950                 if (memcmp(&new_cfg->etscfg.prioritytable,
5951                            &old_cfg->etscfg.prioritytable,
5952                            sizeof(new_cfg->etscfg.prioritytable))) {
5953                         need_reconfig = true;
5954                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
5955                 }
5956
5957                 if (memcmp(&new_cfg->etscfg.tcbwtable,
5958                            &old_cfg->etscfg.tcbwtable,
5959                            sizeof(new_cfg->etscfg.tcbwtable)))
5960                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
5961
5962                 if (memcmp(&new_cfg->etscfg.tsatable,
5963                            &old_cfg->etscfg.tsatable,
5964                            sizeof(new_cfg->etscfg.tsatable)))
5965                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
5966         }
5967
5968         /* Check if PFC configuration has changed */
5969         if (memcmp(&new_cfg->pfc,
5970                    &old_cfg->pfc,
5971                    sizeof(new_cfg->pfc))) {
5972                 need_reconfig = true;
5973                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
5974         }
5975
5976         /* Check if APP Table has changed */
5977         if (memcmp(&new_cfg->app,
5978                    &old_cfg->app,
5979                    sizeof(new_cfg->app))) {
5980                 need_reconfig = true;
5981                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
5982         }
5983
5984         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
5985         return need_reconfig;
5986 }
5987
5988 /**
5989  * i40e_handle_lldp_event - Handle LLDP Change MIB event
5990  * @pf: board private structure
5991  * @e: event info posted on ARQ
5992  **/
5993 static int i40e_handle_lldp_event(struct i40e_pf *pf,
5994                                   struct i40e_arq_event_info *e)
5995 {
5996         struct i40e_aqc_lldp_get_mib *mib =
5997                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
5998         struct i40e_hw *hw = &pf->hw;
5999         struct i40e_dcbx_config tmp_dcbx_cfg;
6000         bool need_reconfig = false;
6001         int ret = 0;
6002         u8 type;
6003
6004         /* Not DCB capable or capability disabled */
6005         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
6006                 return ret;
6007
6008         /* Ignore if event is not for Nearest Bridge */
6009         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
6010                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
6011         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
6012         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
6013                 return ret;
6014
6015         /* Check MIB Type and return if event for Remote MIB update */
6016         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
6017         dev_dbg(&pf->pdev->dev,
6018                 "LLDP event mib type %s\n", type ? "remote" : "local");
6019         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
6020                 /* Update the remote cached instance and return */
6021                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
6022                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
6023                                 &hw->remote_dcbx_config);
6024                 goto exit;
6025         }
6026
6027         /* Store the old configuration */
6028         tmp_dcbx_cfg = hw->local_dcbx_config;
6029
6030         /* Reset the old DCBx configuration data */
6031         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
6032         /* Get updated DCBX data from firmware */
6033         ret = i40e_get_dcb_config(&pf->hw);
6034         if (ret) {
6035                 dev_info(&pf->pdev->dev,
6036                          "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
6037                          i40e_stat_str(&pf->hw, ret),
6038                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6039                 goto exit;
6040         }
6041
6042         /* No change detected in DCBX configs */
6043         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
6044                     sizeof(tmp_dcbx_cfg))) {
6045                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
6046                 goto exit;
6047         }
6048
6049         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
6050                                                &hw->local_dcbx_config);
6051
6052         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
6053
6054         if (!need_reconfig)
6055                 goto exit;
6056
6057         /* Enable DCB tagging only when more than one TC */
6058         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6059                 pf->flags |= I40E_FLAG_DCB_ENABLED;
6060         else
6061                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6062
6063         set_bit(__I40E_PORT_SUSPENDED, pf->state);
6064         /* Reconfiguration needed quiesce all VSIs */
6065         i40e_pf_quiesce_all_vsi(pf);
6066
6067         /* Changes in configuration update VEB/VSI */
6068         i40e_dcb_reconfigure(pf);
6069
6070         ret = i40e_resume_port_tx(pf);
6071
6072         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
6073         /* In case of error no point in resuming VSIs */
6074         if (ret)
6075                 goto exit;
6076
6077         /* Wait for the PF's queues to be disabled */
6078         ret = i40e_pf_wait_queues_disabled(pf);
6079         if (ret) {
6080                 /* Schedule PF reset to recover */
6081                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6082                 i40e_service_event_schedule(pf);
6083         } else {
6084                 i40e_pf_unquiesce_all_vsi(pf);
6085         pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
6086                       I40E_FLAG_CLIENT_L2_CHANGE);
6087         }
6088
6089 exit:
6090         return ret;
6091 }
6092 #endif /* CONFIG_I40E_DCB */
6093
6094 /**
6095  * i40e_do_reset_safe - Protected reset path for userland calls.
6096  * @pf: board private structure
6097  * @reset_flags: which reset is requested
6098  *
6099  **/
6100 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
6101 {
6102         rtnl_lock();
6103         i40e_do_reset(pf, reset_flags, true);
6104         rtnl_unlock();
6105 }
6106
6107 /**
6108  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
6109  * @pf: board private structure
6110  * @e: event info posted on ARQ
6111  *
6112  * Handler for LAN Queue Overflow Event generated by the firmware for PF
6113  * and VF queues
6114  **/
6115 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
6116                                            struct i40e_arq_event_info *e)
6117 {
6118         struct i40e_aqc_lan_overflow *data =
6119                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
6120         u32 queue = le32_to_cpu(data->prtdcb_rupto);
6121         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
6122         struct i40e_hw *hw = &pf->hw;
6123         struct i40e_vf *vf;
6124         u16 vf_id;
6125
6126         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
6127                 queue, qtx_ctl);
6128
6129         /* Queue belongs to VF, find the VF and issue VF reset */
6130         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
6131             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
6132                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
6133                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
6134                 vf_id -= hw->func_caps.vf_base_id;
6135                 vf = &pf->vf[vf_id];
6136                 i40e_vc_notify_vf_reset(vf);
6137                 /* Allow VF to process pending reset notification */
6138                 msleep(20);
6139                 i40e_reset_vf(vf, false);
6140         }
6141 }
6142
6143 /**
6144  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
6145  * @pf: board private structure
6146  **/
6147 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
6148 {
6149         u32 val, fcnt_prog;
6150
6151         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
6152         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
6153         return fcnt_prog;
6154 }
6155
6156 /**
6157  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
6158  * @pf: board private structure
6159  **/
6160 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
6161 {
6162         u32 val, fcnt_prog;
6163
6164         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
6165         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
6166                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
6167                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
6168         return fcnt_prog;
6169 }
6170
6171 /**
6172  * i40e_get_global_fd_count - Get total FD filters programmed on device
6173  * @pf: board private structure
6174  **/
6175 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
6176 {
6177         u32 val, fcnt_prog;
6178
6179         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
6180         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
6181                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
6182                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
6183         return fcnt_prog;
6184 }
6185
6186 /**
6187  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
6188  * @pf: board private structure
6189  **/
6190 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
6191 {
6192         struct i40e_fdir_filter *filter;
6193         u32 fcnt_prog, fcnt_avail;
6194         struct hlist_node *node;
6195
6196         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
6197                 return;
6198
6199         /* Check if we have enough room to re-enable FDir SB capability. */
6200         fcnt_prog = i40e_get_global_fd_count(pf);
6201         fcnt_avail = pf->fdir_pf_filter_count;
6202         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
6203             (pf->fd_add_err == 0) ||
6204             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) {
6205                 if (pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) {
6206                         pf->flags &= ~I40E_FLAG_FD_SB_AUTO_DISABLED;
6207                         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
6208                             (I40E_DEBUG_FD & pf->hw.debug_mask))
6209                                 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
6210                 }
6211         }
6212
6213         /* We should wait for even more space before re-enabling ATR.
6214          * Additionally, we cannot enable ATR as long as we still have TCP SB
6215          * rules active.
6216          */
6217         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
6218             (pf->fd_tcp4_filter_cnt == 0)) {
6219                 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) {
6220                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
6221                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
6222                             (I40E_DEBUG_FD & pf->hw.debug_mask))
6223                                 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
6224                 }
6225         }
6226
6227         /* if hw had a problem adding a filter, delete it */
6228         if (pf->fd_inv > 0) {
6229                 hlist_for_each_entry_safe(filter, node,
6230                                           &pf->fdir_filter_list, fdir_node) {
6231                         if (filter->fd_id == pf->fd_inv) {
6232                                 hlist_del(&filter->fdir_node);
6233                                 kfree(filter);
6234                                 pf->fdir_pf_active_filters--;
6235                         }
6236                 }
6237         }
6238 }
6239
6240 #define I40E_MIN_FD_FLUSH_INTERVAL 10
6241 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
6242 /**
6243  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
6244  * @pf: board private structure
6245  **/
6246 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
6247 {
6248         unsigned long min_flush_time;
6249         int flush_wait_retry = 50;
6250         bool disable_atr = false;
6251         int fd_room;
6252         int reg;
6253
6254         if (!time_after(jiffies, pf->fd_flush_timestamp +
6255                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
6256                 return;
6257
6258         /* If the flush is happening too quick and we have mostly SB rules we
6259          * should not re-enable ATR for some time.
6260          */
6261         min_flush_time = pf->fd_flush_timestamp +
6262                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
6263         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
6264
6265         if (!(time_after(jiffies, min_flush_time)) &&
6266             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
6267                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6268                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
6269                 disable_atr = true;
6270         }
6271
6272         pf->fd_flush_timestamp = jiffies;
6273         pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED;
6274         /* flush all filters */
6275         wr32(&pf->hw, I40E_PFQF_CTL_1,
6276              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
6277         i40e_flush(&pf->hw);
6278         pf->fd_flush_cnt++;
6279         pf->fd_add_err = 0;
6280         do {
6281                 /* Check FD flush status every 5-6msec */
6282                 usleep_range(5000, 6000);
6283                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
6284                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
6285                         break;
6286         } while (flush_wait_retry--);
6287         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
6288                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
6289         } else {
6290                 /* replay sideband filters */
6291                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
6292                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
6293                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
6294                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
6295                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6296                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
6297         }
6298 }
6299
6300 /**
6301  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
6302  * @pf: board private structure
6303  **/
6304 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
6305 {
6306         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
6307 }
6308
6309 /* We can see up to 256 filter programming desc in transit if the filters are
6310  * being applied really fast; before we see the first
6311  * filter miss error on Rx queue 0. Accumulating enough error messages before
6312  * reacting will make sure we don't cause flush too often.
6313  */
6314 #define I40E_MAX_FD_PROGRAM_ERROR 256
6315
6316 /**
6317  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
6318  * @pf: board private structure
6319  **/
6320 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
6321 {
6322
6323         /* if interface is down do nothing */
6324         if (test_bit(__I40E_DOWN, pf->state))
6325                 return;
6326
6327         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
6328                 i40e_fdir_flush_and_replay(pf);
6329
6330         i40e_fdir_check_and_reenable(pf);
6331
6332 }
6333
6334 /**
6335  * i40e_vsi_link_event - notify VSI of a link event
6336  * @vsi: vsi to be notified
6337  * @link_up: link up or down
6338  **/
6339 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
6340 {
6341         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
6342                 return;
6343
6344         switch (vsi->type) {
6345         case I40E_VSI_MAIN:
6346                 if (!vsi->netdev || !vsi->netdev_registered)
6347                         break;
6348
6349                 if (link_up) {
6350                         netif_carrier_on(vsi->netdev);
6351                         netif_tx_wake_all_queues(vsi->netdev);
6352                 } else {
6353                         netif_carrier_off(vsi->netdev);
6354                         netif_tx_stop_all_queues(vsi->netdev);
6355                 }
6356                 break;
6357
6358         case I40E_VSI_SRIOV:
6359         case I40E_VSI_VMDQ2:
6360         case I40E_VSI_CTRL:
6361         case I40E_VSI_IWARP:
6362         case I40E_VSI_MIRROR:
6363         default:
6364                 /* there is no notification for other VSIs */
6365                 break;
6366         }
6367 }
6368
6369 /**
6370  * i40e_veb_link_event - notify elements on the veb of a link event
6371  * @veb: veb to be notified
6372  * @link_up: link up or down
6373  **/
6374 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
6375 {
6376         struct i40e_pf *pf;
6377         int i;
6378
6379         if (!veb || !veb->pf)
6380                 return;
6381         pf = veb->pf;
6382
6383         /* depth first... */
6384         for (i = 0; i < I40E_MAX_VEB; i++)
6385                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
6386                         i40e_veb_link_event(pf->veb[i], link_up);
6387
6388         /* ... now the local VSIs */
6389         for (i = 0; i < pf->num_alloc_vsi; i++)
6390                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
6391                         i40e_vsi_link_event(pf->vsi[i], link_up);
6392 }
6393
6394 /**
6395  * i40e_link_event - Update netif_carrier status
6396  * @pf: board private structure
6397  **/
6398 static void i40e_link_event(struct i40e_pf *pf)
6399 {
6400         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6401         u8 new_link_speed, old_link_speed;
6402         i40e_status status;
6403         bool new_link, old_link;
6404
6405         /* save off old link status information */
6406         pf->hw.phy.link_info_old = pf->hw.phy.link_info;
6407
6408         /* set this to force the get_link_status call to refresh state */
6409         pf->hw.phy.get_link_info = true;
6410
6411         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
6412
6413         status = i40e_get_link_status(&pf->hw, &new_link);
6414
6415         /* On success, disable temp link polling */
6416         if (status == I40E_SUCCESS) {
6417                 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)
6418                         pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING;
6419         } else {
6420                 /* Enable link polling temporarily until i40e_get_link_status
6421                  * returns I40E_SUCCESS
6422                  */
6423                 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING;
6424                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
6425                         status);
6426                 return;
6427         }
6428
6429         old_link_speed = pf->hw.phy.link_info_old.link_speed;
6430         new_link_speed = pf->hw.phy.link_info.link_speed;
6431
6432         if (new_link == old_link &&
6433             new_link_speed == old_link_speed &&
6434             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
6435              new_link == netif_carrier_ok(vsi->netdev)))
6436                 return;
6437
6438         i40e_print_link_message(vsi, new_link);
6439
6440         /* Notify the base of the switch tree connected to
6441          * the link.  Floating VEBs are not notified.
6442          */
6443         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
6444                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
6445         else
6446                 i40e_vsi_link_event(vsi, new_link);
6447
6448         if (pf->vf)
6449                 i40e_vc_notify_link_state(pf);
6450
6451         if (pf->flags & I40E_FLAG_PTP)
6452                 i40e_ptp_set_increment(pf);
6453 }
6454
6455 /**
6456  * i40e_watchdog_subtask - periodic checks not using event driven response
6457  * @pf: board private structure
6458  **/
6459 static void i40e_watchdog_subtask(struct i40e_pf *pf)
6460 {
6461         int i;
6462
6463         /* if interface is down do nothing */
6464         if (test_bit(__I40E_DOWN, pf->state) ||
6465             test_bit(__I40E_CONFIG_BUSY, pf->state))
6466                 return;
6467
6468         /* make sure we don't do these things too often */
6469         if (time_before(jiffies, (pf->service_timer_previous +
6470                                   pf->service_timer_period)))
6471                 return;
6472         pf->service_timer_previous = jiffies;
6473
6474         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
6475             (pf->flags & I40E_FLAG_TEMP_LINK_POLLING))
6476                 i40e_link_event(pf);
6477
6478         /* Update the stats for active netdevs so the network stack
6479          * can look at updated numbers whenever it cares to
6480          */
6481         for (i = 0; i < pf->num_alloc_vsi; i++)
6482                 if (pf->vsi[i] && pf->vsi[i]->netdev)
6483                         i40e_update_stats(pf->vsi[i]);
6484
6485         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
6486                 /* Update the stats for the active switching components */
6487                 for (i = 0; i < I40E_MAX_VEB; i++)
6488                         if (pf->veb[i])
6489                                 i40e_update_veb_stats(pf->veb[i]);
6490         }
6491
6492         i40e_ptp_rx_hang(pf);
6493         i40e_ptp_tx_hang(pf);
6494 }
6495
6496 /**
6497  * i40e_reset_subtask - Set up for resetting the device and driver
6498  * @pf: board private structure
6499  **/
6500 static void i40e_reset_subtask(struct i40e_pf *pf)
6501 {
6502         u32 reset_flags = 0;
6503
6504         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
6505                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
6506                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
6507         }
6508         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
6509                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
6510                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6511         }
6512         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
6513                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
6514                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
6515         }
6516         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
6517                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
6518                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
6519         }
6520         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
6521                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
6522                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
6523         }
6524
6525         /* If there's a recovery already waiting, it takes
6526          * precedence before starting a new reset sequence.
6527          */
6528         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
6529                 i40e_prep_for_reset(pf, false);
6530                 i40e_reset(pf);
6531                 i40e_rebuild(pf, false, false);
6532         }
6533
6534         /* If we're already down or resetting, just bail */
6535         if (reset_flags &&
6536             !test_bit(__I40E_DOWN, pf->state) &&
6537             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
6538                 i40e_do_reset(pf, reset_flags, false);
6539         }
6540 }
6541
6542 /**
6543  * i40e_handle_link_event - Handle link event
6544  * @pf: board private structure
6545  * @e: event info posted on ARQ
6546  **/
6547 static void i40e_handle_link_event(struct i40e_pf *pf,
6548                                    struct i40e_arq_event_info *e)
6549 {
6550         struct i40e_aqc_get_link_status *status =
6551                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
6552
6553         /* Do a new status request to re-enable LSE reporting
6554          * and load new status information into the hw struct
6555          * This completely ignores any state information
6556          * in the ARQ event info, instead choosing to always
6557          * issue the AQ update link status command.
6558          */
6559         i40e_link_event(pf);
6560
6561         /* Check if module meets thermal requirements */
6562         if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
6563                 dev_err(&pf->pdev->dev,
6564                         "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
6565                 dev_err(&pf->pdev->dev,
6566                         "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
6567         } else {
6568                 /* check for unqualified module, if link is down, suppress
6569                  * the message if link was forced to be down.
6570                  */
6571                 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
6572                     (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
6573                     (!(status->link_info & I40E_AQ_LINK_UP)) &&
6574                     (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
6575                         dev_err(&pf->pdev->dev,
6576                                 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
6577                         dev_err(&pf->pdev->dev,
6578                                 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
6579                 }
6580         }
6581 }
6582
6583 /**
6584  * i40e_clean_adminq_subtask - Clean the AdminQ rings
6585  * @pf: board private structure
6586  **/
6587 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
6588 {
6589         struct i40e_arq_event_info event;
6590         struct i40e_hw *hw = &pf->hw;
6591         u16 pending, i = 0;
6592         i40e_status ret;
6593         u16 opcode;
6594         u32 oldval;
6595         u32 val;
6596
6597         /* Do not run clean AQ when PF reset fails */
6598         if (test_bit(__I40E_RESET_FAILED, pf->state))
6599                 return;
6600
6601         /* check for error indications */
6602         val = rd32(&pf->hw, pf->hw.aq.arq.len);
6603         oldval = val;
6604         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
6605                 if (hw->debug_mask & I40E_DEBUG_AQ)
6606                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
6607                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
6608         }
6609         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
6610                 if (hw->debug_mask & I40E_DEBUG_AQ)
6611                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
6612                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
6613                 pf->arq_overflows++;
6614         }
6615         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
6616                 if (hw->debug_mask & I40E_DEBUG_AQ)
6617                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
6618                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
6619         }
6620         if (oldval != val)
6621                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
6622
6623         val = rd32(&pf->hw, pf->hw.aq.asq.len);
6624         oldval = val;
6625         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
6626                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6627                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
6628                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
6629         }
6630         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
6631                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6632                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
6633                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
6634         }
6635         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
6636                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6637                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
6638                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
6639         }
6640         if (oldval != val)
6641                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
6642
6643         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
6644         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
6645         if (!event.msg_buf)
6646                 return;
6647
6648         do {
6649                 ret = i40e_clean_arq_element(hw, &event, &pending);
6650                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
6651                         break;
6652                 else if (ret) {
6653                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
6654                         break;
6655                 }
6656
6657                 opcode = le16_to_cpu(event.desc.opcode);
6658                 switch (opcode) {
6659
6660                 case i40e_aqc_opc_get_link_status:
6661                         i40e_handle_link_event(pf, &event);
6662                         break;
6663                 case i40e_aqc_opc_send_msg_to_pf:
6664                         ret = i40e_vc_process_vf_msg(pf,
6665                                         le16_to_cpu(event.desc.retval),
6666                                         le32_to_cpu(event.desc.cookie_high),
6667                                         le32_to_cpu(event.desc.cookie_low),
6668                                         event.msg_buf,
6669                                         event.msg_len);
6670                         break;
6671                 case i40e_aqc_opc_lldp_update_mib:
6672                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
6673 #ifdef CONFIG_I40E_DCB
6674                         rtnl_lock();
6675                         ret = i40e_handle_lldp_event(pf, &event);
6676                         rtnl_unlock();
6677 #endif /* CONFIG_I40E_DCB */
6678                         break;
6679                 case i40e_aqc_opc_event_lan_overflow:
6680                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
6681                         i40e_handle_lan_overflow_event(pf, &event);
6682                         break;
6683                 case i40e_aqc_opc_send_msg_to_peer:
6684                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
6685                         break;
6686                 case i40e_aqc_opc_nvm_erase:
6687                 case i40e_aqc_opc_nvm_update:
6688                 case i40e_aqc_opc_oem_post_update:
6689                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
6690                                    "ARQ NVM operation 0x%04x completed\n",
6691                                    opcode);
6692                         break;
6693                 default:
6694                         dev_info(&pf->pdev->dev,
6695                                  "ARQ: Unknown event 0x%04x ignored\n",
6696                                  opcode);
6697                         break;
6698                 }
6699         } while (i++ < pf->adminq_work_limit);
6700
6701         if (i < pf->adminq_work_limit)
6702                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
6703
6704         /* re-enable Admin queue interrupt cause */
6705         val = rd32(hw, I40E_PFINT_ICR0_ENA);
6706         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
6707         wr32(hw, I40E_PFINT_ICR0_ENA, val);
6708         i40e_flush(hw);
6709
6710         kfree(event.msg_buf);
6711 }
6712
6713 /**
6714  * i40e_verify_eeprom - make sure eeprom is good to use
6715  * @pf: board private structure
6716  **/
6717 static void i40e_verify_eeprom(struct i40e_pf *pf)
6718 {
6719         int err;
6720
6721         err = i40e_diag_eeprom_test(&pf->hw);
6722         if (err) {
6723                 /* retry in case of garbage read */
6724                 err = i40e_diag_eeprom_test(&pf->hw);
6725                 if (err) {
6726                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
6727                                  err);
6728                         set_bit(__I40E_BAD_EEPROM, pf->state);
6729                 }
6730         }
6731
6732         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
6733                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
6734                 clear_bit(__I40E_BAD_EEPROM, pf->state);
6735         }
6736 }
6737
6738 /**
6739  * i40e_enable_pf_switch_lb
6740  * @pf: pointer to the PF structure
6741  *
6742  * enable switch loop back or die - no point in a return value
6743  **/
6744 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
6745 {
6746         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6747         struct i40e_vsi_context ctxt;
6748         int ret;
6749
6750         ctxt.seid = pf->main_vsi_seid;
6751         ctxt.pf_num = pf->hw.pf_id;
6752         ctxt.vf_num = 0;
6753         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6754         if (ret) {
6755                 dev_info(&pf->pdev->dev,
6756                          "couldn't get PF vsi config, err %s aq_err %s\n",
6757                          i40e_stat_str(&pf->hw, ret),
6758                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6759                 return;
6760         }
6761         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6762         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6763         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6764
6765         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6766         if (ret) {
6767                 dev_info(&pf->pdev->dev,
6768                          "update vsi switch failed, err %s aq_err %s\n",
6769                          i40e_stat_str(&pf->hw, ret),
6770                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6771         }
6772 }
6773
6774 /**
6775  * i40e_disable_pf_switch_lb
6776  * @pf: pointer to the PF structure
6777  *
6778  * disable switch loop back or die - no point in a return value
6779  **/
6780 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
6781 {
6782         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6783         struct i40e_vsi_context ctxt;
6784         int ret;
6785
6786         ctxt.seid = pf->main_vsi_seid;
6787         ctxt.pf_num = pf->hw.pf_id;
6788         ctxt.vf_num = 0;
6789         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6790         if (ret) {
6791                 dev_info(&pf->pdev->dev,
6792                          "couldn't get PF vsi config, err %s aq_err %s\n",
6793                          i40e_stat_str(&pf->hw, ret),
6794                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6795                 return;
6796         }
6797         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6798         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6799         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6800
6801         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6802         if (ret) {
6803                 dev_info(&pf->pdev->dev,
6804                          "update vsi switch failed, err %s aq_err %s\n",
6805                          i40e_stat_str(&pf->hw, ret),
6806                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6807         }
6808 }
6809
6810 /**
6811  * i40e_config_bridge_mode - Configure the HW bridge mode
6812  * @veb: pointer to the bridge instance
6813  *
6814  * Configure the loop back mode for the LAN VSI that is downlink to the
6815  * specified HW bridge instance. It is expected this function is called
6816  * when a new HW bridge is instantiated.
6817  **/
6818 static void i40e_config_bridge_mode(struct i40e_veb *veb)
6819 {
6820         struct i40e_pf *pf = veb->pf;
6821
6822         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
6823                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
6824                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
6825         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
6826                 i40e_disable_pf_switch_lb(pf);
6827         else
6828                 i40e_enable_pf_switch_lb(pf);
6829 }
6830
6831 /**
6832  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
6833  * @veb: pointer to the VEB instance
6834  *
6835  * This is a recursive function that first builds the attached VSIs then
6836  * recurses in to build the next layer of VEB.  We track the connections
6837  * through our own index numbers because the seid's from the HW could
6838  * change across the reset.
6839  **/
6840 static int i40e_reconstitute_veb(struct i40e_veb *veb)
6841 {
6842         struct i40e_vsi *ctl_vsi = NULL;
6843         struct i40e_pf *pf = veb->pf;
6844         int v, veb_idx;
6845         int ret;
6846
6847         /* build VSI that owns this VEB, temporarily attached to base VEB */
6848         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
6849                 if (pf->vsi[v] &&
6850                     pf->vsi[v]->veb_idx == veb->idx &&
6851                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
6852                         ctl_vsi = pf->vsi[v];
6853                         break;
6854                 }
6855         }
6856         if (!ctl_vsi) {
6857                 dev_info(&pf->pdev->dev,
6858                          "missing owner VSI for veb_idx %d\n", veb->idx);
6859                 ret = -ENOENT;
6860                 goto end_reconstitute;
6861         }
6862         if (ctl_vsi != pf->vsi[pf->lan_vsi])
6863                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6864         ret = i40e_add_vsi(ctl_vsi);
6865         if (ret) {
6866                 dev_info(&pf->pdev->dev,
6867                          "rebuild of veb_idx %d owner VSI failed: %d\n",
6868                          veb->idx, ret);
6869                 goto end_reconstitute;
6870         }
6871         i40e_vsi_reset_stats(ctl_vsi);
6872
6873         /* create the VEB in the switch and move the VSI onto the VEB */
6874         ret = i40e_add_veb(veb, ctl_vsi);
6875         if (ret)
6876                 goto end_reconstitute;
6877
6878         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
6879                 veb->bridge_mode = BRIDGE_MODE_VEB;
6880         else
6881                 veb->bridge_mode = BRIDGE_MODE_VEPA;
6882         i40e_config_bridge_mode(veb);
6883
6884         /* create the remaining VSIs attached to this VEB */
6885         for (v = 0; v < pf->num_alloc_vsi; v++) {
6886                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
6887                         continue;
6888
6889                 if (pf->vsi[v]->veb_idx == veb->idx) {
6890                         struct i40e_vsi *vsi = pf->vsi[v];
6891
6892                         vsi->uplink_seid = veb->seid;
6893                         ret = i40e_add_vsi(vsi);
6894                         if (ret) {
6895                                 dev_info(&pf->pdev->dev,
6896                                          "rebuild of vsi_idx %d failed: %d\n",
6897                                          v, ret);
6898                                 goto end_reconstitute;
6899                         }
6900                         i40e_vsi_reset_stats(vsi);
6901                 }
6902         }
6903
6904         /* create any VEBs attached to this VEB - RECURSION */
6905         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
6906                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
6907                         pf->veb[veb_idx]->uplink_seid = veb->seid;
6908                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
6909                         if (ret)
6910                                 break;
6911                 }
6912         }
6913
6914 end_reconstitute:
6915         return ret;
6916 }
6917
6918 /**
6919  * i40e_get_capabilities - get info about the HW
6920  * @pf: the PF struct
6921  **/
6922 static int i40e_get_capabilities(struct i40e_pf *pf)
6923 {
6924         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
6925         u16 data_size;
6926         int buf_len;
6927         int err;
6928
6929         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
6930         do {
6931                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
6932                 if (!cap_buf)
6933                         return -ENOMEM;
6934
6935                 /* this loads the data into the hw struct for us */
6936                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
6937                                             &data_size,
6938                                             i40e_aqc_opc_list_func_capabilities,
6939                                             NULL);
6940                 /* data loaded, buffer no longer needed */
6941                 kfree(cap_buf);
6942
6943                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
6944                         /* retry with a larger buffer */
6945                         buf_len = data_size;
6946                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
6947                         dev_info(&pf->pdev->dev,
6948                                  "capability discovery failed, err %s aq_err %s\n",
6949                                  i40e_stat_str(&pf->hw, err),
6950                                  i40e_aq_str(&pf->hw,
6951                                              pf->hw.aq.asq_last_status));
6952                         return -ENODEV;
6953                 }
6954         } while (err);
6955
6956         if (pf->hw.debug_mask & I40E_DEBUG_USER)
6957                 dev_info(&pf->pdev->dev,
6958                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
6959                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
6960                          pf->hw.func_caps.num_msix_vectors,
6961                          pf->hw.func_caps.num_msix_vectors_vf,
6962                          pf->hw.func_caps.fd_filters_guaranteed,
6963                          pf->hw.func_caps.fd_filters_best_effort,
6964                          pf->hw.func_caps.num_tx_qp,
6965                          pf->hw.func_caps.num_vsis);
6966
6967 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
6968                        + pf->hw.func_caps.num_vfs)
6969         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
6970                 dev_info(&pf->pdev->dev,
6971                          "got num_vsis %d, setting num_vsis to %d\n",
6972                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
6973                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
6974         }
6975
6976         return 0;
6977 }
6978
6979 static int i40e_vsi_clear(struct i40e_vsi *vsi);
6980
6981 /**
6982  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
6983  * @pf: board private structure
6984  **/
6985 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
6986 {
6987         struct i40e_vsi *vsi;
6988
6989         /* quick workaround for an NVM issue that leaves a critical register
6990          * uninitialized
6991          */
6992         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
6993                 static const u32 hkey[] = {
6994                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
6995                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
6996                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
6997                         0x95b3a76d};
6998                 int i;
6999
7000                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
7001                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
7002         }
7003
7004         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
7005                 return;
7006
7007         /* find existing VSI and see if it needs configuring */
7008         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
7009
7010         /* create a new VSI if none exists */
7011         if (!vsi) {
7012                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
7013                                      pf->vsi[pf->lan_vsi]->seid, 0);
7014                 if (!vsi) {
7015                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
7016                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
7017                         return;
7018                 }
7019         }
7020
7021         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
7022 }
7023
7024 /**
7025  * i40e_fdir_teardown - release the Flow Director resources
7026  * @pf: board private structure
7027  **/
7028 static void i40e_fdir_teardown(struct i40e_pf *pf)
7029 {
7030         struct i40e_vsi *vsi;
7031
7032         i40e_fdir_filter_exit(pf);
7033         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
7034         if (vsi)
7035                 i40e_vsi_release(vsi);
7036 }
7037
7038 /**
7039  * i40e_prep_for_reset - prep for the core to reset
7040  * @pf: board private structure
7041  * @lock_acquired: indicates whether or not the lock has been acquired
7042  * before this function was called.
7043  *
7044  * Close up the VFs and other things in prep for PF Reset.
7045   **/
7046 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
7047 {
7048         struct i40e_hw *hw = &pf->hw;
7049         i40e_status ret = 0;
7050         u32 v;
7051
7052         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
7053         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
7054                 return;
7055         if (i40e_check_asq_alive(&pf->hw))
7056                 i40e_vc_notify_reset(pf);
7057
7058         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
7059
7060         /* quiesce the VSIs and their queues that are not already DOWN */
7061         /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */
7062         if (!lock_acquired)
7063                 rtnl_lock();
7064         i40e_pf_quiesce_all_vsi(pf);
7065         if (!lock_acquired)
7066                 rtnl_unlock();
7067
7068         for (v = 0; v < pf->num_alloc_vsi; v++) {
7069                 if (pf->vsi[v])
7070                         pf->vsi[v]->seid = 0;
7071         }
7072
7073         i40e_shutdown_adminq(&pf->hw);
7074
7075         /* call shutdown HMC */
7076         if (hw->hmc.hmc_obj) {
7077                 ret = i40e_shutdown_lan_hmc(hw);
7078                 if (ret)
7079                         dev_warn(&pf->pdev->dev,
7080                                  "shutdown_lan_hmc failed: %d\n", ret);
7081         }
7082 }
7083
7084 /**
7085  * i40e_send_version - update firmware with driver version
7086  * @pf: PF struct
7087  */
7088 static void i40e_send_version(struct i40e_pf *pf)
7089 {
7090         struct i40e_driver_version dv;
7091
7092         dv.major_version = DRV_VERSION_MAJOR;
7093         dv.minor_version = DRV_VERSION_MINOR;
7094         dv.build_version = DRV_VERSION_BUILD;
7095         dv.subbuild_version = 0;
7096         strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
7097         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
7098 }
7099
7100 /**
7101  * i40e_get_oem_version - get OEM specific version information
7102  * @hw: pointer to the hardware structure
7103  **/
7104 static void i40e_get_oem_version(struct i40e_hw *hw)
7105 {
7106         u16 block_offset = 0xffff;
7107         u16 block_length = 0;
7108         u16 capabilities = 0;
7109         u16 gen_snap = 0;
7110         u16 release = 0;
7111
7112 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
7113 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
7114 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
7115 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
7116 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
7117 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
7118 #define I40E_NVM_OEM_LENGTH                     3
7119
7120         /* Check if pointer to OEM version block is valid. */
7121         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
7122         if (block_offset == 0xffff)
7123                 return;
7124
7125         /* Check if OEM version block has correct length. */
7126         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
7127                            &block_length);
7128         if (block_length < I40E_NVM_OEM_LENGTH)
7129                 return;
7130
7131         /* Check if OEM version format is as expected. */
7132         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
7133                            &capabilities);
7134         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
7135                 return;
7136
7137         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
7138                            &gen_snap);
7139         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
7140                            &release);
7141         hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
7142         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
7143 }
7144
7145 /**
7146  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
7147  * @pf: board private structure
7148  **/
7149 static int i40e_reset(struct i40e_pf *pf)
7150 {
7151         struct i40e_hw *hw = &pf->hw;
7152         i40e_status ret;
7153
7154         ret = i40e_pf_reset(hw);
7155         if (ret) {
7156                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
7157                 set_bit(__I40E_RESET_FAILED, pf->state);
7158                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
7159         } else {
7160                 pf->pfr_count++;
7161         }
7162         return ret;
7163 }
7164
7165 /**
7166  * i40e_rebuild - rebuild using a saved config
7167  * @pf: board private structure
7168  * @reinit: if the Main VSI needs to re-initialized.
7169  * @lock_acquired: indicates whether or not the lock has been acquired
7170  * before this function was called.
7171  **/
7172 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
7173 {
7174         struct i40e_hw *hw = &pf->hw;
7175         u8 set_fc_aq_fail = 0;
7176         i40e_status ret;
7177         u32 val;
7178         int v;
7179
7180         if (test_bit(__I40E_DOWN, pf->state))
7181                 goto clear_recovery;
7182         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
7183
7184         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
7185         ret = i40e_init_adminq(&pf->hw);
7186         if (ret) {
7187                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
7188                          i40e_stat_str(&pf->hw, ret),
7189                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7190                 goto clear_recovery;
7191         }
7192         i40e_get_oem_version(&pf->hw);
7193
7194         /* re-verify the eeprom if we just had an EMP reset */
7195         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
7196                 i40e_verify_eeprom(pf);
7197
7198         i40e_clear_pxe_mode(hw);
7199         ret = i40e_get_capabilities(pf);
7200         if (ret)
7201                 goto end_core_reset;
7202
7203         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
7204                                 hw->func_caps.num_rx_qp, 0, 0);
7205         if (ret) {
7206                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
7207                 goto end_core_reset;
7208         }
7209         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
7210         if (ret) {
7211                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
7212                 goto end_core_reset;
7213         }
7214
7215 #ifdef CONFIG_I40E_DCB
7216         ret = i40e_init_pf_dcb(pf);
7217         if (ret) {
7218                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
7219                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
7220                 /* Continue without DCB enabled */
7221         }
7222 #endif /* CONFIG_I40E_DCB */
7223         /* do basic switch setup */
7224         if (!lock_acquired)
7225                 rtnl_lock();
7226         ret = i40e_setup_pf_switch(pf, reinit);
7227         if (ret)
7228                 goto end_unlock;
7229
7230         /* The driver only wants link up/down and module qualification
7231          * reports from firmware.  Note the negative logic.
7232          */
7233         ret = i40e_aq_set_phy_int_mask(&pf->hw,
7234                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
7235                                          I40E_AQ_EVENT_MEDIA_NA |
7236                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
7237         if (ret)
7238                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
7239                          i40e_stat_str(&pf->hw, ret),
7240                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7241
7242         /* make sure our flow control settings are restored */
7243         ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
7244         if (ret)
7245                 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
7246                         i40e_stat_str(&pf->hw, ret),
7247                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7248
7249         /* Rebuild the VSIs and VEBs that existed before reset.
7250          * They are still in our local switch element arrays, so only
7251          * need to rebuild the switch model in the HW.
7252          *
7253          * If there were VEBs but the reconstitution failed, we'll try
7254          * try to recover minimal use by getting the basic PF VSI working.
7255          */
7256         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
7257                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
7258                 /* find the one VEB connected to the MAC, and find orphans */
7259                 for (v = 0; v < I40E_MAX_VEB; v++) {
7260                         if (!pf->veb[v])
7261                                 continue;
7262
7263                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
7264                             pf->veb[v]->uplink_seid == 0) {
7265                                 ret = i40e_reconstitute_veb(pf->veb[v]);
7266
7267                                 if (!ret)
7268                                         continue;
7269
7270                                 /* If Main VEB failed, we're in deep doodoo,
7271                                  * so give up rebuilding the switch and set up
7272                                  * for minimal rebuild of PF VSI.
7273                                  * If orphan failed, we'll report the error
7274                                  * but try to keep going.
7275                                  */
7276                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
7277                                         dev_info(&pf->pdev->dev,
7278                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
7279                                                  ret);
7280                                         pf->vsi[pf->lan_vsi]->uplink_seid
7281                                                                 = pf->mac_seid;
7282                                         break;
7283                                 } else if (pf->veb[v]->uplink_seid == 0) {
7284                                         dev_info(&pf->pdev->dev,
7285                                                  "rebuild of orphan VEB failed: %d\n",
7286                                                  ret);
7287                                 }
7288                         }
7289                 }
7290         }
7291
7292         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
7293                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
7294                 /* no VEB, so rebuild only the Main VSI */
7295                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
7296                 if (ret) {
7297                         dev_info(&pf->pdev->dev,
7298                                  "rebuild of Main VSI failed: %d\n", ret);
7299                         goto end_unlock;
7300                 }
7301         }
7302
7303         /* Reconfigure hardware for allowing smaller MSS in the case
7304          * of TSO, so that we avoid the MDD being fired and causing
7305          * a reset in the case of small MSS+TSO.
7306          */
7307 #define I40E_REG_MSS          0x000E64DC
7308 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
7309 #define I40E_64BYTE_MSS       0x400000
7310         val = rd32(hw, I40E_REG_MSS);
7311         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
7312                 val &= ~I40E_REG_MSS_MIN_MASK;
7313                 val |= I40E_64BYTE_MSS;
7314                 wr32(hw, I40E_REG_MSS, val);
7315         }
7316
7317         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
7318                 msleep(75);
7319                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
7320                 if (ret)
7321                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
7322                                  i40e_stat_str(&pf->hw, ret),
7323                                  i40e_aq_str(&pf->hw,
7324                                              pf->hw.aq.asq_last_status));
7325         }
7326         /* reinit the misc interrupt */
7327         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7328                 ret = i40e_setup_misc_vector(pf);
7329
7330         /* Add a filter to drop all Flow control frames from any VSI from being
7331          * transmitted. By doing so we stop a malicious VF from sending out
7332          * PAUSE or PFC frames and potentially controlling traffic for other
7333          * PF/VF VSIs.
7334          * The FW can still send Flow control frames if enabled.
7335          */
7336         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
7337                                                        pf->main_vsi_seid);
7338
7339         /* restart the VSIs that were rebuilt and running before the reset */
7340         i40e_pf_unquiesce_all_vsi(pf);
7341
7342         /* Release the RTNL lock before we start resetting VFs */
7343         if (!lock_acquired)
7344                 rtnl_unlock();
7345
7346         i40e_reset_all_vfs(pf, true);
7347
7348         /* tell the firmware that we're starting */
7349         i40e_send_version(pf);
7350
7351         /* We've already released the lock, so don't do it again */
7352         goto end_core_reset;
7353
7354 end_unlock:
7355         if (!lock_acquired)
7356                 rtnl_unlock();
7357 end_core_reset:
7358         clear_bit(__I40E_RESET_FAILED, pf->state);
7359 clear_recovery:
7360         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
7361 }
7362
7363 /**
7364  * i40e_reset_and_rebuild - reset and rebuild using a saved config
7365  * @pf: board private structure
7366  * @reinit: if the Main VSI needs to re-initialized.
7367  * @lock_acquired: indicates whether or not the lock has been acquired
7368  * before this function was called.
7369  **/
7370 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
7371                                    bool lock_acquired)
7372 {
7373         int ret;
7374         /* Now we wait for GRST to settle out.
7375          * We don't have to delete the VEBs or VSIs from the hw switch
7376          * because the reset will make them disappear.
7377          */
7378         ret = i40e_reset(pf);
7379         if (!ret)
7380                 i40e_rebuild(pf, reinit, lock_acquired);
7381 }
7382
7383 /**
7384  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
7385  * @pf: board private structure
7386  *
7387  * Close up the VFs and other things in prep for a Core Reset,
7388  * then get ready to rebuild the world.
7389  * @lock_acquired: indicates whether or not the lock has been acquired
7390  * before this function was called.
7391  **/
7392 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
7393 {
7394         i40e_prep_for_reset(pf, lock_acquired);
7395         i40e_reset_and_rebuild(pf, false, lock_acquired);
7396 }
7397
7398 /**
7399  * i40e_handle_mdd_event
7400  * @pf: pointer to the PF structure
7401  *
7402  * Called from the MDD irq handler to identify possibly malicious vfs
7403  **/
7404 static void i40e_handle_mdd_event(struct i40e_pf *pf)
7405 {
7406         struct i40e_hw *hw = &pf->hw;
7407         bool mdd_detected = false;
7408         bool pf_mdd_detected = false;
7409         struct i40e_vf *vf;
7410         u32 reg;
7411         int i;
7412
7413         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
7414                 return;
7415
7416         /* find what triggered the MDD event */
7417         reg = rd32(hw, I40E_GL_MDET_TX);
7418         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
7419                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
7420                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
7421                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
7422                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
7423                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
7424                                 I40E_GL_MDET_TX_EVENT_SHIFT;
7425                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
7426                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
7427                                 pf->hw.func_caps.base_queue;
7428                 if (netif_msg_tx_err(pf))
7429                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
7430                                  event, queue, pf_num, vf_num);
7431                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
7432                 mdd_detected = true;
7433         }
7434         reg = rd32(hw, I40E_GL_MDET_RX);
7435         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
7436                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
7437                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
7438                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
7439                                 I40E_GL_MDET_RX_EVENT_SHIFT;
7440                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
7441                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
7442                                 pf->hw.func_caps.base_queue;
7443                 if (netif_msg_rx_err(pf))
7444                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
7445                                  event, queue, func);
7446                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
7447                 mdd_detected = true;
7448         }
7449
7450         if (mdd_detected) {
7451                 reg = rd32(hw, I40E_PF_MDET_TX);
7452                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
7453                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
7454                         dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
7455                         pf_mdd_detected = true;
7456                 }
7457                 reg = rd32(hw, I40E_PF_MDET_RX);
7458                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
7459                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
7460                         dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
7461                         pf_mdd_detected = true;
7462                 }
7463                 /* Queue belongs to the PF, initiate a reset */
7464                 if (pf_mdd_detected) {
7465                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
7466                         i40e_service_event_schedule(pf);
7467                 }
7468         }
7469
7470         /* see if one of the VFs needs its hand slapped */
7471         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
7472                 vf = &(pf->vf[i]);
7473                 reg = rd32(hw, I40E_VP_MDET_TX(i));
7474                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
7475                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
7476                         vf->num_mdd_events++;
7477                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
7478                                  i);
7479                 }
7480
7481                 reg = rd32(hw, I40E_VP_MDET_RX(i));
7482                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
7483                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
7484                         vf->num_mdd_events++;
7485                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
7486                                  i);
7487                 }
7488
7489                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
7490                         dev_info(&pf->pdev->dev,
7491                                  "Too many MDD events on VF %d, disabled\n", i);
7492                         dev_info(&pf->pdev->dev,
7493                                  "Use PF Control I/F to re-enable the VF\n");
7494                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
7495                 }
7496         }
7497
7498         /* re-enable mdd interrupt cause */
7499         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
7500         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
7501         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
7502         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
7503         i40e_flush(hw);
7504 }
7505
7506 static const char *i40e_tunnel_name(struct i40e_udp_port_config *port)
7507 {
7508         switch (port->type) {
7509         case UDP_TUNNEL_TYPE_VXLAN:
7510                 return "vxlan";
7511         case UDP_TUNNEL_TYPE_GENEVE:
7512                 return "geneve";
7513         default:
7514                 return "unknown";
7515         }
7516 }
7517
7518 /**
7519  * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters
7520  * @pf: board private structure
7521  **/
7522 static void i40e_sync_udp_filters(struct i40e_pf *pf)
7523 {
7524         int i;
7525
7526         /* loop through and set pending bit for all active UDP filters */
7527         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7528                 if (pf->udp_ports[i].port)
7529                         pf->pending_udp_bitmap |= BIT_ULL(i);
7530         }
7531
7532         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
7533 }
7534
7535 /**
7536  * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
7537  * @pf: board private structure
7538  **/
7539 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
7540 {
7541         struct i40e_hw *hw = &pf->hw;
7542         i40e_status ret;
7543         u16 port;
7544         int i;
7545
7546         if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC))
7547                 return;
7548
7549         pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC;
7550
7551         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7552                 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
7553                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
7554                         port = pf->udp_ports[i].port;
7555                         if (port)
7556                                 ret = i40e_aq_add_udp_tunnel(hw, port,
7557                                                         pf->udp_ports[i].type,
7558                                                         NULL, NULL);
7559                         else
7560                                 ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
7561
7562                         if (ret) {
7563                                 dev_info(&pf->pdev->dev,
7564                                          "%s %s port %d, index %d failed, err %s aq_err %s\n",
7565                                          i40e_tunnel_name(&pf->udp_ports[i]),
7566                                          port ? "add" : "delete",
7567                                          port, i,
7568                                          i40e_stat_str(&pf->hw, ret),
7569                                          i40e_aq_str(&pf->hw,
7570                                                      pf->hw.aq.asq_last_status));
7571                                 pf->udp_ports[i].port = 0;
7572                         }
7573                 }
7574         }
7575 }
7576
7577 /**
7578  * i40e_service_task - Run the driver's async subtasks
7579  * @work: pointer to work_struct containing our data
7580  **/
7581 static void i40e_service_task(struct work_struct *work)
7582 {
7583         struct i40e_pf *pf = container_of(work,
7584                                           struct i40e_pf,
7585                                           service_task);
7586         unsigned long start_time = jiffies;
7587
7588         /* don't bother with service tasks if a reset is in progress */
7589         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
7590                 return;
7591
7592         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
7593                 return;
7594
7595         i40e_detect_recover_hung(pf);
7596         i40e_sync_filters_subtask(pf);
7597         i40e_reset_subtask(pf);
7598         i40e_handle_mdd_event(pf);
7599         i40e_vc_process_vflr_event(pf);
7600         i40e_watchdog_subtask(pf);
7601         i40e_fdir_reinit_subtask(pf);
7602         if (pf->flags & I40E_FLAG_CLIENT_RESET) {
7603                 /* Client subtask will reopen next time through. */
7604                 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true);
7605                 pf->flags &= ~I40E_FLAG_CLIENT_RESET;
7606         } else {
7607                 i40e_client_subtask(pf);
7608                 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) {
7609                         i40e_notify_client_of_l2_param_changes(
7610                                                         pf->vsi[pf->lan_vsi]);
7611                         pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE;
7612                 }
7613         }
7614         i40e_sync_filters_subtask(pf);
7615         i40e_sync_udp_filters_subtask(pf);
7616         i40e_clean_adminq_subtask(pf);
7617
7618         /* flush memory to make sure state is correct before next watchdog */
7619         smp_mb__before_atomic();
7620         clear_bit(__I40E_SERVICE_SCHED, pf->state);
7621
7622         /* If the tasks have taken longer than one timer cycle or there
7623          * is more work to be done, reschedule the service task now
7624          * rather than wait for the timer to tick again.
7625          */
7626         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
7627             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
7628             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
7629             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
7630                 i40e_service_event_schedule(pf);
7631 }
7632
7633 /**
7634  * i40e_service_timer - timer callback
7635  * @data: pointer to PF struct
7636  **/
7637 static void i40e_service_timer(unsigned long data)
7638 {
7639         struct i40e_pf *pf = (struct i40e_pf *)data;
7640
7641         mod_timer(&pf->service_timer,
7642                   round_jiffies(jiffies + pf->service_timer_period));
7643         i40e_service_event_schedule(pf);
7644 }
7645
7646 /**
7647  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
7648  * @vsi: the VSI being configured
7649  **/
7650 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
7651 {
7652         struct i40e_pf *pf = vsi->back;
7653
7654         switch (vsi->type) {
7655         case I40E_VSI_MAIN:
7656                 vsi->alloc_queue_pairs = pf->num_lan_qps;
7657                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7658                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7659                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7660                         vsi->num_q_vectors = pf->num_lan_msix;
7661                 else
7662                         vsi->num_q_vectors = 1;
7663
7664                 break;
7665
7666         case I40E_VSI_FDIR:
7667                 vsi->alloc_queue_pairs = 1;
7668                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
7669                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7670                 vsi->num_q_vectors = pf->num_fdsb_msix;
7671                 break;
7672
7673         case I40E_VSI_VMDQ2:
7674                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
7675                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7676                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7677                 vsi->num_q_vectors = pf->num_vmdq_msix;
7678                 break;
7679
7680         case I40E_VSI_SRIOV:
7681                 vsi->alloc_queue_pairs = pf->num_vf_qps;
7682                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7683                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7684                 break;
7685
7686         default:
7687                 WARN_ON(1);
7688                 return -ENODATA;
7689         }
7690
7691         return 0;
7692 }
7693
7694 /**
7695  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
7696  * @type: VSI pointer
7697  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
7698  *
7699  * On error: returns error code (negative)
7700  * On success: returns 0
7701  **/
7702 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
7703 {
7704         struct i40e_ring **next_rings;
7705         int size;
7706         int ret = 0;
7707
7708         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
7709         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
7710                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
7711         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
7712         if (!vsi->tx_rings)
7713                 return -ENOMEM;
7714         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
7715         if (i40e_enabled_xdp_vsi(vsi)) {
7716                 vsi->xdp_rings = next_rings;
7717                 next_rings += vsi->alloc_queue_pairs;
7718         }
7719         vsi->rx_rings = next_rings;
7720
7721         if (alloc_qvectors) {
7722                 /* allocate memory for q_vector pointers */
7723                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
7724                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
7725                 if (!vsi->q_vectors) {
7726                         ret = -ENOMEM;
7727                         goto err_vectors;
7728                 }
7729         }
7730         return ret;
7731
7732 err_vectors:
7733         kfree(vsi->tx_rings);
7734         return ret;
7735 }
7736
7737 /**
7738  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
7739  * @pf: board private structure
7740  * @type: type of VSI
7741  *
7742  * On error: returns error code (negative)
7743  * On success: returns vsi index in PF (positive)
7744  **/
7745 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
7746 {
7747         int ret = -ENODEV;
7748         struct i40e_vsi *vsi;
7749         int vsi_idx;
7750         int i;
7751
7752         /* Need to protect the allocation of the VSIs at the PF level */
7753         mutex_lock(&pf->switch_mutex);
7754
7755         /* VSI list may be fragmented if VSI creation/destruction has
7756          * been happening.  We can afford to do a quick scan to look
7757          * for any free VSIs in the list.
7758          *
7759          * find next empty vsi slot, looping back around if necessary
7760          */
7761         i = pf->next_vsi;
7762         while (i < pf->num_alloc_vsi && pf->vsi[i])
7763                 i++;
7764         if (i >= pf->num_alloc_vsi) {
7765                 i = 0;
7766                 while (i < pf->next_vsi && pf->vsi[i])
7767                         i++;
7768         }
7769
7770         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
7771                 vsi_idx = i;             /* Found one! */
7772         } else {
7773                 ret = -ENODEV;
7774                 goto unlock_pf;  /* out of VSI slots! */
7775         }
7776         pf->next_vsi = ++i;
7777
7778         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
7779         if (!vsi) {
7780                 ret = -ENOMEM;
7781                 goto unlock_pf;
7782         }
7783         vsi->type = type;
7784         vsi->back = pf;
7785         set_bit(__I40E_VSI_DOWN, vsi->state);
7786         vsi->flags = 0;
7787         vsi->idx = vsi_idx;
7788         vsi->int_rate_limit = 0;
7789         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
7790                                 pf->rss_table_size : 64;
7791         vsi->netdev_registered = false;
7792         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
7793         hash_init(vsi->mac_filter_hash);
7794         vsi->irqs_ready = false;
7795
7796         ret = i40e_set_num_rings_in_vsi(vsi);
7797         if (ret)
7798                 goto err_rings;
7799
7800         ret = i40e_vsi_alloc_arrays(vsi, true);
7801         if (ret)
7802                 goto err_rings;
7803
7804         /* Setup default MSIX irq handler for VSI */
7805         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
7806
7807         /* Initialize VSI lock */
7808         spin_lock_init(&vsi->mac_filter_hash_lock);
7809         pf->vsi[vsi_idx] = vsi;
7810         ret = vsi_idx;
7811         goto unlock_pf;
7812
7813 err_rings:
7814         pf->next_vsi = i - 1;
7815         kfree(vsi);
7816 unlock_pf:
7817         mutex_unlock(&pf->switch_mutex);
7818         return ret;
7819 }
7820
7821 /**
7822  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
7823  * @type: VSI pointer
7824  * @free_qvectors: a bool to specify if q_vectors need to be freed.
7825  *
7826  * On error: returns error code (negative)
7827  * On success: returns 0
7828  **/
7829 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
7830 {
7831         /* free the ring and vector containers */
7832         if (free_qvectors) {
7833                 kfree(vsi->q_vectors);
7834                 vsi->q_vectors = NULL;
7835         }
7836         kfree(vsi->tx_rings);
7837         vsi->tx_rings = NULL;
7838         vsi->rx_rings = NULL;
7839         vsi->xdp_rings = NULL;
7840 }
7841
7842 /**
7843  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
7844  * and lookup table
7845  * @vsi: Pointer to VSI structure
7846  */
7847 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
7848 {
7849         if (!vsi)
7850                 return;
7851
7852         kfree(vsi->rss_hkey_user);
7853         vsi->rss_hkey_user = NULL;
7854
7855         kfree(vsi->rss_lut_user);
7856         vsi->rss_lut_user = NULL;
7857 }
7858
7859 /**
7860  * i40e_vsi_clear - Deallocate the VSI provided
7861  * @vsi: the VSI being un-configured
7862  **/
7863 static int i40e_vsi_clear(struct i40e_vsi *vsi)
7864 {
7865         struct i40e_pf *pf;
7866
7867         if (!vsi)
7868                 return 0;
7869
7870         if (!vsi->back)
7871                 goto free_vsi;
7872         pf = vsi->back;
7873
7874         mutex_lock(&pf->switch_mutex);
7875         if (!pf->vsi[vsi->idx]) {
7876                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
7877                         vsi->idx, vsi->idx, vsi, vsi->type);
7878                 goto unlock_vsi;
7879         }
7880
7881         if (pf->vsi[vsi->idx] != vsi) {
7882                 dev_err(&pf->pdev->dev,
7883                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
7884                         pf->vsi[vsi->idx]->idx,
7885                         pf->vsi[vsi->idx],
7886                         pf->vsi[vsi->idx]->type,
7887                         vsi->idx, vsi, vsi->type);
7888                 goto unlock_vsi;
7889         }
7890
7891         /* updates the PF for this cleared vsi */
7892         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
7893         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
7894
7895         i40e_vsi_free_arrays(vsi, true);
7896         i40e_clear_rss_config_user(vsi);
7897
7898         pf->vsi[vsi->idx] = NULL;
7899         if (vsi->idx < pf->next_vsi)
7900                 pf->next_vsi = vsi->idx;
7901
7902 unlock_vsi:
7903         mutex_unlock(&pf->switch_mutex);
7904 free_vsi:
7905         kfree(vsi);
7906
7907         return 0;
7908 }
7909
7910 /**
7911  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
7912  * @vsi: the VSI being cleaned
7913  **/
7914 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
7915 {
7916         int i;
7917
7918         if (vsi->tx_rings && vsi->tx_rings[0]) {
7919                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7920                         kfree_rcu(vsi->tx_rings[i], rcu);
7921                         vsi->tx_rings[i] = NULL;
7922                         vsi->rx_rings[i] = NULL;
7923                         if (vsi->xdp_rings)
7924                                 vsi->xdp_rings[i] = NULL;
7925                 }
7926         }
7927 }
7928
7929 /**
7930  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
7931  * @vsi: the VSI being configured
7932  **/
7933 static int i40e_alloc_rings(struct i40e_vsi *vsi)
7934 {
7935         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
7936         struct i40e_pf *pf = vsi->back;
7937         struct i40e_ring *ring;
7938
7939         /* Set basic values in the rings to be used later during open() */
7940         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7941                 /* allocate space for both Tx and Rx in one shot */
7942                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
7943                 if (!ring)
7944                         goto err_out;
7945
7946                 ring->queue_index = i;
7947                 ring->reg_idx = vsi->base_queue + i;
7948                 ring->ring_active = false;
7949                 ring->vsi = vsi;
7950                 ring->netdev = vsi->netdev;
7951                 ring->dev = &pf->pdev->dev;
7952                 ring->count = vsi->num_desc;
7953                 ring->size = 0;
7954                 ring->dcb_tc = 0;
7955                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
7956                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7957                 ring->tx_itr_setting = pf->tx_itr_default;
7958                 vsi->tx_rings[i] = ring++;
7959
7960                 if (!i40e_enabled_xdp_vsi(vsi))
7961                         goto setup_rx;
7962
7963                 ring->queue_index = vsi->alloc_queue_pairs + i;
7964                 ring->reg_idx = vsi->base_queue + ring->queue_index;
7965                 ring->ring_active = false;
7966                 ring->vsi = vsi;
7967                 ring->netdev = NULL;
7968                 ring->dev = &pf->pdev->dev;
7969                 ring->count = vsi->num_desc;
7970                 ring->size = 0;
7971                 ring->dcb_tc = 0;
7972                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
7973                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7974                 set_ring_xdp(ring);
7975                 ring->tx_itr_setting = pf->tx_itr_default;
7976                 vsi->xdp_rings[i] = ring++;
7977
7978 setup_rx:
7979                 ring->queue_index = i;
7980                 ring->reg_idx = vsi->base_queue + i;
7981                 ring->ring_active = false;
7982                 ring->vsi = vsi;
7983                 ring->netdev = vsi->netdev;
7984                 ring->dev = &pf->pdev->dev;
7985                 ring->count = vsi->num_desc;
7986                 ring->size = 0;
7987                 ring->dcb_tc = 0;
7988                 ring->rx_itr_setting = pf->rx_itr_default;
7989                 vsi->rx_rings[i] = ring;
7990         }
7991
7992         return 0;
7993
7994 err_out:
7995         i40e_vsi_clear_rings(vsi);
7996         return -ENOMEM;
7997 }
7998
7999 /**
8000  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
8001  * @pf: board private structure
8002  * @vectors: the number of MSI-X vectors to request
8003  *
8004  * Returns the number of vectors reserved, or error
8005  **/
8006 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
8007 {
8008         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
8009                                         I40E_MIN_MSIX, vectors);
8010         if (vectors < 0) {
8011                 dev_info(&pf->pdev->dev,
8012                          "MSI-X vector reservation failed: %d\n", vectors);
8013                 vectors = 0;
8014         }
8015
8016         return vectors;
8017 }
8018
8019 /**
8020  * i40e_init_msix - Setup the MSIX capability
8021  * @pf: board private structure
8022  *
8023  * Work with the OS to set up the MSIX vectors needed.
8024  *
8025  * Returns the number of vectors reserved or negative on failure
8026  **/
8027 static int i40e_init_msix(struct i40e_pf *pf)
8028 {
8029         struct i40e_hw *hw = &pf->hw;
8030         int cpus, extra_vectors;
8031         int vectors_left;
8032         int v_budget, i;
8033         int v_actual;
8034         int iwarp_requested = 0;
8035
8036         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
8037                 return -ENODEV;
8038
8039         /* The number of vectors we'll request will be comprised of:
8040          *   - Add 1 for "other" cause for Admin Queue events, etc.
8041          *   - The number of LAN queue pairs
8042          *      - Queues being used for RSS.
8043          *              We don't need as many as max_rss_size vectors.
8044          *              use rss_size instead in the calculation since that
8045          *              is governed by number of cpus in the system.
8046          *      - assumes symmetric Tx/Rx pairing
8047          *   - The number of VMDq pairs
8048          *   - The CPU count within the NUMA node if iWARP is enabled
8049          * Once we count this up, try the request.
8050          *
8051          * If we can't get what we want, we'll simplify to nearly nothing
8052          * and try again.  If that still fails, we punt.
8053          */
8054         vectors_left = hw->func_caps.num_msix_vectors;
8055         v_budget = 0;
8056
8057         /* reserve one vector for miscellaneous handler */
8058         if (vectors_left) {
8059                 v_budget++;
8060                 vectors_left--;
8061         }
8062
8063         /* reserve some vectors for the main PF traffic queues. Initially we
8064          * only reserve at most 50% of the available vectors, in the case that
8065          * the number of online CPUs is large. This ensures that we can enable
8066          * extra features as well. Once we've enabled the other features, we
8067          * will use any remaining vectors to reach as close as we can to the
8068          * number of online CPUs.
8069          */
8070         cpus = num_online_cpus();
8071         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
8072         vectors_left -= pf->num_lan_msix;
8073
8074         /* reserve one vector for sideband flow director */
8075         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8076                 if (vectors_left) {
8077                         pf->num_fdsb_msix = 1;
8078                         v_budget++;
8079                         vectors_left--;
8080                 } else {
8081                         pf->num_fdsb_msix = 0;
8082                 }
8083         }
8084
8085         /* can we reserve enough for iWARP? */
8086         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8087                 iwarp_requested = pf->num_iwarp_msix;
8088
8089                 if (!vectors_left)
8090                         pf->num_iwarp_msix = 0;
8091                 else if (vectors_left < pf->num_iwarp_msix)
8092                         pf->num_iwarp_msix = 1;
8093                 v_budget += pf->num_iwarp_msix;
8094                 vectors_left -= pf->num_iwarp_msix;
8095         }
8096
8097         /* any vectors left over go for VMDq support */
8098         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
8099                 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps;
8100                 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted);
8101
8102                 if (!vectors_left) {
8103                         pf->num_vmdq_msix = 0;
8104                         pf->num_vmdq_qps = 0;
8105                 } else {
8106                         /* if we're short on vectors for what's desired, we limit
8107                          * the queues per vmdq.  If this is still more than are
8108                          * available, the user will need to change the number of
8109                          * queues/vectors used by the PF later with the ethtool
8110                          * channels command
8111                          */
8112                         if (vmdq_vecs < vmdq_vecs_wanted)
8113                                 pf->num_vmdq_qps = 1;
8114                         pf->num_vmdq_msix = pf->num_vmdq_qps;
8115
8116                         v_budget += vmdq_vecs;
8117                         vectors_left -= vmdq_vecs;
8118                 }
8119         }
8120
8121         /* On systems with a large number of SMP cores, we previously limited
8122          * the number of vectors for num_lan_msix to be at most 50% of the
8123          * available vectors, to allow for other features. Now, we add back
8124          * the remaining vectors. However, we ensure that the total
8125          * num_lan_msix will not exceed num_online_cpus(). To do this, we
8126          * calculate the number of vectors we can add without going over the
8127          * cap of CPUs. For systems with a small number of CPUs this will be
8128          * zero.
8129          */
8130         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
8131         pf->num_lan_msix += extra_vectors;
8132         vectors_left -= extra_vectors;
8133
8134         WARN(vectors_left < 0,
8135              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
8136
8137         v_budget += pf->num_lan_msix;
8138         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
8139                                    GFP_KERNEL);
8140         if (!pf->msix_entries)
8141                 return -ENOMEM;
8142
8143         for (i = 0; i < v_budget; i++)
8144                 pf->msix_entries[i].entry = i;
8145         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
8146
8147         if (v_actual < I40E_MIN_MSIX) {
8148                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
8149                 kfree(pf->msix_entries);
8150                 pf->msix_entries = NULL;
8151                 pci_disable_msix(pf->pdev);
8152                 return -ENODEV;
8153
8154         } else if (v_actual == I40E_MIN_MSIX) {
8155                 /* Adjust for minimal MSIX use */
8156                 pf->num_vmdq_vsis = 0;
8157                 pf->num_vmdq_qps = 0;
8158                 pf->num_lan_qps = 1;
8159                 pf->num_lan_msix = 1;
8160
8161         } else if (!vectors_left) {
8162                 /* If we have limited resources, we will start with no vectors
8163                  * for the special features and then allocate vectors to some
8164                  * of these features based on the policy and at the end disable
8165                  * the features that did not get any vectors.
8166                  */
8167                 int vec;
8168
8169                 dev_info(&pf->pdev->dev,
8170                          "MSI-X vector limit reached, attempting to redistribute vectors\n");
8171                 /* reserve the misc vector */
8172                 vec = v_actual - 1;
8173
8174                 /* Scale vector usage down */
8175                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
8176                 pf->num_vmdq_vsis = 1;
8177                 pf->num_vmdq_qps = 1;
8178
8179                 /* partition out the remaining vectors */
8180                 switch (vec) {
8181                 case 2:
8182                         pf->num_lan_msix = 1;
8183                         break;
8184                 case 3:
8185                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8186                                 pf->num_lan_msix = 1;
8187                                 pf->num_iwarp_msix = 1;
8188                         } else {
8189                                 pf->num_lan_msix = 2;
8190                         }
8191                         break;
8192                 default:
8193                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
8194                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
8195                                                  iwarp_requested);
8196                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
8197                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
8198                         } else {
8199                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
8200                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
8201                         }
8202                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8203                                 pf->num_fdsb_msix = 1;
8204                                 vec--;
8205                         }
8206                         pf->num_lan_msix = min_t(int,
8207                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
8208                                                               pf->num_lan_msix);
8209                         pf->num_lan_qps = pf->num_lan_msix;
8210                         break;
8211                 }
8212         }
8213
8214         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
8215             (pf->num_fdsb_msix == 0)) {
8216                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
8217                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8218         }
8219         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
8220             (pf->num_vmdq_msix == 0)) {
8221                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
8222                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
8223         }
8224
8225         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
8226             (pf->num_iwarp_msix == 0)) {
8227                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
8228                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
8229         }
8230         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
8231                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
8232                    pf->num_lan_msix,
8233                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
8234                    pf->num_fdsb_msix,
8235                    pf->num_iwarp_msix);
8236
8237         return v_actual;
8238 }
8239
8240 /**
8241  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
8242  * @vsi: the VSI being configured
8243  * @v_idx: index of the vector in the vsi struct
8244  * @cpu: cpu to be used on affinity_mask
8245  *
8246  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
8247  **/
8248 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
8249 {
8250         struct i40e_q_vector *q_vector;
8251
8252         /* allocate q_vector */
8253         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
8254         if (!q_vector)
8255                 return -ENOMEM;
8256
8257         q_vector->vsi = vsi;
8258         q_vector->v_idx = v_idx;
8259         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
8260
8261         if (vsi->netdev)
8262                 netif_napi_add(vsi->netdev, &q_vector->napi,
8263                                i40e_napi_poll, NAPI_POLL_WEIGHT);
8264
8265         q_vector->rx.latency_range = I40E_LOW_LATENCY;
8266         q_vector->tx.latency_range = I40E_LOW_LATENCY;
8267
8268         /* tie q_vector and vsi together */
8269         vsi->q_vectors[v_idx] = q_vector;
8270
8271         return 0;
8272 }
8273
8274 /**
8275  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
8276  * @vsi: the VSI being configured
8277  *
8278  * We allocate one q_vector per queue interrupt.  If allocation fails we
8279  * return -ENOMEM.
8280  **/
8281 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
8282 {
8283         struct i40e_pf *pf = vsi->back;
8284         int err, v_idx, num_q_vectors, current_cpu;
8285
8286         /* if not MSIX, give the one vector only to the LAN VSI */
8287         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
8288                 num_q_vectors = vsi->num_q_vectors;
8289         else if (vsi == pf->vsi[pf->lan_vsi])
8290                 num_q_vectors = 1;
8291         else
8292                 return -EINVAL;
8293
8294         current_cpu = cpumask_first(cpu_online_mask);
8295
8296         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
8297                 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
8298                 if (err)
8299                         goto err_out;
8300                 current_cpu = cpumask_next(current_cpu, cpu_online_mask);
8301                 if (unlikely(current_cpu >= nr_cpu_ids))
8302                         current_cpu = cpumask_first(cpu_online_mask);
8303         }
8304
8305         return 0;
8306
8307 err_out:
8308         while (v_idx--)
8309                 i40e_free_q_vector(vsi, v_idx);
8310
8311         return err;
8312 }
8313
8314 /**
8315  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
8316  * @pf: board private structure to initialize
8317  **/
8318 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
8319 {
8320         int vectors = 0;
8321         ssize_t size;
8322
8323         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
8324                 vectors = i40e_init_msix(pf);
8325                 if (vectors < 0) {
8326                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
8327                                        I40E_FLAG_IWARP_ENABLED  |
8328                                        I40E_FLAG_RSS_ENABLED    |
8329                                        I40E_FLAG_DCB_CAPABLE    |
8330                                        I40E_FLAG_DCB_ENABLED    |
8331                                        I40E_FLAG_SRIOV_ENABLED  |
8332                                        I40E_FLAG_FD_SB_ENABLED  |
8333                                        I40E_FLAG_FD_ATR_ENABLED |
8334                                        I40E_FLAG_VMDQ_ENABLED);
8335
8336                         /* rework the queue expectations without MSIX */
8337                         i40e_determine_queue_usage(pf);
8338                 }
8339         }
8340
8341         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
8342             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
8343                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
8344                 vectors = pci_enable_msi(pf->pdev);
8345                 if (vectors < 0) {
8346                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
8347                                  vectors);
8348                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
8349                 }
8350                 vectors = 1;  /* one MSI or Legacy vector */
8351         }
8352
8353         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
8354                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
8355
8356         /* set up vector assignment tracking */
8357         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
8358         pf->irq_pile = kzalloc(size, GFP_KERNEL);
8359         if (!pf->irq_pile) {
8360                 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n");
8361                 return -ENOMEM;
8362         }
8363         pf->irq_pile->num_entries = vectors;
8364         pf->irq_pile->search_hint = 0;
8365
8366         /* track first vector for misc interrupts, ignore return */
8367         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
8368
8369         return 0;
8370 }
8371
8372 #ifdef CONFIG_PM
8373 /**
8374  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
8375  * @pf: private board data structure
8376  *
8377  * Restore the interrupt scheme that was cleared when we suspended the
8378  * device. This should be called during resume to re-allocate the q_vectors
8379  * and reacquire IRQs.
8380  */
8381 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
8382 {
8383         int err, i;
8384
8385         /* We cleared the MSI and MSI-X flags when disabling the old interrupt
8386          * scheme. We need to re-enabled them here in order to attempt to
8387          * re-acquire the MSI or MSI-X vectors
8388          */
8389         pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
8390
8391         err = i40e_init_interrupt_scheme(pf);
8392         if (err)
8393                 return err;
8394
8395         /* Now that we've re-acquired IRQs, we need to remap the vectors and
8396          * rings together again.
8397          */
8398         for (i = 0; i < pf->num_alloc_vsi; i++) {
8399                 if (pf->vsi[i]) {
8400                         err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
8401                         if (err)
8402                                 goto err_unwind;
8403                         i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
8404                 }
8405         }
8406
8407         err = i40e_setup_misc_vector(pf);
8408         if (err)
8409                 goto err_unwind;
8410
8411         return 0;
8412
8413 err_unwind:
8414         while (i--) {
8415                 if (pf->vsi[i])
8416                         i40e_vsi_free_q_vectors(pf->vsi[i]);
8417         }
8418
8419         return err;
8420 }
8421 #endif /* CONFIG_PM */
8422
8423 /**
8424  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
8425  * @pf: board private structure
8426  *
8427  * This sets up the handler for MSIX 0, which is used to manage the
8428  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
8429  * when in MSI or Legacy interrupt mode.
8430  **/
8431 static int i40e_setup_misc_vector(struct i40e_pf *pf)
8432 {
8433         struct i40e_hw *hw = &pf->hw;
8434         int err = 0;
8435
8436         /* Only request the IRQ once, the first time through. */
8437         if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
8438                 err = request_irq(pf->msix_entries[0].vector,
8439                                   i40e_intr, 0, pf->int_name, pf);
8440                 if (err) {
8441                         clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
8442                         dev_info(&pf->pdev->dev,
8443                                  "request_irq for %s failed: %d\n",
8444                                  pf->int_name, err);
8445                         return -EFAULT;
8446                 }
8447         }
8448
8449         i40e_enable_misc_int_causes(pf);
8450
8451         /* associate no queues to the misc vector */
8452         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
8453         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
8454
8455         i40e_flush(hw);
8456
8457         i40e_irq_dynamic_enable_icr0(pf, true);
8458
8459         return err;
8460 }
8461
8462 /**
8463  * i40e_config_rss_aq - Prepare for RSS using AQ commands
8464  * @vsi: vsi structure
8465  * @seed: RSS hash seed
8466  **/
8467 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8468                               u8 *lut, u16 lut_size)
8469 {
8470         struct i40e_pf *pf = vsi->back;
8471         struct i40e_hw *hw = &pf->hw;
8472         int ret = 0;
8473
8474         if (seed) {
8475                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
8476                         (struct i40e_aqc_get_set_rss_key_data *)seed;
8477                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
8478                 if (ret) {
8479                         dev_info(&pf->pdev->dev,
8480                                  "Cannot set RSS key, err %s aq_err %s\n",
8481                                  i40e_stat_str(hw, ret),
8482                                  i40e_aq_str(hw, hw->aq.asq_last_status));
8483                         return ret;
8484                 }
8485         }
8486         if (lut) {
8487                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8488
8489                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8490                 if (ret) {
8491                         dev_info(&pf->pdev->dev,
8492                                  "Cannot set RSS lut, err %s aq_err %s\n",
8493                                  i40e_stat_str(hw, ret),
8494                                  i40e_aq_str(hw, hw->aq.asq_last_status));
8495                         return ret;
8496                 }
8497         }
8498         return ret;
8499 }
8500
8501 /**
8502  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
8503  * @vsi: Pointer to vsi structure
8504  * @seed: Buffter to store the hash keys
8505  * @lut: Buffer to store the lookup table entries
8506  * @lut_size: Size of buffer to store the lookup table entries
8507  *
8508  * Return 0 on success, negative on failure
8509  */
8510 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8511                            u8 *lut, u16 lut_size)
8512 {
8513         struct i40e_pf *pf = vsi->back;
8514         struct i40e_hw *hw = &pf->hw;
8515         int ret = 0;
8516
8517         if (seed) {
8518                 ret = i40e_aq_get_rss_key(hw, vsi->id,
8519                         (struct i40e_aqc_get_set_rss_key_data *)seed);
8520                 if (ret) {
8521                         dev_info(&pf->pdev->dev,
8522                                  "Cannot get RSS key, err %s aq_err %s\n",
8523                                  i40e_stat_str(&pf->hw, ret),
8524                                  i40e_aq_str(&pf->hw,
8525                                              pf->hw.aq.asq_last_status));
8526                         return ret;
8527                 }
8528         }
8529
8530         if (lut) {
8531                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8532
8533                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8534                 if (ret) {
8535                         dev_info(&pf->pdev->dev,
8536                                  "Cannot get RSS lut, err %s aq_err %s\n",
8537                                  i40e_stat_str(&pf->hw, ret),
8538                                  i40e_aq_str(&pf->hw,
8539                                              pf->hw.aq.asq_last_status));
8540                         return ret;
8541                 }
8542         }
8543
8544         return ret;
8545 }
8546
8547 /**
8548  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
8549  * @vsi: VSI structure
8550  **/
8551 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
8552 {
8553         u8 seed[I40E_HKEY_ARRAY_SIZE];
8554         struct i40e_pf *pf = vsi->back;
8555         u8 *lut;
8556         int ret;
8557
8558         if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
8559                 return 0;
8560
8561         if (!vsi->rss_size)
8562                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
8563                                       vsi->num_queue_pairs);
8564         if (!vsi->rss_size)
8565                 return -EINVAL;
8566
8567         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8568         if (!lut)
8569                 return -ENOMEM;
8570         /* Use the user configured hash keys and lookup table if there is one,
8571          * otherwise use default
8572          */
8573         if (vsi->rss_lut_user)
8574                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8575         else
8576                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8577         if (vsi->rss_hkey_user)
8578                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8579         else
8580                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8581         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
8582         kfree(lut);
8583
8584         return ret;
8585 }
8586
8587 /**
8588  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
8589  * @vsi: Pointer to vsi structure
8590  * @seed: RSS hash seed
8591  * @lut: Lookup table
8592  * @lut_size: Lookup table size
8593  *
8594  * Returns 0 on success, negative on failure
8595  **/
8596 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
8597                                const u8 *lut, u16 lut_size)
8598 {
8599         struct i40e_pf *pf = vsi->back;
8600         struct i40e_hw *hw = &pf->hw;
8601         u16 vf_id = vsi->vf_id;
8602         u8 i;
8603
8604         /* Fill out hash function seed */
8605         if (seed) {
8606                 u32 *seed_dw = (u32 *)seed;
8607
8608                 if (vsi->type == I40E_VSI_MAIN) {
8609                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8610                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
8611                 } else if (vsi->type == I40E_VSI_SRIOV) {
8612                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
8613                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
8614                 } else {
8615                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
8616                 }
8617         }
8618
8619         if (lut) {
8620                 u32 *lut_dw = (u32 *)lut;
8621
8622                 if (vsi->type == I40E_VSI_MAIN) {
8623                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
8624                                 return -EINVAL;
8625                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8626                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
8627                 } else if (vsi->type == I40E_VSI_SRIOV) {
8628                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
8629                                 return -EINVAL;
8630                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
8631                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
8632                 } else {
8633                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
8634                 }
8635         }
8636         i40e_flush(hw);
8637
8638         return 0;
8639 }
8640
8641 /**
8642  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
8643  * @vsi: Pointer to VSI structure
8644  * @seed: Buffer to store the keys
8645  * @lut: Buffer to store the lookup table entries
8646  * @lut_size: Size of buffer to store the lookup table entries
8647  *
8648  * Returns 0 on success, negative on failure
8649  */
8650 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
8651                             u8 *lut, u16 lut_size)
8652 {
8653         struct i40e_pf *pf = vsi->back;
8654         struct i40e_hw *hw = &pf->hw;
8655         u16 i;
8656
8657         if (seed) {
8658                 u32 *seed_dw = (u32 *)seed;
8659
8660                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8661                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
8662         }
8663         if (lut) {
8664                 u32 *lut_dw = (u32 *)lut;
8665
8666                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
8667                         return -EINVAL;
8668                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8669                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
8670         }
8671
8672         return 0;
8673 }
8674
8675 /**
8676  * i40e_config_rss - Configure RSS keys and lut
8677  * @vsi: Pointer to VSI structure
8678  * @seed: RSS hash seed
8679  * @lut: Lookup table
8680  * @lut_size: Lookup table size
8681  *
8682  * Returns 0 on success, negative on failure
8683  */
8684 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8685 {
8686         struct i40e_pf *pf = vsi->back;
8687
8688         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
8689                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
8690         else
8691                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
8692 }
8693
8694 /**
8695  * i40e_get_rss - Get RSS keys and lut
8696  * @vsi: Pointer to VSI structure
8697  * @seed: Buffer to store the keys
8698  * @lut: Buffer to store the lookup table entries
8699  * lut_size: Size of buffer to store the lookup table entries
8700  *
8701  * Returns 0 on success, negative on failure
8702  */
8703 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8704 {
8705         struct i40e_pf *pf = vsi->back;
8706
8707         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
8708                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
8709         else
8710                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
8711 }
8712
8713 /**
8714  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
8715  * @pf: Pointer to board private structure
8716  * @lut: Lookup table
8717  * @rss_table_size: Lookup table size
8718  * @rss_size: Range of queue number for hashing
8719  */
8720 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
8721                        u16 rss_table_size, u16 rss_size)
8722 {
8723         u16 i;
8724
8725         for (i = 0; i < rss_table_size; i++)
8726                 lut[i] = i % rss_size;
8727 }
8728
8729 /**
8730  * i40e_pf_config_rss - Prepare for RSS if used
8731  * @pf: board private structure
8732  **/
8733 static int i40e_pf_config_rss(struct i40e_pf *pf)
8734 {
8735         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8736         u8 seed[I40E_HKEY_ARRAY_SIZE];
8737         u8 *lut;
8738         struct i40e_hw *hw = &pf->hw;
8739         u32 reg_val;
8740         u64 hena;
8741         int ret;
8742
8743         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
8744         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
8745                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
8746         hena |= i40e_pf_get_default_rss_hena(pf);
8747
8748         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
8749         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
8750
8751         /* Determine the RSS table size based on the hardware capabilities */
8752         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
8753         reg_val = (pf->rss_table_size == 512) ?
8754                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
8755                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
8756         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
8757
8758         /* Determine the RSS size of the VSI */
8759         if (!vsi->rss_size) {
8760                 u16 qcount;
8761
8762                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
8763                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
8764         }
8765         if (!vsi->rss_size)
8766                 return -EINVAL;
8767
8768         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8769         if (!lut)
8770                 return -ENOMEM;
8771
8772         /* Use user configured lut if there is one, otherwise use default */
8773         if (vsi->rss_lut_user)
8774                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8775         else
8776                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8777
8778         /* Use user configured hash key if there is one, otherwise
8779          * use default.
8780          */
8781         if (vsi->rss_hkey_user)
8782                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8783         else
8784                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8785         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
8786         kfree(lut);
8787
8788         return ret;
8789 }
8790
8791 /**
8792  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
8793  * @pf: board private structure
8794  * @queue_count: the requested queue count for rss.
8795  *
8796  * returns 0 if rss is not enabled, if enabled returns the final rss queue
8797  * count which may be different from the requested queue count.
8798  * Note: expects to be called while under rtnl_lock()
8799  **/
8800 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
8801 {
8802         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8803         int new_rss_size;
8804
8805         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
8806                 return 0;
8807
8808         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
8809
8810         if (queue_count != vsi->num_queue_pairs) {
8811                 u16 qcount;
8812
8813                 vsi->req_queue_pairs = queue_count;
8814                 i40e_prep_for_reset(pf, true);
8815
8816                 pf->alloc_rss_size = new_rss_size;
8817
8818                 i40e_reset_and_rebuild(pf, true, true);
8819
8820                 /* Discard the user configured hash keys and lut, if less
8821                  * queues are enabled.
8822                  */
8823                 if (queue_count < vsi->rss_size) {
8824                         i40e_clear_rss_config_user(vsi);
8825                         dev_dbg(&pf->pdev->dev,
8826                                 "discard user configured hash keys and lut\n");
8827                 }
8828
8829                 /* Reset vsi->rss_size, as number of enabled queues changed */
8830                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
8831                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
8832
8833                 i40e_pf_config_rss(pf);
8834         }
8835         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
8836                  vsi->req_queue_pairs, pf->rss_size_max);
8837         return pf->alloc_rss_size;
8838 }
8839
8840 /**
8841  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
8842  * @pf: board private structure
8843  **/
8844 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
8845 {
8846         i40e_status status;
8847         bool min_valid, max_valid;
8848         u32 max_bw, min_bw;
8849
8850         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
8851                                            &min_valid, &max_valid);
8852
8853         if (!status) {
8854                 if (min_valid)
8855                         pf->min_bw = min_bw;
8856                 if (max_valid)
8857                         pf->max_bw = max_bw;
8858         }
8859
8860         return status;
8861 }
8862
8863 /**
8864  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
8865  * @pf: board private structure
8866  **/
8867 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
8868 {
8869         struct i40e_aqc_configure_partition_bw_data bw_data;
8870         i40e_status status;
8871
8872         /* Set the valid bit for this PF */
8873         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
8874         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
8875         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
8876
8877         /* Set the new bandwidths */
8878         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
8879
8880         return status;
8881 }
8882
8883 /**
8884  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
8885  * @pf: board private structure
8886  **/
8887 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
8888 {
8889         /* Commit temporary BW setting to permanent NVM image */
8890         enum i40e_admin_queue_err last_aq_status;
8891         i40e_status ret;
8892         u16 nvm_word;
8893
8894         if (pf->hw.partition_id != 1) {
8895                 dev_info(&pf->pdev->dev,
8896                          "Commit BW only works on partition 1! This is partition %d",
8897                          pf->hw.partition_id);
8898                 ret = I40E_NOT_SUPPORTED;
8899                 goto bw_commit_out;
8900         }
8901
8902         /* Acquire NVM for read access */
8903         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
8904         last_aq_status = pf->hw.aq.asq_last_status;
8905         if (ret) {
8906                 dev_info(&pf->pdev->dev,
8907                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
8908                          i40e_stat_str(&pf->hw, ret),
8909                          i40e_aq_str(&pf->hw, last_aq_status));
8910                 goto bw_commit_out;
8911         }
8912
8913         /* Read word 0x10 of NVM - SW compatibility word 1 */
8914         ret = i40e_aq_read_nvm(&pf->hw,
8915                                I40E_SR_NVM_CONTROL_WORD,
8916                                0x10, sizeof(nvm_word), &nvm_word,
8917                                false, NULL);
8918         /* Save off last admin queue command status before releasing
8919          * the NVM
8920          */
8921         last_aq_status = pf->hw.aq.asq_last_status;
8922         i40e_release_nvm(&pf->hw);
8923         if (ret) {
8924                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
8925                          i40e_stat_str(&pf->hw, ret),
8926                          i40e_aq_str(&pf->hw, last_aq_status));
8927                 goto bw_commit_out;
8928         }
8929
8930         /* Wait a bit for NVM release to complete */
8931         msleep(50);
8932
8933         /* Acquire NVM for write access */
8934         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
8935         last_aq_status = pf->hw.aq.asq_last_status;
8936         if (ret) {
8937                 dev_info(&pf->pdev->dev,
8938                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
8939                          i40e_stat_str(&pf->hw, ret),
8940                          i40e_aq_str(&pf->hw, last_aq_status));
8941                 goto bw_commit_out;
8942         }
8943         /* Write it back out unchanged to initiate update NVM,
8944          * which will force a write of the shadow (alt) RAM to
8945          * the NVM - thus storing the bandwidth values permanently.
8946          */
8947         ret = i40e_aq_update_nvm(&pf->hw,
8948                                  I40E_SR_NVM_CONTROL_WORD,
8949                                  0x10, sizeof(nvm_word),
8950                                  &nvm_word, true, NULL);
8951         /* Save off last admin queue command status before releasing
8952          * the NVM
8953          */
8954         last_aq_status = pf->hw.aq.asq_last_status;
8955         i40e_release_nvm(&pf->hw);
8956         if (ret)
8957                 dev_info(&pf->pdev->dev,
8958                          "BW settings NOT SAVED, err %s aq_err %s\n",
8959                          i40e_stat_str(&pf->hw, ret),
8960                          i40e_aq_str(&pf->hw, last_aq_status));
8961 bw_commit_out:
8962
8963         return ret;
8964 }
8965
8966 /**
8967  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
8968  * @pf: board private structure to initialize
8969  *
8970  * i40e_sw_init initializes the Adapter private data structure.
8971  * Fields are initialized based on PCI device information and
8972  * OS network device settings (MTU size).
8973  **/
8974 static int i40e_sw_init(struct i40e_pf *pf)
8975 {
8976         int err = 0;
8977         int size;
8978
8979         /* Set default capability flags */
8980         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
8981                     I40E_FLAG_MSI_ENABLED     |
8982                     I40E_FLAG_MSIX_ENABLED;
8983
8984         /* Set default ITR */
8985         pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF;
8986         pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF;
8987
8988         /* Depending on PF configurations, it is possible that the RSS
8989          * maximum might end up larger than the available queues
8990          */
8991         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
8992         pf->alloc_rss_size = 1;
8993         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
8994         pf->rss_size_max = min_t(int, pf->rss_size_max,
8995                                  pf->hw.func_caps.num_tx_qp);
8996         if (pf->hw.func_caps.rss) {
8997                 pf->flags |= I40E_FLAG_RSS_ENABLED;
8998                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
8999                                            num_online_cpus());
9000         }
9001
9002         /* MFP mode enabled */
9003         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
9004                 pf->flags |= I40E_FLAG_MFP_ENABLED;
9005                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
9006                 if (i40e_get_partition_bw_setting(pf)) {
9007                         dev_warn(&pf->pdev->dev,
9008                                  "Could not get partition bw settings\n");
9009                 } else {
9010                         dev_info(&pf->pdev->dev,
9011                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
9012                                  pf->min_bw, pf->max_bw);
9013
9014                         /* nudge the Tx scheduler */
9015                         i40e_set_partition_bw_setting(pf);
9016                 }
9017         }
9018
9019         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
9020             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
9021                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
9022                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
9023                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
9024                     pf->hw.num_partitions > 1)
9025                         dev_info(&pf->pdev->dev,
9026                                  "Flow Director Sideband mode Disabled in MFP mode\n");
9027                 else
9028                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
9029                 pf->fdir_pf_filter_count =
9030                                  pf->hw.func_caps.fd_filters_guaranteed;
9031                 pf->hw.fdir_shared_filter_count =
9032                                  pf->hw.func_caps.fd_filters_best_effort;
9033         }
9034
9035         if (pf->hw.mac.type == I40E_MAC_X722) {
9036                 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
9037                                     I40E_HW_128_QP_RSS_CAPABLE |
9038                                     I40E_HW_ATR_EVICT_CAPABLE |
9039                                     I40E_HW_WB_ON_ITR_CAPABLE |
9040                                     I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
9041                                     I40E_HW_NO_PCI_LINK_CHECK |
9042                                     I40E_HW_USE_SET_LLDP_MIB |
9043                                     I40E_HW_GENEVE_OFFLOAD_CAPABLE |
9044                                     I40E_HW_PTP_L4_CAPABLE |
9045                                     I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
9046                                     I40E_HW_OUTER_UDP_CSUM_CAPABLE);
9047
9048 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
9049                 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
9050                     I40E_FDEVICT_PCTYPE_DEFAULT) {
9051                         dev_warn(&pf->pdev->dev,
9052                                  "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
9053                         pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
9054                 }
9055         } else if ((pf->hw.aq.api_maj_ver > 1) ||
9056                    ((pf->hw.aq.api_maj_ver == 1) &&
9057                     (pf->hw.aq.api_min_ver > 4))) {
9058                 /* Supported in FW API version higher than 1.4 */
9059                 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
9060         }
9061
9062         /* Enable HW ATR eviction if possible */
9063         if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
9064                 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
9065
9066         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
9067             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
9068             (pf->hw.aq.fw_maj_ver < 4))) {
9069                 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
9070                 /* No DCB support  for FW < v4.33 */
9071                 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
9072         }
9073
9074         /* Disable FW LLDP if FW < v4.3 */
9075         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
9076             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
9077             (pf->hw.aq.fw_maj_ver < 4)))
9078                 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
9079
9080         /* Use the FW Set LLDP MIB API if FW > v4.40 */
9081         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
9082             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
9083             (pf->hw.aq.fw_maj_ver >= 5)))
9084                 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
9085
9086         /* Enable PTP L4 if FW > v6.0 */
9087         if (pf->hw.mac.type == I40E_MAC_XL710 &&
9088             pf->hw.aq.fw_maj_ver >= 6)
9089                 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
9090
9091         if (pf->hw.func_caps.vmdq) {
9092                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
9093                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
9094                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
9095         }
9096
9097         if (pf->hw.func_caps.iwarp) {
9098                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
9099                 /* IWARP needs one extra vector for CQP just like MISC.*/
9100                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
9101         }
9102
9103 #ifdef CONFIG_PCI_IOV
9104         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
9105                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
9106                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
9107                 pf->num_req_vfs = min_t(int,
9108                                         pf->hw.func_caps.num_vfs,
9109                                         I40E_MAX_VF_COUNT);
9110         }
9111 #endif /* CONFIG_PCI_IOV */
9112         pf->eeprom_version = 0xDEAD;
9113         pf->lan_veb = I40E_NO_VEB;
9114         pf->lan_vsi = I40E_NO_VSI;
9115
9116         /* By default FW has this off for performance reasons */
9117         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
9118
9119         /* set up queue assignment tracking */
9120         size = sizeof(struct i40e_lump_tracking)
9121                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
9122         pf->qp_pile = kzalloc(size, GFP_KERNEL);
9123         if (!pf->qp_pile) {
9124                 err = -ENOMEM;
9125                 goto sw_init_done;
9126         }
9127         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
9128         pf->qp_pile->search_hint = 0;
9129
9130         pf->tx_timeout_recovery_level = 1;
9131
9132         mutex_init(&pf->switch_mutex);
9133
9134 sw_init_done:
9135         return err;
9136 }
9137
9138 /**
9139  * i40e_set_ntuple - set the ntuple feature flag and take action
9140  * @pf: board private structure to initialize
9141  * @features: the feature set that the stack is suggesting
9142  *
9143  * returns a bool to indicate if reset needs to happen
9144  **/
9145 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
9146 {
9147         bool need_reset = false;
9148
9149         /* Check if Flow Director n-tuple support was enabled or disabled.  If
9150          * the state changed, we need to reset.
9151          */
9152         if (features & NETIF_F_NTUPLE) {
9153                 /* Enable filters and mark for reset */
9154                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
9155                         need_reset = true;
9156                 /* enable FD_SB only if there is MSI-X vector */
9157                 if (pf->num_fdsb_msix > 0)
9158                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
9159         } else {
9160                 /* turn off filters, mark for reset and clear SW filter list */
9161                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
9162                         need_reset = true;
9163                         i40e_fdir_filter_exit(pf);
9164                 }
9165                 pf->flags &= ~(I40E_FLAG_FD_SB_ENABLED |
9166                                I40E_FLAG_FD_SB_AUTO_DISABLED);
9167                 /* reset fd counters */
9168                 pf->fd_add_err = 0;
9169                 pf->fd_atr_cnt = 0;
9170                 /* if ATR was auto disabled it can be re-enabled. */
9171                 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) {
9172                         pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED;
9173                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
9174                             (I40E_DEBUG_FD & pf->hw.debug_mask))
9175                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
9176                 }
9177         }
9178         return need_reset;
9179 }
9180
9181 /**
9182  * i40e_clear_rss_lut - clear the rx hash lookup table
9183  * @vsi: the VSI being configured
9184  **/
9185 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
9186 {
9187         struct i40e_pf *pf = vsi->back;
9188         struct i40e_hw *hw = &pf->hw;
9189         u16 vf_id = vsi->vf_id;
9190         u8 i;
9191
9192         if (vsi->type == I40E_VSI_MAIN) {
9193                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
9194                         wr32(hw, I40E_PFQF_HLUT(i), 0);
9195         } else if (vsi->type == I40E_VSI_SRIOV) {
9196                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
9197                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
9198         } else {
9199                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
9200         }
9201 }
9202
9203 /**
9204  * i40e_set_features - set the netdev feature flags
9205  * @netdev: ptr to the netdev being adjusted
9206  * @features: the feature set that the stack is suggesting
9207  * Note: expects to be called while under rtnl_lock()
9208  **/
9209 static int i40e_set_features(struct net_device *netdev,
9210                              netdev_features_t features)
9211 {
9212         struct i40e_netdev_priv *np = netdev_priv(netdev);
9213         struct i40e_vsi *vsi = np->vsi;
9214         struct i40e_pf *pf = vsi->back;
9215         bool need_reset;
9216
9217         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
9218                 i40e_pf_config_rss(pf);
9219         else if (!(features & NETIF_F_RXHASH) &&
9220                  netdev->features & NETIF_F_RXHASH)
9221                 i40e_clear_rss_lut(vsi);
9222
9223         if (features & NETIF_F_HW_VLAN_CTAG_RX)
9224                 i40e_vlan_stripping_enable(vsi);
9225         else
9226                 i40e_vlan_stripping_disable(vsi);
9227
9228         need_reset = i40e_set_ntuple(pf, features);
9229
9230         if (need_reset)
9231                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true);
9232
9233         return 0;
9234 }
9235
9236 /**
9237  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
9238  * @pf: board private structure
9239  * @port: The UDP port to look up
9240  *
9241  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
9242  **/
9243 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
9244 {
9245         u8 i;
9246
9247         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
9248                 if (pf->udp_ports[i].port == port)
9249                         return i;
9250         }
9251
9252         return i;
9253 }
9254
9255 /**
9256  * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
9257  * @netdev: This physical port's netdev
9258  * @ti: Tunnel endpoint information
9259  **/
9260 static void i40e_udp_tunnel_add(struct net_device *netdev,
9261                                 struct udp_tunnel_info *ti)
9262 {
9263         struct i40e_netdev_priv *np = netdev_priv(netdev);
9264         struct i40e_vsi *vsi = np->vsi;
9265         struct i40e_pf *pf = vsi->back;
9266         u16 port = ntohs(ti->port);
9267         u8 next_idx;
9268         u8 idx;
9269
9270         idx = i40e_get_udp_port_idx(pf, port);
9271
9272         /* Check if port already exists */
9273         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
9274                 netdev_info(netdev, "port %d already offloaded\n", port);
9275                 return;
9276         }
9277
9278         /* Now check if there is space to add the new port */
9279         next_idx = i40e_get_udp_port_idx(pf, 0);
9280
9281         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
9282                 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
9283                             port);
9284                 return;
9285         }
9286
9287         switch (ti->type) {
9288         case UDP_TUNNEL_TYPE_VXLAN:
9289                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
9290                 break;
9291         case UDP_TUNNEL_TYPE_GENEVE:
9292                 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE))
9293                         return;
9294                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
9295                 break;
9296         default:
9297                 return;
9298         }
9299
9300         /* New port: add it and mark its index in the bitmap */
9301         pf->udp_ports[next_idx].port = port;
9302         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
9303         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
9304 }
9305
9306 /**
9307  * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
9308  * @netdev: This physical port's netdev
9309  * @ti: Tunnel endpoint information
9310  **/
9311 static void i40e_udp_tunnel_del(struct net_device *netdev,
9312                                 struct udp_tunnel_info *ti)
9313 {
9314         struct i40e_netdev_priv *np = netdev_priv(netdev);
9315         struct i40e_vsi *vsi = np->vsi;
9316         struct i40e_pf *pf = vsi->back;
9317         u16 port = ntohs(ti->port);
9318         u8 idx;
9319
9320         idx = i40e_get_udp_port_idx(pf, port);
9321
9322         /* Check if port already exists */
9323         if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
9324                 goto not_found;
9325
9326         switch (ti->type) {
9327         case UDP_TUNNEL_TYPE_VXLAN:
9328                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
9329                         goto not_found;
9330                 break;
9331         case UDP_TUNNEL_TYPE_GENEVE:
9332                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
9333                         goto not_found;
9334                 break;
9335         default:
9336                 goto not_found;
9337         }
9338
9339         /* if port exists, set it to 0 (mark for deletion)
9340          * and make it pending
9341          */
9342         pf->udp_ports[idx].port = 0;
9343         pf->pending_udp_bitmap |= BIT_ULL(idx);
9344         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
9345
9346         return;
9347 not_found:
9348         netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
9349                     port);
9350 }
9351
9352 static int i40e_get_phys_port_id(struct net_device *netdev,
9353                                  struct netdev_phys_item_id *ppid)
9354 {
9355         struct i40e_netdev_priv *np = netdev_priv(netdev);
9356         struct i40e_pf *pf = np->vsi->back;
9357         struct i40e_hw *hw = &pf->hw;
9358
9359         if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
9360                 return -EOPNOTSUPP;
9361
9362         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
9363         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
9364
9365         return 0;
9366 }
9367
9368 /**
9369  * i40e_ndo_fdb_add - add an entry to the hardware database
9370  * @ndm: the input from the stack
9371  * @tb: pointer to array of nladdr (unused)
9372  * @dev: the net device pointer
9373  * @addr: the MAC address entry being added
9374  * @flags: instructions from stack about fdb operation
9375  */
9376 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
9377                             struct net_device *dev,
9378                             const unsigned char *addr, u16 vid,
9379                             u16 flags)
9380 {
9381         struct i40e_netdev_priv *np = netdev_priv(dev);
9382         struct i40e_pf *pf = np->vsi->back;
9383         int err = 0;
9384
9385         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
9386                 return -EOPNOTSUPP;
9387
9388         if (vid) {
9389                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
9390                 return -EINVAL;
9391         }
9392
9393         /* Hardware does not support aging addresses so if a
9394          * ndm_state is given only allow permanent addresses
9395          */
9396         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
9397                 netdev_info(dev, "FDB only supports static addresses\n");
9398                 return -EINVAL;
9399         }
9400
9401         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
9402                 err = dev_uc_add_excl(dev, addr);
9403         else if (is_multicast_ether_addr(addr))
9404                 err = dev_mc_add_excl(dev, addr);
9405         else
9406                 err = -EINVAL;
9407
9408         /* Only return duplicate errors if NLM_F_EXCL is set */
9409         if (err == -EEXIST && !(flags & NLM_F_EXCL))
9410                 err = 0;
9411
9412         return err;
9413 }
9414
9415 /**
9416  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
9417  * @dev: the netdev being configured
9418  * @nlh: RTNL message
9419  *
9420  * Inserts a new hardware bridge if not already created and
9421  * enables the bridging mode requested (VEB or VEPA). If the
9422  * hardware bridge has already been inserted and the request
9423  * is to change the mode then that requires a PF reset to
9424  * allow rebuild of the components with required hardware
9425  * bridge mode enabled.
9426  *
9427  * Note: expects to be called while under rtnl_lock()
9428  **/
9429 static int i40e_ndo_bridge_setlink(struct net_device *dev,
9430                                    struct nlmsghdr *nlh,
9431                                    u16 flags)
9432 {
9433         struct i40e_netdev_priv *np = netdev_priv(dev);
9434         struct i40e_vsi *vsi = np->vsi;
9435         struct i40e_pf *pf = vsi->back;
9436         struct i40e_veb *veb = NULL;
9437         struct nlattr *attr, *br_spec;
9438         int i, rem;
9439
9440         /* Only for PF VSI for now */
9441         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9442                 return -EOPNOTSUPP;
9443
9444         /* Find the HW bridge for PF VSI */
9445         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9446                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9447                         veb = pf->veb[i];
9448         }
9449
9450         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
9451
9452         nla_for_each_nested(attr, br_spec, rem) {
9453                 __u16 mode;
9454
9455                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
9456                         continue;
9457
9458                 mode = nla_get_u16(attr);
9459                 if ((mode != BRIDGE_MODE_VEPA) &&
9460                     (mode != BRIDGE_MODE_VEB))
9461                         return -EINVAL;
9462
9463                 /* Insert a new HW bridge */
9464                 if (!veb) {
9465                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
9466                                              vsi->tc_config.enabled_tc);
9467                         if (veb) {
9468                                 veb->bridge_mode = mode;
9469                                 i40e_config_bridge_mode(veb);
9470                         } else {
9471                                 /* No Bridge HW offload available */
9472                                 return -ENOENT;
9473                         }
9474                         break;
9475                 } else if (mode != veb->bridge_mode) {
9476                         /* Existing HW bridge but different mode needs reset */
9477                         veb->bridge_mode = mode;
9478                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
9479                         if (mode == BRIDGE_MODE_VEB)
9480                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
9481                         else
9482                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
9483                         i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED),
9484                                       true);
9485                         break;
9486                 }
9487         }
9488
9489         return 0;
9490 }
9491
9492 /**
9493  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
9494  * @skb: skb buff
9495  * @pid: process id
9496  * @seq: RTNL message seq #
9497  * @dev: the netdev being configured
9498  * @filter_mask: unused
9499  * @nlflags: netlink flags passed in
9500  *
9501  * Return the mode in which the hardware bridge is operating in
9502  * i.e VEB or VEPA.
9503  **/
9504 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
9505                                    struct net_device *dev,
9506                                    u32 __always_unused filter_mask,
9507                                    int nlflags)
9508 {
9509         struct i40e_netdev_priv *np = netdev_priv(dev);
9510         struct i40e_vsi *vsi = np->vsi;
9511         struct i40e_pf *pf = vsi->back;
9512         struct i40e_veb *veb = NULL;
9513         int i;
9514
9515         /* Only for PF VSI for now */
9516         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9517                 return -EOPNOTSUPP;
9518
9519         /* Find the HW bridge for the PF VSI */
9520         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9521                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9522                         veb = pf->veb[i];
9523         }
9524
9525         if (!veb)
9526                 return 0;
9527
9528         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
9529                                        0, 0, nlflags, filter_mask, NULL);
9530 }
9531
9532 /**
9533  * i40e_features_check - Validate encapsulated packet conforms to limits
9534  * @skb: skb buff
9535  * @dev: This physical port's netdev
9536  * @features: Offload features that the stack believes apply
9537  **/
9538 static netdev_features_t i40e_features_check(struct sk_buff *skb,
9539                                              struct net_device *dev,
9540                                              netdev_features_t features)
9541 {
9542         size_t len;
9543
9544         /* No point in doing any of this if neither checksum nor GSO are
9545          * being requested for this frame.  We can rule out both by just
9546          * checking for CHECKSUM_PARTIAL
9547          */
9548         if (skb->ip_summed != CHECKSUM_PARTIAL)
9549                 return features;
9550
9551         /* We cannot support GSO if the MSS is going to be less than
9552          * 64 bytes.  If it is then we need to drop support for GSO.
9553          */
9554         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
9555                 features &= ~NETIF_F_GSO_MASK;
9556
9557         /* MACLEN can support at most 63 words */
9558         len = skb_network_header(skb) - skb->data;
9559         if (len & ~(63 * 2))
9560                 goto out_err;
9561
9562         /* IPLEN and EIPLEN can support at most 127 dwords */
9563         len = skb_transport_header(skb) - skb_network_header(skb);
9564         if (len & ~(127 * 4))
9565                 goto out_err;
9566
9567         if (skb->encapsulation) {
9568                 /* L4TUNLEN can support 127 words */
9569                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
9570                 if (len & ~(127 * 2))
9571                         goto out_err;
9572
9573                 /* IPLEN can support at most 127 dwords */
9574                 len = skb_inner_transport_header(skb) -
9575                       skb_inner_network_header(skb);
9576                 if (len & ~(127 * 4))
9577                         goto out_err;
9578         }
9579
9580         /* No need to validate L4LEN as TCP is the only protocol with a
9581          * a flexible value and we support all possible values supported
9582          * by TCP, which is at most 15 dwords
9583          */
9584
9585         return features;
9586 out_err:
9587         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
9588 }
9589
9590 /**
9591  * i40e_xdp_setup - add/remove an XDP program
9592  * @vsi: VSI to changed
9593  * @prog: XDP program
9594  **/
9595 static int i40e_xdp_setup(struct i40e_vsi *vsi,
9596                           struct bpf_prog *prog)
9597 {
9598         int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
9599         struct i40e_pf *pf = vsi->back;
9600         struct bpf_prog *old_prog;
9601         bool need_reset;
9602         int i;
9603
9604         /* Don't allow frames that span over multiple buffers */
9605         if (frame_size > vsi->rx_buf_len)
9606                 return -EINVAL;
9607
9608         if (!i40e_enabled_xdp_vsi(vsi) && !prog)
9609                 return 0;
9610
9611         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
9612         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
9613
9614         if (need_reset)
9615                 i40e_prep_for_reset(pf, true);
9616
9617         old_prog = xchg(&vsi->xdp_prog, prog);
9618
9619         if (need_reset)
9620                 i40e_reset_and_rebuild(pf, true, true);
9621
9622         for (i = 0; i < vsi->num_queue_pairs; i++)
9623                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
9624
9625         if (old_prog)
9626                 bpf_prog_put(old_prog);
9627
9628         return 0;
9629 }
9630
9631 /**
9632  * i40e_xdp - implements ndo_xdp for i40e
9633  * @dev: netdevice
9634  * @xdp: XDP command
9635  **/
9636 static int i40e_xdp(struct net_device *dev,
9637                     struct netdev_xdp *xdp)
9638 {
9639         struct i40e_netdev_priv *np = netdev_priv(dev);
9640         struct i40e_vsi *vsi = np->vsi;
9641
9642         if (vsi->type != I40E_VSI_MAIN)
9643                 return -EINVAL;
9644
9645         switch (xdp->command) {
9646         case XDP_SETUP_PROG:
9647                 return i40e_xdp_setup(vsi, xdp->prog);
9648         case XDP_QUERY_PROG:
9649                 xdp->prog_attached = i40e_enabled_xdp_vsi(vsi);
9650                 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
9651                 return 0;
9652         default:
9653                 return -EINVAL;
9654         }
9655 }
9656
9657 static const struct net_device_ops i40e_netdev_ops = {
9658         .ndo_open               = i40e_open,
9659         .ndo_stop               = i40e_close,
9660         .ndo_start_xmit         = i40e_lan_xmit_frame,
9661         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
9662         .ndo_set_rx_mode        = i40e_set_rx_mode,
9663         .ndo_validate_addr      = eth_validate_addr,
9664         .ndo_set_mac_address    = i40e_set_mac,
9665         .ndo_change_mtu         = i40e_change_mtu,
9666         .ndo_do_ioctl           = i40e_ioctl,
9667         .ndo_tx_timeout         = i40e_tx_timeout,
9668         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
9669         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
9670 #ifdef CONFIG_NET_POLL_CONTROLLER
9671         .ndo_poll_controller    = i40e_netpoll,
9672 #endif
9673         .ndo_setup_tc           = __i40e_setup_tc,
9674         .ndo_set_features       = i40e_set_features,
9675         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
9676         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
9677         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
9678         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
9679         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
9680         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
9681         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
9682         .ndo_udp_tunnel_add     = i40e_udp_tunnel_add,
9683         .ndo_udp_tunnel_del     = i40e_udp_tunnel_del,
9684         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
9685         .ndo_fdb_add            = i40e_ndo_fdb_add,
9686         .ndo_features_check     = i40e_features_check,
9687         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
9688         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
9689         .ndo_xdp                = i40e_xdp,
9690 };
9691
9692 /**
9693  * i40e_config_netdev - Setup the netdev flags
9694  * @vsi: the VSI being configured
9695  *
9696  * Returns 0 on success, negative value on failure
9697  **/
9698 static int i40e_config_netdev(struct i40e_vsi *vsi)
9699 {
9700         struct i40e_pf *pf = vsi->back;
9701         struct i40e_hw *hw = &pf->hw;
9702         struct i40e_netdev_priv *np;
9703         struct net_device *netdev;
9704         u8 broadcast[ETH_ALEN];
9705         u8 mac_addr[ETH_ALEN];
9706         int etherdev_size;
9707         netdev_features_t hw_enc_features;
9708         netdev_features_t hw_features;
9709
9710         etherdev_size = sizeof(struct i40e_netdev_priv);
9711         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
9712         if (!netdev)
9713                 return -ENOMEM;
9714
9715         vsi->netdev = netdev;
9716         np = netdev_priv(netdev);
9717         np->vsi = vsi;
9718
9719         hw_enc_features = NETIF_F_SG                    |
9720                           NETIF_F_IP_CSUM               |
9721                           NETIF_F_IPV6_CSUM             |
9722                           NETIF_F_HIGHDMA               |
9723                           NETIF_F_SOFT_FEATURES         |
9724                           NETIF_F_TSO                   |
9725                           NETIF_F_TSO_ECN               |
9726                           NETIF_F_TSO6                  |
9727                           NETIF_F_GSO_GRE               |
9728                           NETIF_F_GSO_GRE_CSUM          |
9729                           NETIF_F_GSO_PARTIAL           |
9730                           NETIF_F_GSO_UDP_TUNNEL        |
9731                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
9732                           NETIF_F_SCTP_CRC              |
9733                           NETIF_F_RXHASH                |
9734                           NETIF_F_RXCSUM                |
9735                           0;
9736
9737         if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
9738                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
9739
9740         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
9741
9742         netdev->hw_enc_features |= hw_enc_features;
9743
9744         /* record features VLANs can make use of */
9745         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
9746
9747         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
9748                 netdev->hw_features |= NETIF_F_NTUPLE;
9749         hw_features = hw_enc_features           |
9750                       NETIF_F_HW_VLAN_CTAG_TX   |
9751                       NETIF_F_HW_VLAN_CTAG_RX;
9752
9753         netdev->hw_features |= hw_features;
9754
9755         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
9756         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
9757
9758         if (vsi->type == I40E_VSI_MAIN) {
9759                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
9760                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
9761                 /* The following steps are necessary for two reasons. First,
9762                  * some older NVM configurations load a default MAC-VLAN
9763                  * filter that will accept any tagged packet, and we want to
9764                  * replace this with a normal filter. Additionally, it is
9765                  * possible our MAC address was provided by the platform using
9766                  * Open Firmware or similar.
9767                  *
9768                  * Thus, we need to remove the default filter and install one
9769                  * specific to the MAC address.
9770                  */
9771                 i40e_rm_default_mac_filter(vsi, mac_addr);
9772                 spin_lock_bh(&vsi->mac_filter_hash_lock);
9773                 i40e_add_mac_filter(vsi, mac_addr);
9774                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9775         } else {
9776                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
9777                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
9778                  * the end, which is 4 bytes long, so force truncation of the
9779                  * original name by IFNAMSIZ - 4
9780                  */
9781                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
9782                          IFNAMSIZ - 4,
9783                          pf->vsi[pf->lan_vsi]->netdev->name);
9784                 random_ether_addr(mac_addr);
9785
9786                 spin_lock_bh(&vsi->mac_filter_hash_lock);
9787                 i40e_add_mac_filter(vsi, mac_addr);
9788                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9789         }
9790
9791         /* Add the broadcast filter so that we initially will receive
9792          * broadcast packets. Note that when a new VLAN is first added the
9793          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
9794          * specific filters as part of transitioning into "vlan" operation.
9795          * When more VLANs are added, the driver will copy each existing MAC
9796          * filter and add it for the new VLAN.
9797          *
9798          * Broadcast filters are handled specially by
9799          * i40e_sync_filters_subtask, as the driver must to set the broadcast
9800          * promiscuous bit instead of adding this directly as a MAC/VLAN
9801          * filter. The subtask will update the correct broadcast promiscuous
9802          * bits as VLANs become active or inactive.
9803          */
9804         eth_broadcast_addr(broadcast);
9805         spin_lock_bh(&vsi->mac_filter_hash_lock);
9806         i40e_add_mac_filter(vsi, broadcast);
9807         spin_unlock_bh(&vsi->mac_filter_hash_lock);
9808
9809         ether_addr_copy(netdev->dev_addr, mac_addr);
9810         ether_addr_copy(netdev->perm_addr, mac_addr);
9811
9812         netdev->priv_flags |= IFF_UNICAST_FLT;
9813         netdev->priv_flags |= IFF_SUPP_NOFCS;
9814         /* Setup netdev TC information */
9815         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
9816
9817         netdev->netdev_ops = &i40e_netdev_ops;
9818         netdev->watchdog_timeo = 5 * HZ;
9819         i40e_set_ethtool_ops(netdev);
9820
9821         /* MTU range: 68 - 9706 */
9822         netdev->min_mtu = ETH_MIN_MTU;
9823         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
9824
9825         return 0;
9826 }
9827
9828 /**
9829  * i40e_vsi_delete - Delete a VSI from the switch
9830  * @vsi: the VSI being removed
9831  *
9832  * Returns 0 on success, negative value on failure
9833  **/
9834 static void i40e_vsi_delete(struct i40e_vsi *vsi)
9835 {
9836         /* remove default VSI is not allowed */
9837         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
9838                 return;
9839
9840         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
9841 }
9842
9843 /**
9844  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
9845  * @vsi: the VSI being queried
9846  *
9847  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
9848  **/
9849 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
9850 {
9851         struct i40e_veb *veb;
9852         struct i40e_pf *pf = vsi->back;
9853
9854         /* Uplink is not a bridge so default to VEB */
9855         if (vsi->veb_idx == I40E_NO_VEB)
9856                 return 1;
9857
9858         veb = pf->veb[vsi->veb_idx];
9859         if (!veb) {
9860                 dev_info(&pf->pdev->dev,
9861                          "There is no veb associated with the bridge\n");
9862                 return -ENOENT;
9863         }
9864
9865         /* Uplink is a bridge in VEPA mode */
9866         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
9867                 return 0;
9868         } else {
9869                 /* Uplink is a bridge in VEB mode */
9870                 return 1;
9871         }
9872
9873         /* VEPA is now default bridge, so return 0 */
9874         return 0;
9875 }
9876
9877 /**
9878  * i40e_add_vsi - Add a VSI to the switch
9879  * @vsi: the VSI being configured
9880  *
9881  * This initializes a VSI context depending on the VSI type to be added and
9882  * passes it down to the add_vsi aq command.
9883  **/
9884 static int i40e_add_vsi(struct i40e_vsi *vsi)
9885 {
9886         int ret = -ENODEV;
9887         struct i40e_pf *pf = vsi->back;
9888         struct i40e_hw *hw = &pf->hw;
9889         struct i40e_vsi_context ctxt;
9890         struct i40e_mac_filter *f;
9891         struct hlist_node *h;
9892         int bkt;
9893
9894         u8 enabled_tc = 0x1; /* TC0 enabled */
9895         int f_count = 0;
9896
9897         memset(&ctxt, 0, sizeof(ctxt));
9898         switch (vsi->type) {
9899         case I40E_VSI_MAIN:
9900                 /* The PF's main VSI is already setup as part of the
9901                  * device initialization, so we'll not bother with
9902                  * the add_vsi call, but we will retrieve the current
9903                  * VSI context.
9904                  */
9905                 ctxt.seid = pf->main_vsi_seid;
9906                 ctxt.pf_num = pf->hw.pf_id;
9907                 ctxt.vf_num = 0;
9908                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9909                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9910                 if (ret) {
9911                         dev_info(&pf->pdev->dev,
9912                                  "couldn't get PF vsi config, err %s aq_err %s\n",
9913                                  i40e_stat_str(&pf->hw, ret),
9914                                  i40e_aq_str(&pf->hw,
9915                                              pf->hw.aq.asq_last_status));
9916                         return -ENOENT;
9917                 }
9918                 vsi->info = ctxt.info;
9919                 vsi->info.valid_sections = 0;
9920
9921                 vsi->seid = ctxt.seid;
9922                 vsi->id = ctxt.vsi_number;
9923
9924                 enabled_tc = i40e_pf_get_tc_map(pf);
9925
9926                 /* Source pruning is enabled by default, so the flag is
9927                  * negative logic - if it's set, we need to fiddle with
9928                  * the VSI to disable source pruning.
9929                  */
9930                 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
9931                         memset(&ctxt, 0, sizeof(ctxt));
9932                         ctxt.seid = pf->main_vsi_seid;
9933                         ctxt.pf_num = pf->hw.pf_id;
9934                         ctxt.vf_num = 0;
9935                         ctxt.info.valid_sections |=
9936                                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9937                         ctxt.info.switch_id =
9938                                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
9939                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
9940                         if (ret) {
9941                                 dev_info(&pf->pdev->dev,
9942                                          "update vsi failed, err %s aq_err %s\n",
9943                                          i40e_stat_str(&pf->hw, ret),
9944                                          i40e_aq_str(&pf->hw,
9945                                                      pf->hw.aq.asq_last_status));
9946                                 ret = -ENOENT;
9947                                 goto err;
9948                         }
9949                 }
9950
9951                 /* MFP mode setup queue map and update VSI */
9952                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
9953                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
9954                         memset(&ctxt, 0, sizeof(ctxt));
9955                         ctxt.seid = pf->main_vsi_seid;
9956                         ctxt.pf_num = pf->hw.pf_id;
9957                         ctxt.vf_num = 0;
9958                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
9959                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
9960                         if (ret) {
9961                                 dev_info(&pf->pdev->dev,
9962                                          "update vsi failed, err %s aq_err %s\n",
9963                                          i40e_stat_str(&pf->hw, ret),
9964                                          i40e_aq_str(&pf->hw,
9965                                                     pf->hw.aq.asq_last_status));
9966                                 ret = -ENOENT;
9967                                 goto err;
9968                         }
9969                         /* update the local VSI info queue map */
9970                         i40e_vsi_update_queue_map(vsi, &ctxt);
9971                         vsi->info.valid_sections = 0;
9972                 } else {
9973                         /* Default/Main VSI is only enabled for TC0
9974                          * reconfigure it to enable all TCs that are
9975                          * available on the port in SFP mode.
9976                          * For MFP case the iSCSI PF would use this
9977                          * flow to enable LAN+iSCSI TC.
9978                          */
9979                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
9980                         if (ret) {
9981                                 /* Single TC condition is not fatal,
9982                                  * message and continue
9983                                  */
9984                                 dev_info(&pf->pdev->dev,
9985                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
9986                                          enabled_tc,
9987                                          i40e_stat_str(&pf->hw, ret),
9988                                          i40e_aq_str(&pf->hw,
9989                                                     pf->hw.aq.asq_last_status));
9990                         }
9991                 }
9992                 break;
9993
9994         case I40E_VSI_FDIR:
9995                 ctxt.pf_num = hw->pf_id;
9996                 ctxt.vf_num = 0;
9997                 ctxt.uplink_seid = vsi->uplink_seid;
9998                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9999                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10000                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
10001                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
10002                         ctxt.info.valid_sections |=
10003                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10004                         ctxt.info.switch_id =
10005                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10006                 }
10007                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
10008                 break;
10009
10010         case I40E_VSI_VMDQ2:
10011                 ctxt.pf_num = hw->pf_id;
10012                 ctxt.vf_num = 0;
10013                 ctxt.uplink_seid = vsi->uplink_seid;
10014                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
10015                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
10016
10017                 /* This VSI is connected to VEB so the switch_id
10018                  * should be set to zero by default.
10019                  */
10020                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
10021                         ctxt.info.valid_sections |=
10022                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10023                         ctxt.info.switch_id =
10024                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10025                 }
10026
10027                 /* Setup the VSI tx/rx queue map for TC0 only for now */
10028                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
10029                 break;
10030
10031         case I40E_VSI_SRIOV:
10032                 ctxt.pf_num = hw->pf_id;
10033                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
10034                 ctxt.uplink_seid = vsi->uplink_seid;
10035                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
10036                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
10037
10038                 /* This VSI is connected to VEB so the switch_id
10039                  * should be set to zero by default.
10040                  */
10041                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
10042                         ctxt.info.valid_sections |=
10043                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10044                         ctxt.info.switch_id =
10045                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10046                 }
10047
10048                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
10049                         ctxt.info.valid_sections |=
10050                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
10051                         ctxt.info.queueing_opt_flags |=
10052                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
10053                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
10054                 }
10055
10056                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
10057                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
10058                 if (pf->vf[vsi->vf_id].spoofchk) {
10059                         ctxt.info.valid_sections |=
10060                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
10061                         ctxt.info.sec_flags |=
10062                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
10063                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
10064                 }
10065                 /* Setup the VSI tx/rx queue map for TC0 only for now */
10066                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
10067                 break;
10068
10069         case I40E_VSI_IWARP:
10070                 /* send down message to iWARP */
10071                 break;
10072
10073         default:
10074                 return -ENODEV;
10075         }
10076
10077         if (vsi->type != I40E_VSI_MAIN) {
10078                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
10079                 if (ret) {
10080                         dev_info(&vsi->back->pdev->dev,
10081                                  "add vsi failed, err %s aq_err %s\n",
10082                                  i40e_stat_str(&pf->hw, ret),
10083                                  i40e_aq_str(&pf->hw,
10084                                              pf->hw.aq.asq_last_status));
10085                         ret = -ENOENT;
10086                         goto err;
10087                 }
10088                 vsi->info = ctxt.info;
10089                 vsi->info.valid_sections = 0;
10090                 vsi->seid = ctxt.seid;
10091                 vsi->id = ctxt.vsi_number;
10092         }
10093
10094         vsi->active_filters = 0;
10095         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
10096         spin_lock_bh(&vsi->mac_filter_hash_lock);
10097         /* If macvlan filters already exist, force them to get loaded */
10098         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
10099                 f->state = I40E_FILTER_NEW;
10100                 f_count++;
10101         }
10102         spin_unlock_bh(&vsi->mac_filter_hash_lock);
10103
10104         if (f_count) {
10105                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
10106                 pf->flags |= I40E_FLAG_FILTER_SYNC;
10107         }
10108
10109         /* Update VSI BW information */
10110         ret = i40e_vsi_get_bw_info(vsi);
10111         if (ret) {
10112                 dev_info(&pf->pdev->dev,
10113                          "couldn't get vsi bw info, err %s aq_err %s\n",
10114                          i40e_stat_str(&pf->hw, ret),
10115                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10116                 /* VSI is already added so not tearing that up */
10117                 ret = 0;
10118         }
10119
10120 err:
10121         return ret;
10122 }
10123
10124 /**
10125  * i40e_vsi_release - Delete a VSI and free its resources
10126  * @vsi: the VSI being removed
10127  *
10128  * Returns 0 on success or < 0 on error
10129  **/
10130 int i40e_vsi_release(struct i40e_vsi *vsi)
10131 {
10132         struct i40e_mac_filter *f;
10133         struct hlist_node *h;
10134         struct i40e_veb *veb = NULL;
10135         struct i40e_pf *pf;
10136         u16 uplink_seid;
10137         int i, n, bkt;
10138
10139         pf = vsi->back;
10140
10141         /* release of a VEB-owner or last VSI is not allowed */
10142         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
10143                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
10144                          vsi->seid, vsi->uplink_seid);
10145                 return -ENODEV;
10146         }
10147         if (vsi == pf->vsi[pf->lan_vsi] &&
10148             !test_bit(__I40E_DOWN, pf->state)) {
10149                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
10150                 return -ENODEV;
10151         }
10152
10153         uplink_seid = vsi->uplink_seid;
10154         if (vsi->type != I40E_VSI_SRIOV) {
10155                 if (vsi->netdev_registered) {
10156                         vsi->netdev_registered = false;
10157                         if (vsi->netdev) {
10158                                 /* results in a call to i40e_close() */
10159                                 unregister_netdev(vsi->netdev);
10160                         }
10161                 } else {
10162                         i40e_vsi_close(vsi);
10163                 }
10164                 i40e_vsi_disable_irq(vsi);
10165         }
10166
10167         spin_lock_bh(&vsi->mac_filter_hash_lock);
10168
10169         /* clear the sync flag on all filters */
10170         if (vsi->netdev) {
10171                 __dev_uc_unsync(vsi->netdev, NULL);
10172                 __dev_mc_unsync(vsi->netdev, NULL);
10173         }
10174
10175         /* make sure any remaining filters are marked for deletion */
10176         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
10177                 __i40e_del_filter(vsi, f);
10178
10179         spin_unlock_bh(&vsi->mac_filter_hash_lock);
10180
10181         i40e_sync_vsi_filters(vsi);
10182
10183         i40e_vsi_delete(vsi);
10184         i40e_vsi_free_q_vectors(vsi);
10185         if (vsi->netdev) {
10186                 free_netdev(vsi->netdev);
10187                 vsi->netdev = NULL;
10188         }
10189         i40e_vsi_clear_rings(vsi);
10190         i40e_vsi_clear(vsi);
10191
10192         /* If this was the last thing on the VEB, except for the
10193          * controlling VSI, remove the VEB, which puts the controlling
10194          * VSI onto the next level down in the switch.
10195          *
10196          * Well, okay, there's one more exception here: don't remove
10197          * the orphan VEBs yet.  We'll wait for an explicit remove request
10198          * from up the network stack.
10199          */
10200         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
10201                 if (pf->vsi[i] &&
10202                     pf->vsi[i]->uplink_seid == uplink_seid &&
10203                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10204                         n++;      /* count the VSIs */
10205                 }
10206         }
10207         for (i = 0; i < I40E_MAX_VEB; i++) {
10208                 if (!pf->veb[i])
10209                         continue;
10210                 if (pf->veb[i]->uplink_seid == uplink_seid)
10211                         n++;     /* count the VEBs */
10212                 if (pf->veb[i]->seid == uplink_seid)
10213                         veb = pf->veb[i];
10214         }
10215         if (n == 0 && veb && veb->uplink_seid != 0)
10216                 i40e_veb_release(veb);
10217
10218         return 0;
10219 }
10220
10221 /**
10222  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
10223  * @vsi: ptr to the VSI
10224  *
10225  * This should only be called after i40e_vsi_mem_alloc() which allocates the
10226  * corresponding SW VSI structure and initializes num_queue_pairs for the
10227  * newly allocated VSI.
10228  *
10229  * Returns 0 on success or negative on failure
10230  **/
10231 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
10232 {
10233         int ret = -ENOENT;
10234         struct i40e_pf *pf = vsi->back;
10235
10236         if (vsi->q_vectors[0]) {
10237                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
10238                          vsi->seid);
10239                 return -EEXIST;
10240         }
10241
10242         if (vsi->base_vector) {
10243                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
10244                          vsi->seid, vsi->base_vector);
10245                 return -EEXIST;
10246         }
10247
10248         ret = i40e_vsi_alloc_q_vectors(vsi);
10249         if (ret) {
10250                 dev_info(&pf->pdev->dev,
10251                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
10252                          vsi->num_q_vectors, vsi->seid, ret);
10253                 vsi->num_q_vectors = 0;
10254                 goto vector_setup_out;
10255         }
10256
10257         /* In Legacy mode, we do not have to get any other vector since we
10258          * piggyback on the misc/ICR0 for queue interrupts.
10259         */
10260         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
10261                 return ret;
10262         if (vsi->num_q_vectors)
10263                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
10264                                                  vsi->num_q_vectors, vsi->idx);
10265         if (vsi->base_vector < 0) {
10266                 dev_info(&pf->pdev->dev,
10267                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
10268                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
10269                 i40e_vsi_free_q_vectors(vsi);
10270                 ret = -ENOENT;
10271                 goto vector_setup_out;
10272         }
10273
10274 vector_setup_out:
10275         return ret;
10276 }
10277
10278 /**
10279  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
10280  * @vsi: pointer to the vsi.
10281  *
10282  * This re-allocates a vsi's queue resources.
10283  *
10284  * Returns pointer to the successfully allocated and configured VSI sw struct
10285  * on success, otherwise returns NULL on failure.
10286  **/
10287 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
10288 {
10289         u16 alloc_queue_pairs;
10290         struct i40e_pf *pf;
10291         u8 enabled_tc;
10292         int ret;
10293
10294         if (!vsi)
10295                 return NULL;
10296
10297         pf = vsi->back;
10298
10299         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
10300         i40e_vsi_clear_rings(vsi);
10301
10302         i40e_vsi_free_arrays(vsi, false);
10303         i40e_set_num_rings_in_vsi(vsi);
10304         ret = i40e_vsi_alloc_arrays(vsi, false);
10305         if (ret)
10306                 goto err_vsi;
10307
10308         alloc_queue_pairs = vsi->alloc_queue_pairs *
10309                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
10310
10311         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
10312         if (ret < 0) {
10313                 dev_info(&pf->pdev->dev,
10314                          "failed to get tracking for %d queues for VSI %d err %d\n",
10315                          alloc_queue_pairs, vsi->seid, ret);
10316                 goto err_vsi;
10317         }
10318         vsi->base_queue = ret;
10319
10320         /* Update the FW view of the VSI. Force a reset of TC and queue
10321          * layout configurations.
10322          */
10323         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
10324         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
10325         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
10326         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
10327         if (vsi->type == I40E_VSI_MAIN)
10328                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
10329
10330         /* assign it some queues */
10331         ret = i40e_alloc_rings(vsi);
10332         if (ret)
10333                 goto err_rings;
10334
10335         /* map all of the rings to the q_vectors */
10336         i40e_vsi_map_rings_to_vectors(vsi);
10337         return vsi;
10338
10339 err_rings:
10340         i40e_vsi_free_q_vectors(vsi);
10341         if (vsi->netdev_registered) {
10342                 vsi->netdev_registered = false;
10343                 unregister_netdev(vsi->netdev);
10344                 free_netdev(vsi->netdev);
10345                 vsi->netdev = NULL;
10346         }
10347         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
10348 err_vsi:
10349         i40e_vsi_clear(vsi);
10350         return NULL;
10351 }
10352
10353 /**
10354  * i40e_vsi_setup - Set up a VSI by a given type
10355  * @pf: board private structure
10356  * @type: VSI type
10357  * @uplink_seid: the switch element to link to
10358  * @param1: usage depends upon VSI type. For VF types, indicates VF id
10359  *
10360  * This allocates the sw VSI structure and its queue resources, then add a VSI
10361  * to the identified VEB.
10362  *
10363  * Returns pointer to the successfully allocated and configure VSI sw struct on
10364  * success, otherwise returns NULL on failure.
10365  **/
10366 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
10367                                 u16 uplink_seid, u32 param1)
10368 {
10369         struct i40e_vsi *vsi = NULL;
10370         struct i40e_veb *veb = NULL;
10371         u16 alloc_queue_pairs;
10372         int ret, i;
10373         int v_idx;
10374
10375         /* The requested uplink_seid must be either
10376          *     - the PF's port seid
10377          *              no VEB is needed because this is the PF
10378          *              or this is a Flow Director special case VSI
10379          *     - seid of an existing VEB
10380          *     - seid of a VSI that owns an existing VEB
10381          *     - seid of a VSI that doesn't own a VEB
10382          *              a new VEB is created and the VSI becomes the owner
10383          *     - seid of the PF VSI, which is what creates the first VEB
10384          *              this is a special case of the previous
10385          *
10386          * Find which uplink_seid we were given and create a new VEB if needed
10387          */
10388         for (i = 0; i < I40E_MAX_VEB; i++) {
10389                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
10390                         veb = pf->veb[i];
10391                         break;
10392                 }
10393         }
10394
10395         if (!veb && uplink_seid != pf->mac_seid) {
10396
10397                 for (i = 0; i < pf->num_alloc_vsi; i++) {
10398                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
10399                                 vsi = pf->vsi[i];
10400                                 break;
10401                         }
10402                 }
10403                 if (!vsi) {
10404                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
10405                                  uplink_seid);
10406                         return NULL;
10407                 }
10408
10409                 if (vsi->uplink_seid == pf->mac_seid)
10410                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
10411                                              vsi->tc_config.enabled_tc);
10412                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
10413                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
10414                                              vsi->tc_config.enabled_tc);
10415                 if (veb) {
10416                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
10417                                 dev_info(&vsi->back->pdev->dev,
10418                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
10419                                 return NULL;
10420                         }
10421                         /* We come up by default in VEPA mode if SRIOV is not
10422                          * already enabled, in which case we can't force VEPA
10423                          * mode.
10424                          */
10425                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
10426                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
10427                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
10428                         }
10429                         i40e_config_bridge_mode(veb);
10430                 }
10431                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
10432                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
10433                                 veb = pf->veb[i];
10434                 }
10435                 if (!veb) {
10436                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
10437                         return NULL;
10438                 }
10439
10440                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10441                 uplink_seid = veb->seid;
10442         }
10443
10444         /* get vsi sw struct */
10445         v_idx = i40e_vsi_mem_alloc(pf, type);
10446         if (v_idx < 0)
10447                 goto err_alloc;
10448         vsi = pf->vsi[v_idx];
10449         if (!vsi)
10450                 goto err_alloc;
10451         vsi->type = type;
10452         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
10453
10454         if (type == I40E_VSI_MAIN)
10455                 pf->lan_vsi = v_idx;
10456         else if (type == I40E_VSI_SRIOV)
10457                 vsi->vf_id = param1;
10458         /* assign it some queues */
10459         alloc_queue_pairs = vsi->alloc_queue_pairs *
10460                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
10461
10462         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
10463         if (ret < 0) {
10464                 dev_info(&pf->pdev->dev,
10465                          "failed to get tracking for %d queues for VSI %d err=%d\n",
10466                          alloc_queue_pairs, vsi->seid, ret);
10467                 goto err_vsi;
10468         }
10469         vsi->base_queue = ret;
10470
10471         /* get a VSI from the hardware */
10472         vsi->uplink_seid = uplink_seid;
10473         ret = i40e_add_vsi(vsi);
10474         if (ret)
10475                 goto err_vsi;
10476
10477         switch (vsi->type) {
10478         /* setup the netdev if needed */
10479         case I40E_VSI_MAIN:
10480         case I40E_VSI_VMDQ2:
10481                 ret = i40e_config_netdev(vsi);
10482                 if (ret)
10483                         goto err_netdev;
10484                 ret = register_netdev(vsi->netdev);
10485                 if (ret)
10486                         goto err_netdev;
10487                 vsi->netdev_registered = true;
10488                 netif_carrier_off(vsi->netdev);
10489 #ifdef CONFIG_I40E_DCB
10490                 /* Setup DCB netlink interface */
10491                 i40e_dcbnl_setup(vsi);
10492 #endif /* CONFIG_I40E_DCB */
10493                 /* fall through */
10494
10495         case I40E_VSI_FDIR:
10496                 /* set up vectors and rings if needed */
10497                 ret = i40e_vsi_setup_vectors(vsi);
10498                 if (ret)
10499                         goto err_msix;
10500
10501                 ret = i40e_alloc_rings(vsi);
10502                 if (ret)
10503                         goto err_rings;
10504
10505                 /* map all of the rings to the q_vectors */
10506                 i40e_vsi_map_rings_to_vectors(vsi);
10507
10508                 i40e_vsi_reset_stats(vsi);
10509                 break;
10510
10511         default:
10512                 /* no netdev or rings for the other VSI types */
10513                 break;
10514         }
10515
10516         if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
10517             (vsi->type == I40E_VSI_VMDQ2)) {
10518                 ret = i40e_vsi_config_rss(vsi);
10519         }
10520         return vsi;
10521
10522 err_rings:
10523         i40e_vsi_free_q_vectors(vsi);
10524 err_msix:
10525         if (vsi->netdev_registered) {
10526                 vsi->netdev_registered = false;
10527                 unregister_netdev(vsi->netdev);
10528                 free_netdev(vsi->netdev);
10529                 vsi->netdev = NULL;
10530         }
10531 err_netdev:
10532         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
10533 err_vsi:
10534         i40e_vsi_clear(vsi);
10535 err_alloc:
10536         return NULL;
10537 }
10538
10539 /**
10540  * i40e_veb_get_bw_info - Query VEB BW information
10541  * @veb: the veb to query
10542  *
10543  * Query the Tx scheduler BW configuration data for given VEB
10544  **/
10545 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
10546 {
10547         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
10548         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
10549         struct i40e_pf *pf = veb->pf;
10550         struct i40e_hw *hw = &pf->hw;
10551         u32 tc_bw_max;
10552         int ret = 0;
10553         int i;
10554
10555         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
10556                                                   &bw_data, NULL);
10557         if (ret) {
10558                 dev_info(&pf->pdev->dev,
10559                          "query veb bw config failed, err %s aq_err %s\n",
10560                          i40e_stat_str(&pf->hw, ret),
10561                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10562                 goto out;
10563         }
10564
10565         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
10566                                                    &ets_data, NULL);
10567         if (ret) {
10568                 dev_info(&pf->pdev->dev,
10569                          "query veb bw ets config failed, err %s aq_err %s\n",
10570                          i40e_stat_str(&pf->hw, ret),
10571                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10572                 goto out;
10573         }
10574
10575         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
10576         veb->bw_max_quanta = ets_data.tc_bw_max;
10577         veb->is_abs_credits = bw_data.absolute_credits_enable;
10578         veb->enabled_tc = ets_data.tc_valid_bits;
10579         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
10580                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
10581         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
10582                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
10583                 veb->bw_tc_limit_credits[i] =
10584                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
10585                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
10586         }
10587
10588 out:
10589         return ret;
10590 }
10591
10592 /**
10593  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
10594  * @pf: board private structure
10595  *
10596  * On error: returns error code (negative)
10597  * On success: returns vsi index in PF (positive)
10598  **/
10599 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
10600 {
10601         int ret = -ENOENT;
10602         struct i40e_veb *veb;
10603         int i;
10604
10605         /* Need to protect the allocation of switch elements at the PF level */
10606         mutex_lock(&pf->switch_mutex);
10607
10608         /* VEB list may be fragmented if VEB creation/destruction has
10609          * been happening.  We can afford to do a quick scan to look
10610          * for any free slots in the list.
10611          *
10612          * find next empty veb slot, looping back around if necessary
10613          */
10614         i = 0;
10615         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
10616                 i++;
10617         if (i >= I40E_MAX_VEB) {
10618                 ret = -ENOMEM;
10619                 goto err_alloc_veb;  /* out of VEB slots! */
10620         }
10621
10622         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
10623         if (!veb) {
10624                 ret = -ENOMEM;
10625                 goto err_alloc_veb;
10626         }
10627         veb->pf = pf;
10628         veb->idx = i;
10629         veb->enabled_tc = 1;
10630
10631         pf->veb[i] = veb;
10632         ret = i;
10633 err_alloc_veb:
10634         mutex_unlock(&pf->switch_mutex);
10635         return ret;
10636 }
10637
10638 /**
10639  * i40e_switch_branch_release - Delete a branch of the switch tree
10640  * @branch: where to start deleting
10641  *
10642  * This uses recursion to find the tips of the branch to be
10643  * removed, deleting until we get back to and can delete this VEB.
10644  **/
10645 static void i40e_switch_branch_release(struct i40e_veb *branch)
10646 {
10647         struct i40e_pf *pf = branch->pf;
10648         u16 branch_seid = branch->seid;
10649         u16 veb_idx = branch->idx;
10650         int i;
10651
10652         /* release any VEBs on this VEB - RECURSION */
10653         for (i = 0; i < I40E_MAX_VEB; i++) {
10654                 if (!pf->veb[i])
10655                         continue;
10656                 if (pf->veb[i]->uplink_seid == branch->seid)
10657                         i40e_switch_branch_release(pf->veb[i]);
10658         }
10659
10660         /* Release the VSIs on this VEB, but not the owner VSI.
10661          *
10662          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
10663          *       the VEB itself, so don't use (*branch) after this loop.
10664          */
10665         for (i = 0; i < pf->num_alloc_vsi; i++) {
10666                 if (!pf->vsi[i])
10667                         continue;
10668                 if (pf->vsi[i]->uplink_seid == branch_seid &&
10669                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10670                         i40e_vsi_release(pf->vsi[i]);
10671                 }
10672         }
10673
10674         /* There's one corner case where the VEB might not have been
10675          * removed, so double check it here and remove it if needed.
10676          * This case happens if the veb was created from the debugfs
10677          * commands and no VSIs were added to it.
10678          */
10679         if (pf->veb[veb_idx])
10680                 i40e_veb_release(pf->veb[veb_idx]);
10681 }
10682
10683 /**
10684  * i40e_veb_clear - remove veb struct
10685  * @veb: the veb to remove
10686  **/
10687 static void i40e_veb_clear(struct i40e_veb *veb)
10688 {
10689         if (!veb)
10690                 return;
10691
10692         if (veb->pf) {
10693                 struct i40e_pf *pf = veb->pf;
10694
10695                 mutex_lock(&pf->switch_mutex);
10696                 if (pf->veb[veb->idx] == veb)
10697                         pf->veb[veb->idx] = NULL;
10698                 mutex_unlock(&pf->switch_mutex);
10699         }
10700
10701         kfree(veb);
10702 }
10703
10704 /**
10705  * i40e_veb_release - Delete a VEB and free its resources
10706  * @veb: the VEB being removed
10707  **/
10708 void i40e_veb_release(struct i40e_veb *veb)
10709 {
10710         struct i40e_vsi *vsi = NULL;
10711         struct i40e_pf *pf;
10712         int i, n = 0;
10713
10714         pf = veb->pf;
10715
10716         /* find the remaining VSI and check for extras */
10717         for (i = 0; i < pf->num_alloc_vsi; i++) {
10718                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
10719                         n++;
10720                         vsi = pf->vsi[i];
10721                 }
10722         }
10723         if (n != 1) {
10724                 dev_info(&pf->pdev->dev,
10725                          "can't remove VEB %d with %d VSIs left\n",
10726                          veb->seid, n);
10727                 return;
10728         }
10729
10730         /* move the remaining VSI to uplink veb */
10731         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
10732         if (veb->uplink_seid) {
10733                 vsi->uplink_seid = veb->uplink_seid;
10734                 if (veb->uplink_seid == pf->mac_seid)
10735                         vsi->veb_idx = I40E_NO_VEB;
10736                 else
10737                         vsi->veb_idx = veb->veb_idx;
10738         } else {
10739                 /* floating VEB */
10740                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10741                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
10742         }
10743
10744         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10745         i40e_veb_clear(veb);
10746 }
10747
10748 /**
10749  * i40e_add_veb - create the VEB in the switch
10750  * @veb: the VEB to be instantiated
10751  * @vsi: the controlling VSI
10752  **/
10753 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
10754 {
10755         struct i40e_pf *pf = veb->pf;
10756         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
10757         int ret;
10758
10759         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
10760                               veb->enabled_tc, false,
10761                               &veb->seid, enable_stats, NULL);
10762
10763         /* get a VEB from the hardware */
10764         if (ret) {
10765                 dev_info(&pf->pdev->dev,
10766                          "couldn't add VEB, err %s aq_err %s\n",
10767                          i40e_stat_str(&pf->hw, ret),
10768                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10769                 return -EPERM;
10770         }
10771
10772         /* get statistics counter */
10773         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
10774                                          &veb->stats_idx, NULL, NULL, NULL);
10775         if (ret) {
10776                 dev_info(&pf->pdev->dev,
10777                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
10778                          i40e_stat_str(&pf->hw, ret),
10779                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10780                 return -EPERM;
10781         }
10782         ret = i40e_veb_get_bw_info(veb);
10783         if (ret) {
10784                 dev_info(&pf->pdev->dev,
10785                          "couldn't get VEB bw info, err %s aq_err %s\n",
10786                          i40e_stat_str(&pf->hw, ret),
10787                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10788                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10789                 return -ENOENT;
10790         }
10791
10792         vsi->uplink_seid = veb->seid;
10793         vsi->veb_idx = veb->idx;
10794         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10795
10796         return 0;
10797 }
10798
10799 /**
10800  * i40e_veb_setup - Set up a VEB
10801  * @pf: board private structure
10802  * @flags: VEB setup flags
10803  * @uplink_seid: the switch element to link to
10804  * @vsi_seid: the initial VSI seid
10805  * @enabled_tc: Enabled TC bit-map
10806  *
10807  * This allocates the sw VEB structure and links it into the switch
10808  * It is possible and legal for this to be a duplicate of an already
10809  * existing VEB.  It is also possible for both uplink and vsi seids
10810  * to be zero, in order to create a floating VEB.
10811  *
10812  * Returns pointer to the successfully allocated VEB sw struct on
10813  * success, otherwise returns NULL on failure.
10814  **/
10815 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
10816                                 u16 uplink_seid, u16 vsi_seid,
10817                                 u8 enabled_tc)
10818 {
10819         struct i40e_veb *veb, *uplink_veb = NULL;
10820         int vsi_idx, veb_idx;
10821         int ret;
10822
10823         /* if one seid is 0, the other must be 0 to create a floating relay */
10824         if ((uplink_seid == 0 || vsi_seid == 0) &&
10825             (uplink_seid + vsi_seid != 0)) {
10826                 dev_info(&pf->pdev->dev,
10827                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
10828                          uplink_seid, vsi_seid);
10829                 return NULL;
10830         }
10831
10832         /* make sure there is such a vsi and uplink */
10833         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
10834                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
10835                         break;
10836         if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
10837                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
10838                          vsi_seid);
10839                 return NULL;
10840         }
10841
10842         if (uplink_seid && uplink_seid != pf->mac_seid) {
10843                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10844                         if (pf->veb[veb_idx] &&
10845                             pf->veb[veb_idx]->seid == uplink_seid) {
10846                                 uplink_veb = pf->veb[veb_idx];
10847                                 break;
10848                         }
10849                 }
10850                 if (!uplink_veb) {
10851                         dev_info(&pf->pdev->dev,
10852                                  "uplink seid %d not found\n", uplink_seid);
10853                         return NULL;
10854                 }
10855         }
10856
10857         /* get veb sw struct */
10858         veb_idx = i40e_veb_mem_alloc(pf);
10859         if (veb_idx < 0)
10860                 goto err_alloc;
10861         veb = pf->veb[veb_idx];
10862         veb->flags = flags;
10863         veb->uplink_seid = uplink_seid;
10864         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
10865         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
10866
10867         /* create the VEB in the switch */
10868         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
10869         if (ret)
10870                 goto err_veb;
10871         if (vsi_idx == pf->lan_vsi)
10872                 pf->lan_veb = veb->idx;
10873
10874         return veb;
10875
10876 err_veb:
10877         i40e_veb_clear(veb);
10878 err_alloc:
10879         return NULL;
10880 }
10881
10882 /**
10883  * i40e_setup_pf_switch_element - set PF vars based on switch type
10884  * @pf: board private structure
10885  * @ele: element we are building info from
10886  * @num_reported: total number of elements
10887  * @printconfig: should we print the contents
10888  *
10889  * helper function to assist in extracting a few useful SEID values.
10890  **/
10891 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
10892                                 struct i40e_aqc_switch_config_element_resp *ele,
10893                                 u16 num_reported, bool printconfig)
10894 {
10895         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
10896         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
10897         u8 element_type = ele->element_type;
10898         u16 seid = le16_to_cpu(ele->seid);
10899
10900         if (printconfig)
10901                 dev_info(&pf->pdev->dev,
10902                          "type=%d seid=%d uplink=%d downlink=%d\n",
10903                          element_type, seid, uplink_seid, downlink_seid);
10904
10905         switch (element_type) {
10906         case I40E_SWITCH_ELEMENT_TYPE_MAC:
10907                 pf->mac_seid = seid;
10908                 break;
10909         case I40E_SWITCH_ELEMENT_TYPE_VEB:
10910                 /* Main VEB? */
10911                 if (uplink_seid != pf->mac_seid)
10912                         break;
10913                 if (pf->lan_veb == I40E_NO_VEB) {
10914                         int v;
10915
10916                         /* find existing or else empty VEB */
10917                         for (v = 0; v < I40E_MAX_VEB; v++) {
10918                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
10919                                         pf->lan_veb = v;
10920                                         break;
10921                                 }
10922                         }
10923                         if (pf->lan_veb == I40E_NO_VEB) {
10924                                 v = i40e_veb_mem_alloc(pf);
10925                                 if (v < 0)
10926                                         break;
10927                                 pf->lan_veb = v;
10928                         }
10929                 }
10930
10931                 pf->veb[pf->lan_veb]->seid = seid;
10932                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
10933                 pf->veb[pf->lan_veb]->pf = pf;
10934                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
10935                 break;
10936         case I40E_SWITCH_ELEMENT_TYPE_VSI:
10937                 if (num_reported != 1)
10938                         break;
10939                 /* This is immediately after a reset so we can assume this is
10940                  * the PF's VSI
10941                  */
10942                 pf->mac_seid = uplink_seid;
10943                 pf->pf_seid = downlink_seid;
10944                 pf->main_vsi_seid = seid;
10945                 if (printconfig)
10946                         dev_info(&pf->pdev->dev,
10947                                  "pf_seid=%d main_vsi_seid=%d\n",
10948                                  pf->pf_seid, pf->main_vsi_seid);
10949                 break;
10950         case I40E_SWITCH_ELEMENT_TYPE_PF:
10951         case I40E_SWITCH_ELEMENT_TYPE_VF:
10952         case I40E_SWITCH_ELEMENT_TYPE_EMP:
10953         case I40E_SWITCH_ELEMENT_TYPE_BMC:
10954         case I40E_SWITCH_ELEMENT_TYPE_PE:
10955         case I40E_SWITCH_ELEMENT_TYPE_PA:
10956                 /* ignore these for now */
10957                 break;
10958         default:
10959                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
10960                          element_type, seid);
10961                 break;
10962         }
10963 }
10964
10965 /**
10966  * i40e_fetch_switch_configuration - Get switch config from firmware
10967  * @pf: board private structure
10968  * @printconfig: should we print the contents
10969  *
10970  * Get the current switch configuration from the device and
10971  * extract a few useful SEID values.
10972  **/
10973 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
10974 {
10975         struct i40e_aqc_get_switch_config_resp *sw_config;
10976         u16 next_seid = 0;
10977         int ret = 0;
10978         u8 *aq_buf;
10979         int i;
10980
10981         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
10982         if (!aq_buf)
10983                 return -ENOMEM;
10984
10985         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
10986         do {
10987                 u16 num_reported, num_total;
10988
10989                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
10990                                                 I40E_AQ_LARGE_BUF,
10991                                                 &next_seid, NULL);
10992                 if (ret) {
10993                         dev_info(&pf->pdev->dev,
10994                                  "get switch config failed err %s aq_err %s\n",
10995                                  i40e_stat_str(&pf->hw, ret),
10996                                  i40e_aq_str(&pf->hw,
10997                                              pf->hw.aq.asq_last_status));
10998                         kfree(aq_buf);
10999                         return -ENOENT;
11000                 }
11001
11002                 num_reported = le16_to_cpu(sw_config->header.num_reported);
11003                 num_total = le16_to_cpu(sw_config->header.num_total);
11004
11005                 if (printconfig)
11006                         dev_info(&pf->pdev->dev,
11007                                  "header: %d reported %d total\n",
11008                                  num_reported, num_total);
11009
11010                 for (i = 0; i < num_reported; i++) {
11011                         struct i40e_aqc_switch_config_element_resp *ele =
11012                                 &sw_config->element[i];
11013
11014                         i40e_setup_pf_switch_element(pf, ele, num_reported,
11015                                                      printconfig);
11016                 }
11017         } while (next_seid != 0);
11018
11019         kfree(aq_buf);
11020         return ret;
11021 }
11022
11023 /**
11024  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
11025  * @pf: board private structure
11026  * @reinit: if the Main VSI needs to re-initialized.
11027  *
11028  * Returns 0 on success, negative value on failure
11029  **/
11030 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
11031 {
11032         u16 flags = 0;
11033         int ret;
11034
11035         /* find out what's out there already */
11036         ret = i40e_fetch_switch_configuration(pf, false);
11037         if (ret) {
11038                 dev_info(&pf->pdev->dev,
11039                          "couldn't fetch switch config, err %s aq_err %s\n",
11040                          i40e_stat_str(&pf->hw, ret),
11041                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11042                 return ret;
11043         }
11044         i40e_pf_reset_stats(pf);
11045
11046         /* set the switch config bit for the whole device to
11047          * support limited promisc or true promisc
11048          * when user requests promisc. The default is limited
11049          * promisc.
11050         */
11051
11052         if ((pf->hw.pf_id == 0) &&
11053             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT))
11054                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
11055
11056         if (pf->hw.pf_id == 0) {
11057                 u16 valid_flags;
11058
11059                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
11060                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags,
11061                                                 NULL);
11062                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
11063                         dev_info(&pf->pdev->dev,
11064                                  "couldn't set switch config bits, err %s aq_err %s\n",
11065                                  i40e_stat_str(&pf->hw, ret),
11066                                  i40e_aq_str(&pf->hw,
11067                                              pf->hw.aq.asq_last_status));
11068                         /* not a fatal problem, just keep going */
11069                 }
11070         }
11071
11072         /* first time setup */
11073         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
11074                 struct i40e_vsi *vsi = NULL;
11075                 u16 uplink_seid;
11076
11077                 /* Set up the PF VSI associated with the PF's main VSI
11078                  * that is already in the HW switch
11079                  */
11080                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
11081                         uplink_seid = pf->veb[pf->lan_veb]->seid;
11082                 else
11083                         uplink_seid = pf->mac_seid;
11084                 if (pf->lan_vsi == I40E_NO_VSI)
11085                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
11086                 else if (reinit)
11087                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
11088                 if (!vsi) {
11089                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
11090                         i40e_fdir_teardown(pf);
11091                         return -EAGAIN;
11092                 }
11093         } else {
11094                 /* force a reset of TC and queue layout configurations */
11095                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
11096
11097                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
11098                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
11099                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
11100         }
11101         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
11102
11103         i40e_fdir_sb_setup(pf);
11104
11105         /* Setup static PF queue filter control settings */
11106         ret = i40e_setup_pf_filter_control(pf);
11107         if (ret) {
11108                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
11109                          ret);
11110                 /* Failure here should not stop continuing other steps */
11111         }
11112
11113         /* enable RSS in the HW, even for only one queue, as the stack can use
11114          * the hash
11115          */
11116         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
11117                 i40e_pf_config_rss(pf);
11118
11119         /* fill in link information and enable LSE reporting */
11120         i40e_link_event(pf);
11121
11122         /* Initialize user-specific link properties */
11123         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
11124                                   I40E_AQ_AN_COMPLETED) ? true : false);
11125
11126         i40e_ptp_init(pf);
11127
11128         /* repopulate tunnel port filters */
11129         i40e_sync_udp_filters(pf);
11130
11131         return ret;
11132 }
11133
11134 /**
11135  * i40e_determine_queue_usage - Work out queue distribution
11136  * @pf: board private structure
11137  **/
11138 static void i40e_determine_queue_usage(struct i40e_pf *pf)
11139 {
11140         int queues_left;
11141         int q_max;
11142
11143         pf->num_lan_qps = 0;
11144
11145         /* Find the max queues to be put into basic use.  We'll always be
11146          * using TC0, whether or not DCB is running, and TC0 will get the
11147          * big RSS set.
11148          */
11149         queues_left = pf->hw.func_caps.num_tx_qp;
11150
11151         if ((queues_left == 1) ||
11152             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
11153                 /* one qp for PF, no queues for anything else */
11154                 queues_left = 0;
11155                 pf->alloc_rss_size = pf->num_lan_qps = 1;
11156
11157                 /* make sure all the fancies are disabled */
11158                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
11159                                I40E_FLAG_IWARP_ENABLED  |
11160                                I40E_FLAG_FD_SB_ENABLED  |
11161                                I40E_FLAG_FD_ATR_ENABLED |
11162                                I40E_FLAG_DCB_CAPABLE    |
11163                                I40E_FLAG_DCB_ENABLED    |
11164                                I40E_FLAG_SRIOV_ENABLED  |
11165                                I40E_FLAG_VMDQ_ENABLED);
11166         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
11167                                   I40E_FLAG_FD_SB_ENABLED |
11168                                   I40E_FLAG_FD_ATR_ENABLED |
11169                                   I40E_FLAG_DCB_CAPABLE))) {
11170                 /* one qp for PF */
11171                 pf->alloc_rss_size = pf->num_lan_qps = 1;
11172                 queues_left -= pf->num_lan_qps;
11173
11174                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
11175                                I40E_FLAG_IWARP_ENABLED  |
11176                                I40E_FLAG_FD_SB_ENABLED  |
11177                                I40E_FLAG_FD_ATR_ENABLED |
11178                                I40E_FLAG_DCB_ENABLED    |
11179                                I40E_FLAG_VMDQ_ENABLED);
11180         } else {
11181                 /* Not enough queues for all TCs */
11182                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
11183                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
11184                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
11185                                         I40E_FLAG_DCB_ENABLED);
11186                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
11187                 }
11188
11189                 /* limit lan qps to the smaller of qps, cpus or msix */
11190                 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
11191                 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
11192                 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
11193                 pf->num_lan_qps = q_max;
11194
11195                 queues_left -= pf->num_lan_qps;
11196         }
11197
11198         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11199                 if (queues_left > 1) {
11200                         queues_left -= 1; /* save 1 queue for FD */
11201                 } else {
11202                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11203                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
11204                 }
11205         }
11206
11207         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11208             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
11209                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
11210                                         (queues_left / pf->num_vf_qps));
11211                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
11212         }
11213
11214         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11215             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
11216                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
11217                                           (queues_left / pf->num_vmdq_qps));
11218                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
11219         }
11220
11221         pf->queues_left = queues_left;
11222         dev_dbg(&pf->pdev->dev,
11223                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
11224                 pf->hw.func_caps.num_tx_qp,
11225                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
11226                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
11227                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
11228                 queues_left);
11229 }
11230
11231 /**
11232  * i40e_setup_pf_filter_control - Setup PF static filter control
11233  * @pf: PF to be setup
11234  *
11235  * i40e_setup_pf_filter_control sets up a PF's initial filter control
11236  * settings. If PE/FCoE are enabled then it will also set the per PF
11237  * based filter sizes required for them. It also enables Flow director,
11238  * ethertype and macvlan type filter settings for the pf.
11239  *
11240  * Returns 0 on success, negative on failure
11241  **/
11242 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
11243 {
11244         struct i40e_filter_control_settings *settings = &pf->filter_settings;
11245
11246         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
11247
11248         /* Flow Director is enabled */
11249         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
11250                 settings->enable_fdir = true;
11251
11252         /* Ethtype and MACVLAN filters enabled for PF */
11253         settings->enable_ethtype = true;
11254         settings->enable_macvlan = true;
11255
11256         if (i40e_set_filter_control(&pf->hw, settings))
11257                 return -ENOENT;
11258
11259         return 0;
11260 }
11261
11262 #define INFO_STRING_LEN 255
11263 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
11264 static void i40e_print_features(struct i40e_pf *pf)
11265 {
11266         struct i40e_hw *hw = &pf->hw;
11267         char *buf;
11268         int i;
11269
11270         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
11271         if (!buf)
11272                 return;
11273
11274         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
11275 #ifdef CONFIG_PCI_IOV
11276         i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
11277 #endif
11278         i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
11279                       pf->hw.func_caps.num_vsis,
11280                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
11281         if (pf->flags & I40E_FLAG_RSS_ENABLED)
11282                 i += snprintf(&buf[i], REMAIN(i), " RSS");
11283         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
11284                 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
11285         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11286                 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
11287                 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
11288         }
11289         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
11290                 i += snprintf(&buf[i], REMAIN(i), " DCB");
11291         i += snprintf(&buf[i], REMAIN(i), " VxLAN");
11292         i += snprintf(&buf[i], REMAIN(i), " Geneve");
11293         if (pf->flags & I40E_FLAG_PTP)
11294                 i += snprintf(&buf[i], REMAIN(i), " PTP");
11295         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
11296                 i += snprintf(&buf[i], REMAIN(i), " VEB");
11297         else
11298                 i += snprintf(&buf[i], REMAIN(i), " VEPA");
11299
11300         dev_info(&pf->pdev->dev, "%s\n", buf);
11301         kfree(buf);
11302         WARN_ON(i > INFO_STRING_LEN);
11303 }
11304
11305 /**
11306  * i40e_get_platform_mac_addr - get platform-specific MAC address
11307  * @pdev: PCI device information struct
11308  * @pf: board private structure
11309  *
11310  * Look up the MAC address for the device. First we'll try
11311  * eth_platform_get_mac_address, which will check Open Firmware, or arch
11312  * specific fallback. Otherwise, we'll default to the stored value in
11313  * firmware.
11314  **/
11315 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
11316 {
11317         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
11318                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
11319 }
11320
11321 /**
11322  * i40e_probe - Device initialization routine
11323  * @pdev: PCI device information struct
11324  * @ent: entry in i40e_pci_tbl
11325  *
11326  * i40e_probe initializes a PF identified by a pci_dev structure.
11327  * The OS initialization, configuring of the PF private structure,
11328  * and a hardware reset occur.
11329  *
11330  * Returns 0 on success, negative on failure
11331  **/
11332 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
11333 {
11334         struct i40e_aq_get_phy_abilities_resp abilities;
11335         struct i40e_pf *pf;
11336         struct i40e_hw *hw;
11337         static u16 pfs_found;
11338         u16 wol_nvm_bits;
11339         u16 link_status;
11340         int err;
11341         u32 val;
11342         u32 i;
11343         u8 set_fc_aq_fail;
11344
11345         err = pci_enable_device_mem(pdev);
11346         if (err)
11347                 return err;
11348
11349         /* set up for high or low dma */
11350         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
11351         if (err) {
11352                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
11353                 if (err) {
11354                         dev_err(&pdev->dev,
11355                                 "DMA configuration failed: 0x%x\n", err);
11356                         goto err_dma;
11357                 }
11358         }
11359
11360         /* set up pci connections */
11361         err = pci_request_mem_regions(pdev, i40e_driver_name);
11362         if (err) {
11363                 dev_info(&pdev->dev,
11364                          "pci_request_selected_regions failed %d\n", err);
11365                 goto err_pci_reg;
11366         }
11367
11368         pci_enable_pcie_error_reporting(pdev);
11369         pci_set_master(pdev);
11370
11371         /* Now that we have a PCI connection, we need to do the
11372          * low level device setup.  This is primarily setting up
11373          * the Admin Queue structures and then querying for the
11374          * device's current profile information.
11375          */
11376         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
11377         if (!pf) {
11378                 err = -ENOMEM;
11379                 goto err_pf_alloc;
11380         }
11381         pf->next_vsi = 0;
11382         pf->pdev = pdev;
11383         set_bit(__I40E_DOWN, pf->state);
11384
11385         hw = &pf->hw;
11386         hw->back = pf;
11387
11388         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
11389                                 I40E_MAX_CSR_SPACE);
11390
11391         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
11392         if (!hw->hw_addr) {
11393                 err = -EIO;
11394                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
11395                          (unsigned int)pci_resource_start(pdev, 0),
11396                          pf->ioremap_len, err);
11397                 goto err_ioremap;
11398         }
11399         hw->vendor_id = pdev->vendor;
11400         hw->device_id = pdev->device;
11401         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
11402         hw->subsystem_vendor_id = pdev->subsystem_vendor;
11403         hw->subsystem_device_id = pdev->subsystem_device;
11404         hw->bus.device = PCI_SLOT(pdev->devfn);
11405         hw->bus.func = PCI_FUNC(pdev->devfn);
11406         hw->bus.bus_id = pdev->bus->number;
11407         pf->instance = pfs_found;
11408
11409         /* Select something other than the 802.1ad ethertype for the
11410          * switch to use internally and drop on ingress.
11411          */
11412         hw->switch_tag = 0xffff;
11413         hw->first_tag = ETH_P_8021AD;
11414         hw->second_tag = ETH_P_8021Q;
11415
11416         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
11417         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
11418
11419         /* set up the locks for the AQ, do this only once in probe
11420          * and destroy them only once in remove
11421          */
11422         mutex_init(&hw->aq.asq_mutex);
11423         mutex_init(&hw->aq.arq_mutex);
11424
11425         pf->msg_enable = netif_msg_init(debug,
11426                                         NETIF_MSG_DRV |
11427                                         NETIF_MSG_PROBE |
11428                                         NETIF_MSG_LINK);
11429         if (debug < -1)
11430                 pf->hw.debug_mask = debug;
11431
11432         /* do a special CORER for clearing PXE mode once at init */
11433         if (hw->revision_id == 0 &&
11434             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
11435                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
11436                 i40e_flush(hw);
11437                 msleep(200);
11438                 pf->corer_count++;
11439
11440                 i40e_clear_pxe_mode(hw);
11441         }
11442
11443         /* Reset here to make sure all is clean and to define PF 'n' */
11444         i40e_clear_hw(hw);
11445         err = i40e_pf_reset(hw);
11446         if (err) {
11447                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
11448                 goto err_pf_reset;
11449         }
11450         pf->pfr_count++;
11451
11452         hw->aq.num_arq_entries = I40E_AQ_LEN;
11453         hw->aq.num_asq_entries = I40E_AQ_LEN;
11454         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
11455         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
11456         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
11457
11458         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
11459                  "%s-%s:misc",
11460                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
11461
11462         err = i40e_init_shared_code(hw);
11463         if (err) {
11464                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
11465                          err);
11466                 goto err_pf_reset;
11467         }
11468
11469         /* set up a default setting for link flow control */
11470         pf->hw.fc.requested_mode = I40E_FC_NONE;
11471
11472         err = i40e_init_adminq(hw);
11473         if (err) {
11474                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
11475                         dev_info(&pdev->dev,
11476                                  "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
11477                 else
11478                         dev_info(&pdev->dev,
11479                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
11480
11481                 goto err_pf_reset;
11482         }
11483         i40e_get_oem_version(hw);
11484
11485         /* provide nvm, fw, api versions */
11486         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n",
11487                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
11488                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
11489                  i40e_nvm_version_str(hw));
11490
11491         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
11492             hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
11493                 dev_info(&pdev->dev,
11494                          "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
11495         else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
11496                 dev_info(&pdev->dev,
11497                          "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
11498
11499         i40e_verify_eeprom(pf);
11500
11501         /* Rev 0 hardware was never productized */
11502         if (hw->revision_id < 1)
11503                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
11504
11505         i40e_clear_pxe_mode(hw);
11506         err = i40e_get_capabilities(pf);
11507         if (err)
11508                 goto err_adminq_setup;
11509
11510         err = i40e_sw_init(pf);
11511         if (err) {
11512                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
11513                 goto err_sw_init;
11514         }
11515
11516         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
11517                                 hw->func_caps.num_rx_qp, 0, 0);
11518         if (err) {
11519                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
11520                 goto err_init_lan_hmc;
11521         }
11522
11523         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
11524         if (err) {
11525                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
11526                 err = -ENOENT;
11527                 goto err_configure_lan_hmc;
11528         }
11529
11530         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
11531          * Ignore error return codes because if it was already disabled via
11532          * hardware settings this will fail
11533          */
11534         if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
11535                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
11536                 i40e_aq_stop_lldp(hw, true, NULL);
11537         }
11538
11539         /* allow a platform config to override the HW addr */
11540         i40e_get_platform_mac_addr(pdev, pf);
11541
11542         if (!is_valid_ether_addr(hw->mac.addr)) {
11543                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
11544                 err = -EIO;
11545                 goto err_mac_addr;
11546         }
11547         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
11548         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
11549         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
11550         if (is_valid_ether_addr(hw->mac.port_addr))
11551                 pf->hw_features |= I40E_HW_PORT_ID_VALID;
11552
11553         pci_set_drvdata(pdev, pf);
11554         pci_save_state(pdev);
11555 #ifdef CONFIG_I40E_DCB
11556         err = i40e_init_pf_dcb(pf);
11557         if (err) {
11558                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
11559                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
11560                 /* Continue without DCB enabled */
11561         }
11562 #endif /* CONFIG_I40E_DCB */
11563
11564         /* set up periodic task facility */
11565         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
11566         pf->service_timer_period = HZ;
11567
11568         INIT_WORK(&pf->service_task, i40e_service_task);
11569         clear_bit(__I40E_SERVICE_SCHED, pf->state);
11570
11571         /* NVM bit on means WoL disabled for the port */
11572         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
11573         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
11574                 pf->wol_en = false;
11575         else
11576                 pf->wol_en = true;
11577         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
11578
11579         /* set up the main switch operations */
11580         i40e_determine_queue_usage(pf);
11581         err = i40e_init_interrupt_scheme(pf);
11582         if (err)
11583                 goto err_switch_setup;
11584
11585         /* The number of VSIs reported by the FW is the minimum guaranteed
11586          * to us; HW supports far more and we share the remaining pool with
11587          * the other PFs. We allocate space for more than the guarantee with
11588          * the understanding that we might not get them all later.
11589          */
11590         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
11591                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
11592         else
11593                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
11594
11595         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
11596         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
11597                           GFP_KERNEL);
11598         if (!pf->vsi) {
11599                 err = -ENOMEM;
11600                 goto err_switch_setup;
11601         }
11602
11603 #ifdef CONFIG_PCI_IOV
11604         /* prep for VF support */
11605         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11606             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11607             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
11608                 if (pci_num_vf(pdev))
11609                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
11610         }
11611 #endif
11612         err = i40e_setup_pf_switch(pf, false);
11613         if (err) {
11614                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
11615                 goto err_vsis;
11616         }
11617
11618         /* Make sure flow control is set according to current settings */
11619         err = i40e_set_fc(hw, &set_fc_aq_fail, true);
11620         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET)
11621                 dev_dbg(&pf->pdev->dev,
11622                         "Set fc with err %s aq_err %s on get_phy_cap\n",
11623                         i40e_stat_str(hw, err),
11624                         i40e_aq_str(hw, hw->aq.asq_last_status));
11625         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET)
11626                 dev_dbg(&pf->pdev->dev,
11627                         "Set fc with err %s aq_err %s on set_phy_config\n",
11628                         i40e_stat_str(hw, err),
11629                         i40e_aq_str(hw, hw->aq.asq_last_status));
11630         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE)
11631                 dev_dbg(&pf->pdev->dev,
11632                         "Set fc with err %s aq_err %s on get_link_info\n",
11633                         i40e_stat_str(hw, err),
11634                         i40e_aq_str(hw, hw->aq.asq_last_status));
11635
11636         /* if FDIR VSI was set up, start it now */
11637         for (i = 0; i < pf->num_alloc_vsi; i++) {
11638                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
11639                         i40e_vsi_open(pf->vsi[i]);
11640                         break;
11641                 }
11642         }
11643
11644         /* The driver only wants link up/down and module qualification
11645          * reports from firmware.  Note the negative logic.
11646          */
11647         err = i40e_aq_set_phy_int_mask(&pf->hw,
11648                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
11649                                          I40E_AQ_EVENT_MEDIA_NA |
11650                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
11651         if (err)
11652                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
11653                          i40e_stat_str(&pf->hw, err),
11654                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11655
11656         /* Reconfigure hardware for allowing smaller MSS in the case
11657          * of TSO, so that we avoid the MDD being fired and causing
11658          * a reset in the case of small MSS+TSO.
11659          */
11660         val = rd32(hw, I40E_REG_MSS);
11661         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11662                 val &= ~I40E_REG_MSS_MIN_MASK;
11663                 val |= I40E_64BYTE_MSS;
11664                 wr32(hw, I40E_REG_MSS, val);
11665         }
11666
11667         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
11668                 msleep(75);
11669                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11670                 if (err)
11671                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
11672                                  i40e_stat_str(&pf->hw, err),
11673                                  i40e_aq_str(&pf->hw,
11674                                              pf->hw.aq.asq_last_status));
11675         }
11676         /* The main driver is (mostly) up and happy. We need to set this state
11677          * before setting up the misc vector or we get a race and the vector
11678          * ends up disabled forever.
11679          */
11680         clear_bit(__I40E_DOWN, pf->state);
11681
11682         /* In case of MSIX we are going to setup the misc vector right here
11683          * to handle admin queue events etc. In case of legacy and MSI
11684          * the misc functionality and queue processing is combined in
11685          * the same vector and that gets setup at open.
11686          */
11687         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11688                 err = i40e_setup_misc_vector(pf);
11689                 if (err) {
11690                         dev_info(&pdev->dev,
11691                                  "setup of misc vector failed: %d\n", err);
11692                         goto err_vsis;
11693                 }
11694         }
11695
11696 #ifdef CONFIG_PCI_IOV
11697         /* prep for VF support */
11698         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11699             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11700             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
11701                 /* disable link interrupts for VFs */
11702                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
11703                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
11704                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
11705                 i40e_flush(hw);
11706
11707                 if (pci_num_vf(pdev)) {
11708                         dev_info(&pdev->dev,
11709                                  "Active VFs found, allocating resources.\n");
11710                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
11711                         if (err)
11712                                 dev_info(&pdev->dev,
11713                                          "Error %d allocating resources for existing VFs\n",
11714                                          err);
11715                 }
11716         }
11717 #endif /* CONFIG_PCI_IOV */
11718
11719         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11720                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
11721                                                       pf->num_iwarp_msix,
11722                                                       I40E_IWARP_IRQ_PILE_ID);
11723                 if (pf->iwarp_base_vector < 0) {
11724                         dev_info(&pdev->dev,
11725                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
11726                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
11727                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11728                 }
11729         }
11730
11731         i40e_dbg_pf_init(pf);
11732
11733         /* tell the firmware that we're starting */
11734         i40e_send_version(pf);
11735
11736         /* since everything's happy, start the service_task timer */
11737         mod_timer(&pf->service_timer,
11738                   round_jiffies(jiffies + pf->service_timer_period));
11739
11740         /* add this PF to client device list and launch a client service task */
11741         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11742                 err = i40e_lan_add_device(pf);
11743                 if (err)
11744                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
11745                                  err);
11746         }
11747
11748 #define PCI_SPEED_SIZE 8
11749 #define PCI_WIDTH_SIZE 8
11750         /* Devices on the IOSF bus do not have this information
11751          * and will report PCI Gen 1 x 1 by default so don't bother
11752          * checking them.
11753          */
11754         if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
11755                 char speed[PCI_SPEED_SIZE] = "Unknown";
11756                 char width[PCI_WIDTH_SIZE] = "Unknown";
11757
11758                 /* Get the negotiated link width and speed from PCI config
11759                  * space
11760                  */
11761                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
11762                                           &link_status);
11763
11764                 i40e_set_pci_config_data(hw, link_status);
11765
11766                 switch (hw->bus.speed) {
11767                 case i40e_bus_speed_8000:
11768                         strncpy(speed, "8.0", PCI_SPEED_SIZE); break;
11769                 case i40e_bus_speed_5000:
11770                         strncpy(speed, "5.0", PCI_SPEED_SIZE); break;
11771                 case i40e_bus_speed_2500:
11772                         strncpy(speed, "2.5", PCI_SPEED_SIZE); break;
11773                 default:
11774                         break;
11775                 }
11776                 switch (hw->bus.width) {
11777                 case i40e_bus_width_pcie_x8:
11778                         strncpy(width, "8", PCI_WIDTH_SIZE); break;
11779                 case i40e_bus_width_pcie_x4:
11780                         strncpy(width, "4", PCI_WIDTH_SIZE); break;
11781                 case i40e_bus_width_pcie_x2:
11782                         strncpy(width, "2", PCI_WIDTH_SIZE); break;
11783                 case i40e_bus_width_pcie_x1:
11784                         strncpy(width, "1", PCI_WIDTH_SIZE); break;
11785                 default:
11786                         break;
11787                 }
11788
11789                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
11790                          speed, width);
11791
11792                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
11793                     hw->bus.speed < i40e_bus_speed_8000) {
11794                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
11795                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
11796                 }
11797         }
11798
11799         /* get the requested speeds from the fw */
11800         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
11801         if (err)
11802                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
11803                         i40e_stat_str(&pf->hw, err),
11804                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11805         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
11806
11807         /* get the supported phy types from the fw */
11808         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
11809         if (err)
11810                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
11811                         i40e_stat_str(&pf->hw, err),
11812                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11813
11814         /* Add a filter to drop all Flow control frames from any VSI from being
11815          * transmitted. By doing so we stop a malicious VF from sending out
11816          * PAUSE or PFC frames and potentially controlling traffic for other
11817          * PF/VF VSIs.
11818          * The FW can still send Flow control frames if enabled.
11819          */
11820         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11821                                                        pf->main_vsi_seid);
11822
11823         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
11824                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
11825                 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
11826         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
11827                 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
11828         /* print a string summarizing features */
11829         i40e_print_features(pf);
11830
11831         return 0;
11832
11833         /* Unwind what we've done if something failed in the setup */
11834 err_vsis:
11835         set_bit(__I40E_DOWN, pf->state);
11836         i40e_clear_interrupt_scheme(pf);
11837         kfree(pf->vsi);
11838 err_switch_setup:
11839         i40e_reset_interrupt_capability(pf);
11840         del_timer_sync(&pf->service_timer);
11841 err_mac_addr:
11842 err_configure_lan_hmc:
11843         (void)i40e_shutdown_lan_hmc(hw);
11844 err_init_lan_hmc:
11845         kfree(pf->qp_pile);
11846 err_sw_init:
11847 err_adminq_setup:
11848 err_pf_reset:
11849         iounmap(hw->hw_addr);
11850 err_ioremap:
11851         kfree(pf);
11852 err_pf_alloc:
11853         pci_disable_pcie_error_reporting(pdev);
11854         pci_release_mem_regions(pdev);
11855 err_pci_reg:
11856 err_dma:
11857         pci_disable_device(pdev);
11858         return err;
11859 }
11860
11861 /**
11862  * i40e_remove - Device removal routine
11863  * @pdev: PCI device information struct
11864  *
11865  * i40e_remove is called by the PCI subsystem to alert the driver
11866  * that is should release a PCI device.  This could be caused by a
11867  * Hot-Plug event, or because the driver is going to be removed from
11868  * memory.
11869  **/
11870 static void i40e_remove(struct pci_dev *pdev)
11871 {
11872         struct i40e_pf *pf = pci_get_drvdata(pdev);
11873         struct i40e_hw *hw = &pf->hw;
11874         i40e_status ret_code;
11875         int i;
11876
11877         i40e_dbg_pf_exit(pf);
11878
11879         i40e_ptp_stop(pf);
11880
11881         /* Disable RSS in hw */
11882         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
11883         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
11884
11885         /* no more scheduling of any task */
11886         set_bit(__I40E_SUSPENDED, pf->state);
11887         set_bit(__I40E_DOWN, pf->state);
11888         if (pf->service_timer.data)
11889                 del_timer_sync(&pf->service_timer);
11890         if (pf->service_task.func)
11891                 cancel_work_sync(&pf->service_task);
11892
11893         /* Client close must be called explicitly here because the timer
11894          * has been stopped.
11895          */
11896         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
11897
11898         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
11899                 i40e_free_vfs(pf);
11900                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
11901         }
11902
11903         i40e_fdir_teardown(pf);
11904
11905         /* If there is a switch structure or any orphans, remove them.
11906          * This will leave only the PF's VSI remaining.
11907          */
11908         for (i = 0; i < I40E_MAX_VEB; i++) {
11909                 if (!pf->veb[i])
11910                         continue;
11911
11912                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
11913                     pf->veb[i]->uplink_seid == 0)
11914                         i40e_switch_branch_release(pf->veb[i]);
11915         }
11916
11917         /* Now we can shutdown the PF's VSI, just before we kill
11918          * adminq and hmc.
11919          */
11920         if (pf->vsi[pf->lan_vsi])
11921                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
11922
11923         /* remove attached clients */
11924         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11925                 ret_code = i40e_lan_del_device(pf);
11926                 if (ret_code)
11927                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
11928                                  ret_code);
11929         }
11930
11931         /* shutdown and destroy the HMC */
11932         if (hw->hmc.hmc_obj) {
11933                 ret_code = i40e_shutdown_lan_hmc(hw);
11934                 if (ret_code)
11935                         dev_warn(&pdev->dev,
11936                                  "Failed to destroy the HMC resources: %d\n",
11937                                  ret_code);
11938         }
11939
11940         /* shutdown the adminq */
11941         i40e_shutdown_adminq(hw);
11942
11943         /* destroy the locks only once, here */
11944         mutex_destroy(&hw->aq.arq_mutex);
11945         mutex_destroy(&hw->aq.asq_mutex);
11946
11947         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
11948         i40e_clear_interrupt_scheme(pf);
11949         for (i = 0; i < pf->num_alloc_vsi; i++) {
11950                 if (pf->vsi[i]) {
11951                         i40e_vsi_clear_rings(pf->vsi[i]);
11952                         i40e_vsi_clear(pf->vsi[i]);
11953                         pf->vsi[i] = NULL;
11954                 }
11955         }
11956
11957         for (i = 0; i < I40E_MAX_VEB; i++) {
11958                 kfree(pf->veb[i]);
11959                 pf->veb[i] = NULL;
11960         }
11961
11962         kfree(pf->qp_pile);
11963         kfree(pf->vsi);
11964
11965         iounmap(hw->hw_addr);
11966         kfree(pf);
11967         pci_release_mem_regions(pdev);
11968
11969         pci_disable_pcie_error_reporting(pdev);
11970         pci_disable_device(pdev);
11971 }
11972
11973 /**
11974  * i40e_pci_error_detected - warning that something funky happened in PCI land
11975  * @pdev: PCI device information struct
11976  *
11977  * Called to warn that something happened and the error handling steps
11978  * are in progress.  Allows the driver to quiesce things, be ready for
11979  * remediation.
11980  **/
11981 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
11982                                                 enum pci_channel_state error)
11983 {
11984         struct i40e_pf *pf = pci_get_drvdata(pdev);
11985
11986         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
11987
11988         if (!pf) {
11989                 dev_info(&pdev->dev,
11990                          "Cannot recover - error happened during device probe\n");
11991                 return PCI_ERS_RESULT_DISCONNECT;
11992         }
11993
11994         /* shutdown all operations */
11995         if (!test_bit(__I40E_SUSPENDED, pf->state))
11996                 i40e_prep_for_reset(pf, false);
11997
11998         /* Request a slot reset */
11999         return PCI_ERS_RESULT_NEED_RESET;
12000 }
12001
12002 /**
12003  * i40e_pci_error_slot_reset - a PCI slot reset just happened
12004  * @pdev: PCI device information struct
12005  *
12006  * Called to find if the driver can work with the device now that
12007  * the pci slot has been reset.  If a basic connection seems good
12008  * (registers are readable and have sane content) then return a
12009  * happy little PCI_ERS_RESULT_xxx.
12010  **/
12011 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
12012 {
12013         struct i40e_pf *pf = pci_get_drvdata(pdev);
12014         pci_ers_result_t result;
12015         int err;
12016         u32 reg;
12017
12018         dev_dbg(&pdev->dev, "%s\n", __func__);
12019         if (pci_enable_device_mem(pdev)) {
12020                 dev_info(&pdev->dev,
12021                          "Cannot re-enable PCI device after reset.\n");
12022                 result = PCI_ERS_RESULT_DISCONNECT;
12023         } else {
12024                 pci_set_master(pdev);
12025                 pci_restore_state(pdev);
12026                 pci_save_state(pdev);
12027                 pci_wake_from_d3(pdev, false);
12028
12029                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
12030                 if (reg == 0)
12031                         result = PCI_ERS_RESULT_RECOVERED;
12032                 else
12033                         result = PCI_ERS_RESULT_DISCONNECT;
12034         }
12035
12036         err = pci_cleanup_aer_uncorrect_error_status(pdev);
12037         if (err) {
12038                 dev_info(&pdev->dev,
12039                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
12040                          err);
12041                 /* non-fatal, continue */
12042         }
12043
12044         return result;
12045 }
12046
12047 /**
12048  * i40e_pci_error_resume - restart operations after PCI error recovery
12049  * @pdev: PCI device information struct
12050  *
12051  * Called to allow the driver to bring things back up after PCI error
12052  * and/or reset recovery has finished.
12053  **/
12054 static void i40e_pci_error_resume(struct pci_dev *pdev)
12055 {
12056         struct i40e_pf *pf = pci_get_drvdata(pdev);
12057
12058         dev_dbg(&pdev->dev, "%s\n", __func__);
12059         if (test_bit(__I40E_SUSPENDED, pf->state))
12060                 return;
12061
12062         i40e_handle_reset_warning(pf, false);
12063 }
12064
12065 /**
12066  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
12067  * using the mac_address_write admin q function
12068  * @pf: pointer to i40e_pf struct
12069  **/
12070 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
12071 {
12072         struct i40e_hw *hw = &pf->hw;
12073         i40e_status ret;
12074         u8 mac_addr[6];
12075         u16 flags = 0;
12076
12077         /* Get current MAC address in case it's an LAA */
12078         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
12079                 ether_addr_copy(mac_addr,
12080                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
12081         } else {
12082                 dev_err(&pf->pdev->dev,
12083                         "Failed to retrieve MAC address; using default\n");
12084                 ether_addr_copy(mac_addr, hw->mac.addr);
12085         }
12086
12087         /* The FW expects the mac address write cmd to first be called with
12088          * one of these flags before calling it again with the multicast
12089          * enable flags.
12090          */
12091         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
12092
12093         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
12094                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
12095
12096         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
12097         if (ret) {
12098                 dev_err(&pf->pdev->dev,
12099                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
12100                 return;
12101         }
12102
12103         flags = I40E_AQC_MC_MAG_EN
12104                         | I40E_AQC_WOL_PRESERVE_ON_PFR
12105                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
12106         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
12107         if (ret)
12108                 dev_err(&pf->pdev->dev,
12109                         "Failed to enable Multicast Magic Packet wake up\n");
12110 }
12111
12112 /**
12113  * i40e_shutdown - PCI callback for shutting down
12114  * @pdev: PCI device information struct
12115  **/
12116 static void i40e_shutdown(struct pci_dev *pdev)
12117 {
12118         struct i40e_pf *pf = pci_get_drvdata(pdev);
12119         struct i40e_hw *hw = &pf->hw;
12120
12121         set_bit(__I40E_SUSPENDED, pf->state);
12122         set_bit(__I40E_DOWN, pf->state);
12123         rtnl_lock();
12124         i40e_prep_for_reset(pf, true);
12125         rtnl_unlock();
12126
12127         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12128         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12129
12130         del_timer_sync(&pf->service_timer);
12131         cancel_work_sync(&pf->service_task);
12132         i40e_fdir_teardown(pf);
12133
12134         /* Client close must be called explicitly here because the timer
12135          * has been stopped.
12136          */
12137         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
12138
12139         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
12140                 i40e_enable_mc_magic_wake(pf);
12141
12142         i40e_prep_for_reset(pf, false);
12143
12144         wr32(hw, I40E_PFPM_APM,
12145              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12146         wr32(hw, I40E_PFPM_WUFC,
12147              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12148
12149         i40e_clear_interrupt_scheme(pf);
12150
12151         if (system_state == SYSTEM_POWER_OFF) {
12152                 pci_wake_from_d3(pdev, pf->wol_en);
12153                 pci_set_power_state(pdev, PCI_D3hot);
12154         }
12155 }
12156
12157 #ifdef CONFIG_PM
12158 /**
12159  * i40e_suspend - PM callback for moving to D3
12160  * @dev: generic device information structure
12161  **/
12162 static int i40e_suspend(struct device *dev)
12163 {
12164         struct pci_dev *pdev = to_pci_dev(dev);
12165         struct i40e_pf *pf = pci_get_drvdata(pdev);
12166         struct i40e_hw *hw = &pf->hw;
12167
12168         /* If we're already suspended, then there is nothing to do */
12169         if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
12170                 return 0;
12171
12172         set_bit(__I40E_DOWN, pf->state);
12173
12174         /* Ensure service task will not be running */
12175         del_timer_sync(&pf->service_timer);
12176         cancel_work_sync(&pf->service_task);
12177
12178         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
12179                 i40e_enable_mc_magic_wake(pf);
12180
12181         i40e_prep_for_reset(pf, false);
12182
12183         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
12184         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
12185
12186         /* Clear the interrupt scheme and release our IRQs so that the system
12187          * can safely hibernate even when there are a large number of CPUs.
12188          * Otherwise hibernation might fail when mapping all the vectors back
12189          * to CPU0.
12190          */
12191         i40e_clear_interrupt_scheme(pf);
12192
12193         return 0;
12194 }
12195
12196 /**
12197  * i40e_resume - PM callback for waking up from D3
12198  * @dev: generic device information structure
12199  **/
12200 static int i40e_resume(struct device *dev)
12201 {
12202         struct pci_dev *pdev = to_pci_dev(dev);
12203         struct i40e_pf *pf = pci_get_drvdata(pdev);
12204         int err;
12205
12206         /* If we're not suspended, then there is nothing to do */
12207         if (!test_bit(__I40E_SUSPENDED, pf->state))
12208                 return 0;
12209
12210         /* We cleared the interrupt scheme when we suspended, so we need to
12211          * restore it now to resume device functionality.
12212          */
12213         err = i40e_restore_interrupt_scheme(pf);
12214         if (err) {
12215                 dev_err(&pdev->dev, "Cannot restore interrupt scheme: %d\n",
12216                         err);
12217         }
12218
12219         clear_bit(__I40E_DOWN, pf->state);
12220         i40e_reset_and_rebuild(pf, false, false);
12221
12222         /* Clear suspended state last after everything is recovered */
12223         clear_bit(__I40E_SUSPENDED, pf->state);
12224
12225         /* Restart the service task */
12226         mod_timer(&pf->service_timer,
12227                   round_jiffies(jiffies + pf->service_timer_period));
12228
12229         return 0;
12230 }
12231
12232 #endif /* CONFIG_PM */
12233
12234 static const struct pci_error_handlers i40e_err_handler = {
12235         .error_detected = i40e_pci_error_detected,
12236         .slot_reset = i40e_pci_error_slot_reset,
12237         .resume = i40e_pci_error_resume,
12238 };
12239
12240 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
12241
12242 static struct pci_driver i40e_driver = {
12243         .name     = i40e_driver_name,
12244         .id_table = i40e_pci_tbl,
12245         .probe    = i40e_probe,
12246         .remove   = i40e_remove,
12247 #ifdef CONFIG_PM
12248         .driver   = {
12249                 .pm = &i40e_pm_ops,
12250         },
12251 #endif /* CONFIG_PM */
12252         .shutdown = i40e_shutdown,
12253         .err_handler = &i40e_err_handler,
12254         .sriov_configure = i40e_pci_sriov_configure,
12255 };
12256
12257 /**
12258  * i40e_init_module - Driver registration routine
12259  *
12260  * i40e_init_module is the first routine called when the driver is
12261  * loaded. All it does is register with the PCI subsystem.
12262  **/
12263 static int __init i40e_init_module(void)
12264 {
12265         pr_info("%s: %s - version %s\n", i40e_driver_name,
12266                 i40e_driver_string, i40e_driver_version_str);
12267         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
12268
12269         /* There is no need to throttle the number of active tasks because
12270          * each device limits its own task using a state bit for scheduling
12271          * the service task, and the device tasks do not interfere with each
12272          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
12273          * since we need to be able to guarantee forward progress even under
12274          * memory pressure.
12275          */
12276         i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
12277         if (!i40e_wq) {
12278                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
12279                 return -ENOMEM;
12280         }
12281
12282         i40e_dbg_init();
12283         return pci_register_driver(&i40e_driver);
12284 }
12285 module_init(i40e_init_module);
12286
12287 /**
12288  * i40e_exit_module - Driver exit cleanup routine
12289  *
12290  * i40e_exit_module is called just before the driver is removed
12291  * from memory.
12292  **/
12293 static void __exit i40e_exit_module(void)
12294 {
12295         pci_unregister_driver(&i40e_driver);
12296         destroy_workqueue(i40e_wq);
12297         i40e_dbg_exit();
12298 }
12299 module_exit(i40e_exit_module);