drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2016 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 #include <linux/etherdevice.h>
  28 #include <linux/of_net.h>
  29 #include <linux/pci.h>
  30
  31 /* Local includes */
  32 #include "i40e.h"
  33 #include "i40e_diag.h"
  34 #if IS_ENABLED(CONFIG_VXLAN)
  35 #include <net/vxlan.h>
  36 #endif
  37 #if IS_ENABLED(CONFIG_GENEVE)
  38 #include <net/geneve.h>
  39 #endif
  40
  41 const char i40e_driver_name[] = "i40e";
  42 static const char i40e_driver_string[] =
  43                         "Intel(R) Ethernet Connection XL710 Network Driver";
  44
  45 #define DRV_KERN "-k"
  46
  47 #define DRV_VERSION_MAJOR 1
  48 #define DRV_VERSION_MINOR 5
  49 #define DRV_VERSION_BUILD 5
  50 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  51              __stringify(DRV_VERSION_MINOR) "." \
  52              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  53 const char i40e_driver_version_str[] = DRV_VERSION;
  54 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  55
  56 /* a bit of forward declarations */
  57 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  58 static void i40e_handle_reset_warning(struct i40e_pf *pf);
  59 static int i40e_add_vsi(struct i40e_vsi *vsi);
  60 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  61 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  62 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  63 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  64 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  65 static void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
  66                               u16 rss_table_size, u16 rss_size);
  67 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  68 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  69
  70 /* i40e_pci_tbl - PCI Device ID Table
  71  *
  72  * Last entry must be all 0s
  73  *
  74  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  75  *   Class, Class Mask, private data (not used) }
  76  */
  77 static const struct pci_device_id i40e_pci_tbl[] = {
  78         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  79         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  80         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  86         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  87         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  88         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  89         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  90         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  91         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  92         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  93         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  94         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  95         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  96         /* required last entry */
  97         {0, }
  98 };
  99 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
 100
 101 #define I40E_MAX_VF_COUNT 128
 102 static int debug = -1;
 103 module_param(debug, int, 0);
 104 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 105
 106 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 107 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
 108 MODULE_LICENSE("GPL");
 109 MODULE_VERSION(DRV_VERSION);
 110
 111 static struct workqueue_struct *i40e_wq;
 112
 113 /**
 114  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 115  * @hw:   pointer to the HW structure
 116  * @mem:  ptr to mem struct to fill out
 117  * @size: size of memory requested
 118  * @alignment: what to align the allocation to
 119  **/
 120 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 121                             u64 size, u32 alignment)
 122 {
 123         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 124
 125         mem->size = ALIGN(size, alignment);
 126         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 127                                       &mem->pa, GFP_KERNEL);
 128         if (!mem->va)
 129                 return -ENOMEM;
 130
 131         return 0;
 132 }
 133
 134 /**
 135  * i40e_free_dma_mem_d - OS specific memory free for shared code
 136  * @hw:   pointer to the HW structure
 137  * @mem:  ptr to mem struct to free
 138  **/
 139 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 140 {
 141         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 142
 143         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 144         mem->va = NULL;
 145         mem->pa = 0;
 146         mem->size = 0;
 147
 148         return 0;
 149 }
 150
 151 /**
 152  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 153  * @hw:   pointer to the HW structure
 154  * @mem:  ptr to mem struct to fill out
 155  * @size: size of memory requested
 156  **/
 157 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 158                              u32 size)
 159 {
 160         mem->size = size;
 161         mem->va = kzalloc(size, GFP_KERNEL);
 162
 163         if (!mem->va)
 164                 return -ENOMEM;
 165
 166         return 0;
 167 }
 168
 169 /**
 170  * i40e_free_virt_mem_d - OS specific memory free for shared code
 171  * @hw:   pointer to the HW structure
 172  * @mem:  ptr to mem struct to free
 173  **/
 174 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 175 {
 176         /* it's ok to kfree a NULL pointer */
 177         kfree(mem->va);
 178         mem->va = NULL;
 179         mem->size = 0;
 180
 181         return 0;
 182 }
 183
 184 /**
 185  * i40e_get_lump - find a lump of free generic resource
 186  * @pf: board private structure
 187  * @pile: the pile of resource to search
 188  * @needed: the number of items needed
 189  * @id: an owner id to stick on the items assigned
 190  *
 191  * Returns the base item index of the lump, or negative for error
 192  *
 193  * The search_hint trick and lack of advanced fit-finding only work
 194  * because we're highly likely to have all the same size lump requests.
 195  * Linear search time and any fragmentation should be minimal.
 196  **/
 197 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 198                          u16 needed, u16 id)
 199 {
 200         int ret = -ENOMEM;
 201         int i, j;
 202
 203         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 204                 dev_info(&pf->pdev->dev,
 205                          "param err: pile=%p needed=%d id=0x%04x\n",
 206                          pile, needed, id);
 207                 return -EINVAL;
 208         }
 209
 210         /* start the linear search with an imperfect hint */
 211         i = pile->search_hint;
 212         while (i < pile->num_entries) {
 213                 /* skip already allocated entries */
 214                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 215                         i++;
 216                         continue;
 217                 }
 218
 219                 /* do we have enough in this lump? */
 220                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 221                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 222                                 break;
 223                 }
 224
 225                 if (j == needed) {
 226                         /* there was enough, so assign it to the requestor */
 227                         for (j = 0; j < needed; j++)
 228                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 229                         ret = i;
 230                         pile->search_hint = i + j;
 231                         break;
 232                 }
 233
 234                 /* not enough, so skip over it and continue looking */
 235                 i += j;
 236         }
 237
 238         return ret;
 239 }
 240
 241 /**
 242  * i40e_put_lump - return a lump of generic resource
 243  * @pile: the pile of resource to search
 244  * @index: the base item index
 245  * @id: the owner id of the items assigned
 246  *
 247  * Returns the count of items in the lump
 248  **/
 249 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 250 {
 251         int valid_id = (id | I40E_PILE_VALID_BIT);
 252         int count = 0;
 253         int i;
 254
 255         if (!pile || index >= pile->num_entries)
 256                 return -EINVAL;
 257
 258         for (i = index;
 259              i < pile->num_entries && pile->list[i] == valid_id;
 260              i++) {
 261                 pile->list[i] = 0;
 262                 count++;
 263         }
 264
 265         if (count && index < pile->search_hint)
 266                 pile->search_hint = index;
 267
 268         return count;
 269 }
 270
 271 /**
 272  * i40e_find_vsi_from_id - searches for the vsi with the given id
 273  * @pf - the pf structure to search for the vsi
 274  * @id - id of the vsi it is searching for
 275  **/
 276 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 277 {
 278         int i;
 279
 280         for (i = 0; i < pf->num_alloc_vsi; i++)
 281                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 282                         return pf->vsi[i];
 283
 284         return NULL;
 285 }
 286
 287 /**
 288  * i40e_service_event_schedule - Schedule the service task to wake up
 289  * @pf: board private structure
 290  *
 291  * If not already scheduled, this puts the task into the work queue
 292  **/
 293 void i40e_service_event_schedule(struct i40e_pf *pf)
 294 {
 295         if (!test_bit(__I40E_DOWN, &pf->state) &&
 296             !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state) &&
 297             !test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state))
 298                 queue_work(i40e_wq, &pf->service_task);
 299 }
 300
 301 /**
 302  * i40e_tx_timeout - Respond to a Tx Hang
 303  * @netdev: network interface device structure
 304  *
 305  * If any port has noticed a Tx timeout, it is likely that the whole
 306  * device is munged, not just the one netdev port, so go for the full
 307  * reset.
 308  **/
 309 #ifdef I40E_FCOE
 310 void i40e_tx_timeout(struct net_device *netdev)
 311 #else
 312 static void i40e_tx_timeout(struct net_device *netdev)
 313 #endif
 314 {
 315         struct i40e_netdev_priv *np = netdev_priv(netdev);
 316         struct i40e_vsi *vsi = np->vsi;
 317         struct i40e_pf *pf = vsi->back;
 318         struct i40e_ring *tx_ring = NULL;
 319         unsigned int i, hung_queue = 0;
 320         u32 head, val;
 321
 322         pf->tx_timeout_count++;
 323
 324         /* find the stopped queue the same way the stack does */
 325         for (i = 0; i < netdev->num_tx_queues; i++) {
 326                 struct netdev_queue *q;
 327                 unsigned long trans_start;
 328
 329                 q = netdev_get_tx_queue(netdev, i);
 330                 trans_start = q->trans_start ? : netdev->trans_start;
 331                 if (netif_xmit_stopped(q) &&
 332                     time_after(jiffies,
 333                                (trans_start + netdev->watchdog_timeo))) {
 334                         hung_queue = i;
 335                         break;
 336                 }
 337         }
 338
 339         if (i == netdev->num_tx_queues) {
 340                 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
 341         } else {
 342                 /* now that we have an index, find the tx_ring struct */
 343                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 344                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 345                                 if (hung_queue ==
 346                                     vsi->tx_rings[i]->queue_index) {
 347                                         tx_ring = vsi->tx_rings[i];
 348                                         break;
 349                                 }
 350                         }
 351                 }
 352         }
 353
 354         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 355                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 356         else if (time_before(jiffies,
 357                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 358                 return;   /* don't do any new action before the next timeout */
 359
 360         if (tx_ring) {
 361                 head = i40e_get_head(tx_ring);
 362                 /* Read interrupt register */
 363                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 364                         val = rd32(&pf->hw,
 365                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 366                                                 tx_ring->vsi->base_vector - 1));
 367                 else
 368                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 369
 370                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 371                             vsi->seid, hung_queue, tx_ring->next_to_clean,
 372                             head, tx_ring->next_to_use,
 373                             readl(tx_ring->tail), val);
 374         }
 375
 376         pf->tx_timeout_last_recovery = jiffies;
 377         netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
 378                     pf->tx_timeout_recovery_level, hung_queue);
 379
 380         switch (pf->tx_timeout_recovery_level) {
 381         case 1:
 382                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
 383                 break;
 384         case 2:
 385                 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
 386                 break;
 387         case 3:
 388                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
 389                 break;
 390         default:
 391                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 392                 break;
 393         }
 394
 395         i40e_service_event_schedule(pf);
 396         pf->tx_timeout_recovery_level++;
 397 }
 398
 399 /**
 400  * i40e_release_rx_desc - Store the new tail and head values
 401  * @rx_ring: ring to bump
 402  * @val: new head index
 403  **/
 404 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
 405 {
 406         rx_ring->next_to_use = val;
 407
 408         /* Force memory writes to complete before letting h/w
 409          * know there are new descriptors to fetch.  (Only
 410          * applicable for weak-ordered memory model archs,
 411          * such as IA-64).
 412          */
 413         wmb();
 414         writel(val, rx_ring->tail);
 415 }
 416
 417 /**
 418  * i40e_get_vsi_stats_struct - Get System Network Statistics
 419  * @vsi: the VSI we care about
 420  *
 421  * Returns the address of the device statistics structure.
 422  * The statistics are actually updated from the service task.
 423  **/
 424 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 425 {
 426         return &vsi->net_stats;
 427 }
 428
 429 /**
 430  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 431  * @netdev: network interface device structure
 432  *
 433  * Returns the address of the device statistics structure.
 434  * The statistics are actually updated from the service task.
 435  **/
 436 #ifdef I40E_FCOE
 437 struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
 438                                              struct net_device *netdev,
 439                                              struct rtnl_link_stats64 *stats)
 440 #else
 441 static struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
 442                                              struct net_device *netdev,
 443                                              struct rtnl_link_stats64 *stats)
 444 #endif
 445 {
 446         struct i40e_netdev_priv *np = netdev_priv(netdev);
 447         struct i40e_ring *tx_ring, *rx_ring;
 448         struct i40e_vsi *vsi = np->vsi;
 449         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 450         int i;
 451
 452         if (test_bit(__I40E_DOWN, &vsi->state))
 453                 return stats;
 454
 455         if (!vsi->tx_rings)
 456                 return stats;
 457
 458         rcu_read_lock();
 459         for (i = 0; i < vsi->num_queue_pairs; i++) {
 460                 u64 bytes, packets;
 461                 unsigned int start;
 462
 463                 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
 464                 if (!tx_ring)
 465                         continue;
 466
 467                 do {
 468                         start = u64_stats_fetch_begin_irq(&tx_ring->syncp);
 469                         packets = tx_ring->stats.packets;
 470                         bytes   = tx_ring->stats.bytes;
 471                 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));
 472
 473                 stats->tx_packets += packets;
 474                 stats->tx_bytes   += bytes;
 475                 rx_ring = &tx_ring[1];
 476
 477                 do {
 478                         start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
 479                         packets = rx_ring->stats.packets;
 480                         bytes   = rx_ring->stats.bytes;
 481                 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
 482
 483                 stats->rx_packets += packets;
 484                 stats->rx_bytes   += bytes;
 485         }
 486         rcu_read_unlock();
 487
 488         /* following stats updated by i40e_watchdog_subtask() */
 489         stats->multicast        = vsi_stats->multicast;
 490         stats->tx_errors        = vsi_stats->tx_errors;
 491         stats->tx_dropped       = vsi_stats->tx_dropped;
 492         stats->rx_errors        = vsi_stats->rx_errors;
 493         stats->rx_dropped       = vsi_stats->rx_dropped;
 494         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 495         stats->rx_length_errors = vsi_stats->rx_length_errors;
 496
 497         return stats;
 498 }
 499
 500 /**
 501  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 502  * @vsi: the VSI to have its stats reset
 503  **/
 504 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 505 {
 506         struct rtnl_link_stats64 *ns;
 507         int i;
 508
 509         if (!vsi)
 510                 return;
 511
 512         ns = i40e_get_vsi_stats_struct(vsi);
 513         memset(ns, 0, sizeof(*ns));
 514         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 515         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 516         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 517         if (vsi->rx_rings && vsi->rx_rings[0]) {
 518                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 519                         memset(&vsi->rx_rings[i]->stats, 0,
 520                                sizeof(vsi->rx_rings[i]->stats));
 521                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 522                                sizeof(vsi->rx_rings[i]->rx_stats));
 523                         memset(&vsi->tx_rings[i]->stats, 0,
 524                                sizeof(vsi->tx_rings[i]->stats));
 525                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 526                                sizeof(vsi->tx_rings[i]->tx_stats));
 527                 }
 528         }
 529         vsi->stat_offsets_loaded = false;
 530 }
 531
 532 /**
 533  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 534  * @pf: the PF to be reset
 535  **/
 536 void i40e_pf_reset_stats(struct i40e_pf *pf)
 537 {
 538         int i;
 539
 540         memset(&pf->stats, 0, sizeof(pf->stats));
 541         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 542         pf->stat_offsets_loaded = false;
 543
 544         for (i = 0; i < I40E_MAX_VEB; i++) {
 545                 if (pf->veb[i]) {
 546                         memset(&pf->veb[i]->stats, 0,
 547                                sizeof(pf->veb[i]->stats));
 548                         memset(&pf->veb[i]->stats_offsets, 0,
 549                                sizeof(pf->veb[i]->stats_offsets));
 550                         pf->veb[i]->stat_offsets_loaded = false;
 551                 }
 552         }
 553 }
 554
 555 /**
 556  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 557  * @hw: ptr to the hardware info
 558  * @hireg: the high 32 bit reg to read
 559  * @loreg: the low 32 bit reg to read
 560  * @offset_loaded: has the initial offset been loaded yet
 561  * @offset: ptr to current offset value
 562  * @stat: ptr to the stat
 563  *
 564  * Since the device stats are not reset at PFReset, they likely will not
 565  * be zeroed when the driver starts.  We'll save the first values read
 566  * and use them as offsets to be subtracted from the raw values in order
 567  * to report stats that count from zero.  In the process, we also manage
 568  * the potential roll-over.
 569  **/
 570 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 571                                bool offset_loaded, u64 *offset, u64 *stat)
 572 {
 573         u64 new_data;
 574
 575         if (hw->device_id == I40E_DEV_ID_QEMU) {
 576                 new_data = rd32(hw, loreg);
 577                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 578         } else {
 579                 new_data = rd64(hw, loreg);
 580         }
 581         if (!offset_loaded)
 582                 *offset = new_data;
 583         if (likely(new_data >= *offset))
 584                 *stat = new_data - *offset;
 585         else
 586                 *stat = (new_data + BIT_ULL(48)) - *offset;
 587         *stat &= 0xFFFFFFFFFFFFULL;
 588 }
 589
 590 /**
 591  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 592  * @hw: ptr to the hardware info
 593  * @reg: the hw reg to read
 594  * @offset_loaded: has the initial offset been loaded yet
 595  * @offset: ptr to current offset value
 596  * @stat: ptr to the stat
 597  **/
 598 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 599                                bool offset_loaded, u64 *offset, u64 *stat)
 600 {
 601         u32 new_data;
 602
 603         new_data = rd32(hw, reg);
 604         if (!offset_loaded)
 605                 *offset = new_data;
 606         if (likely(new_data >= *offset))
 607                 *stat = (u32)(new_data - *offset);
 608         else
 609                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 610 }
 611
 612 /**
 613  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 614  * @vsi: the VSI to be updated
 615  **/
 616 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 617 {
 618         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 619         struct i40e_pf *pf = vsi->back;
 620         struct i40e_hw *hw = &pf->hw;
 621         struct i40e_eth_stats *oes;
 622         struct i40e_eth_stats *es;     /* device's eth stats */
 623
 624         es = &vsi->eth_stats;
 625         oes = &vsi->eth_stats_offsets;
 626
 627         /* Gather up the stats that the hw collects */
 628         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 629                            vsi->stat_offsets_loaded,
 630                            &oes->tx_errors, &es->tx_errors);
 631         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 632                            vsi->stat_offsets_loaded,
 633                            &oes->rx_discards, &es->rx_discards);
 634         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 635                            vsi->stat_offsets_loaded,
 636                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 637         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 638                            vsi->stat_offsets_loaded,
 639                            &oes->tx_errors, &es->tx_errors);
 640
 641         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 642                            I40E_GLV_GORCL(stat_idx),
 643                            vsi->stat_offsets_loaded,
 644                            &oes->rx_bytes, &es->rx_bytes);
 645         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 646                            I40E_GLV_UPRCL(stat_idx),
 647                            vsi->stat_offsets_loaded,
 648                            &oes->rx_unicast, &es->rx_unicast);
 649         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 650                            I40E_GLV_MPRCL(stat_idx),
 651                            vsi->stat_offsets_loaded,
 652                            &oes->rx_multicast, &es->rx_multicast);
 653         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 654                            I40E_GLV_BPRCL(stat_idx),
 655                            vsi->stat_offsets_loaded,
 656                            &oes->rx_broadcast, &es->rx_broadcast);
 657
 658         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 659                            I40E_GLV_GOTCL(stat_idx),
 660                            vsi->stat_offsets_loaded,
 661                            &oes->tx_bytes, &es->tx_bytes);
 662         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 663                            I40E_GLV_UPTCL(stat_idx),
 664                            vsi->stat_offsets_loaded,
 665                            &oes->tx_unicast, &es->tx_unicast);
 666         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 667                            I40E_GLV_MPTCL(stat_idx),
 668                            vsi->stat_offsets_loaded,
 669                            &oes->tx_multicast, &es->tx_multicast);
 670         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 671                            I40E_GLV_BPTCL(stat_idx),
 672                            vsi->stat_offsets_loaded,
 673                            &oes->tx_broadcast, &es->tx_broadcast);
 674         vsi->stat_offsets_loaded = true;
 675 }
 676
 677 /**
 678  * i40e_update_veb_stats - Update Switch component statistics
 679  * @veb: the VEB being updated
 680  **/
 681 static void i40e_update_veb_stats(struct i40e_veb *veb)
 682 {
 683         struct i40e_pf *pf = veb->pf;
 684         struct i40e_hw *hw = &pf->hw;
 685         struct i40e_eth_stats *oes;
 686         struct i40e_eth_stats *es;     /* device's eth stats */
 687         struct i40e_veb_tc_stats *veb_oes;
 688         struct i40e_veb_tc_stats *veb_es;
 689         int i, idx = 0;
 690
 691         idx = veb->stats_idx;
 692         es = &veb->stats;
 693         oes = &veb->stats_offsets;
 694         veb_es = &veb->tc_stats;
 695         veb_oes = &veb->tc_stats_offsets;
 696
 697         /* Gather up the stats that the hw collects */
 698         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 699                            veb->stat_offsets_loaded,
 700                            &oes->tx_discards, &es->tx_discards);
 701         if (hw->revision_id > 0)
 702                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 703                                    veb->stat_offsets_loaded,
 704                                    &oes->rx_unknown_protocol,
 705                                    &es->rx_unknown_protocol);
 706         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 707                            veb->stat_offsets_loaded,
 708                            &oes->rx_bytes, &es->rx_bytes);
 709         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 710                            veb->stat_offsets_loaded,
 711                            &oes->rx_unicast, &es->rx_unicast);
 712         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 713                            veb->stat_offsets_loaded,
 714                            &oes->rx_multicast, &es->rx_multicast);
 715         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 716                            veb->stat_offsets_loaded,
 717                            &oes->rx_broadcast, &es->rx_broadcast);
 718
 719         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 720                            veb->stat_offsets_loaded,
 721                            &oes->tx_bytes, &es->tx_bytes);
 722         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 723                            veb->stat_offsets_loaded,
 724                            &oes->tx_unicast, &es->tx_unicast);
 725         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 726                            veb->stat_offsets_loaded,
 727                            &oes->tx_multicast, &es->tx_multicast);
 728         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 729                            veb->stat_offsets_loaded,
 730                            &oes->tx_broadcast, &es->tx_broadcast);
 731         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 732                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 733                                    I40E_GLVEBTC_RPCL(i, idx),
 734                                    veb->stat_offsets_loaded,
 735                                    &veb_oes->tc_rx_packets[i],
 736                                    &veb_es->tc_rx_packets[i]);
 737                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 738                                    I40E_GLVEBTC_RBCL(i, idx),
 739                                    veb->stat_offsets_loaded,
 740                                    &veb_oes->tc_rx_bytes[i],
 741                                    &veb_es->tc_rx_bytes[i]);
 742                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 743                                    I40E_GLVEBTC_TPCL(i, idx),
 744                                    veb->stat_offsets_loaded,
 745                                    &veb_oes->tc_tx_packets[i],
 746                                    &veb_es->tc_tx_packets[i]);
 747                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 748                                    I40E_GLVEBTC_TBCL(i, idx),
 749                                    veb->stat_offsets_loaded,
 750                                    &veb_oes->tc_tx_bytes[i],
 751                                    &veb_es->tc_tx_bytes[i]);
 752         }
 753         veb->stat_offsets_loaded = true;
 754 }
 755
 756 #ifdef I40E_FCOE
 757 /**
 758  * i40e_update_fcoe_stats - Update FCoE-specific ethernet statistics counters.
 759  * @vsi: the VSI that is capable of doing FCoE
 760  **/
 761 static void i40e_update_fcoe_stats(struct i40e_vsi *vsi)
 762 {
 763         struct i40e_pf *pf = vsi->back;
 764         struct i40e_hw *hw = &pf->hw;
 765         struct i40e_fcoe_stats *ofs;
 766         struct i40e_fcoe_stats *fs;     /* device's eth stats */
 767         int idx;
 768
 769         if (vsi->type != I40E_VSI_FCOE)
 770                 return;
 771
 772         idx = hw->pf_id + I40E_FCOE_PF_STAT_OFFSET;
 773         fs = &vsi->fcoe_stats;
 774         ofs = &vsi->fcoe_stats_offsets;
 775
 776         i40e_stat_update32(hw, I40E_GL_FCOEPRC(idx),
 777                            vsi->fcoe_stat_offsets_loaded,
 778                            &ofs->rx_fcoe_packets, &fs->rx_fcoe_packets);
 779         i40e_stat_update48(hw, I40E_GL_FCOEDWRCH(idx), I40E_GL_FCOEDWRCL(idx),
 780                            vsi->fcoe_stat_offsets_loaded,
 781                            &ofs->rx_fcoe_dwords, &fs->rx_fcoe_dwords);
 782         i40e_stat_update32(hw, I40E_GL_FCOERPDC(idx),
 783                            vsi->fcoe_stat_offsets_loaded,
 784                            &ofs->rx_fcoe_dropped, &fs->rx_fcoe_dropped);
 785         i40e_stat_update32(hw, I40E_GL_FCOEPTC(idx),
 786                            vsi->fcoe_stat_offsets_loaded,
 787                            &ofs->tx_fcoe_packets, &fs->tx_fcoe_packets);
 788         i40e_stat_update48(hw, I40E_GL_FCOEDWTCH(idx), I40E_GL_FCOEDWTCL(idx),
 789                            vsi->fcoe_stat_offsets_loaded,
 790                            &ofs->tx_fcoe_dwords, &fs->tx_fcoe_dwords);
 791         i40e_stat_update32(hw, I40E_GL_FCOECRC(idx),
 792                            vsi->fcoe_stat_offsets_loaded,
 793                            &ofs->fcoe_bad_fccrc, &fs->fcoe_bad_fccrc);
 794         i40e_stat_update32(hw, I40E_GL_FCOELAST(idx),
 795                            vsi->fcoe_stat_offsets_loaded,
 796                            &ofs->fcoe_last_error, &fs->fcoe_last_error);
 797         i40e_stat_update32(hw, I40E_GL_FCOEDDPC(idx),
 798                            vsi->fcoe_stat_offsets_loaded,
 799                            &ofs->fcoe_ddp_count, &fs->fcoe_ddp_count);
 800
 801         vsi->fcoe_stat_offsets_loaded = true;
 802 }
 803
 804 #endif
 805 /**
 806  * i40e_update_vsi_stats - Update the vsi statistics counters.
 807  * @vsi: the VSI to be updated
 808  *
 809  * There are a few instances where we store the same stat in a
 810  * couple of different structs.  This is partly because we have
 811  * the netdev stats that need to be filled out, which is slightly
 812  * different from the "eth_stats" defined by the chip and used in
 813  * VF communications.  We sort it out here.
 814  **/
 815 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 816 {
 817         struct i40e_pf *pf = vsi->back;
 818         struct rtnl_link_stats64 *ons;
 819         struct rtnl_link_stats64 *ns;   /* netdev stats */
 820         struct i40e_eth_stats *oes;
 821         struct i40e_eth_stats *es;     /* device's eth stats */
 822         u32 tx_restart, tx_busy;
 823         u64 tx_lost_interrupt;
 824         struct i40e_ring *p;
 825         u32 rx_page, rx_buf;
 826         u64 bytes, packets;
 827         unsigned int start;
 828         u64 tx_linearize;
 829         u64 tx_force_wb;
 830         u64 rx_p, rx_b;
 831         u64 tx_p, tx_b;
 832         u16 q;
 833
 834         if (test_bit(__I40E_DOWN, &vsi->state) ||
 835             test_bit(__I40E_CONFIG_BUSY, &pf->state))
 836                 return;
 837
 838         ns = i40e_get_vsi_stats_struct(vsi);
 839         ons = &vsi->net_stats_offsets;
 840         es = &vsi->eth_stats;
 841         oes = &vsi->eth_stats_offsets;
 842
 843         /* Gather up the netdev and vsi stats that the driver collects
 844          * on the fly during packet processing
 845          */
 846         rx_b = rx_p = 0;
 847         tx_b = tx_p = 0;
 848         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 849         tx_lost_interrupt = 0;
 850         rx_page = 0;
 851         rx_buf = 0;
 852         rcu_read_lock();
 853         for (q = 0; q < vsi->num_queue_pairs; q++) {
 854                 /* locate Tx ring */
 855                 p = ACCESS_ONCE(vsi->tx_rings[q]);
 856
 857                 do {
 858                         start = u64_stats_fetch_begin_irq(&p->syncp);
 859                         packets = p->stats.packets;
 860                         bytes = p->stats.bytes;
 861                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 862                 tx_b += bytes;
 863                 tx_p += packets;
 864                 tx_restart += p->tx_stats.restart_queue;
 865                 tx_busy += p->tx_stats.tx_busy;
 866                 tx_linearize += p->tx_stats.tx_linearize;
 867                 tx_force_wb += p->tx_stats.tx_force_wb;
 868                 tx_lost_interrupt += p->tx_stats.tx_lost_interrupt;
 869
 870                 /* Rx queue is part of the same block as Tx queue */
 871                 p = &p[1];
 872                 do {
 873                         start = u64_stats_fetch_begin_irq(&p->syncp);
 874                         packets = p->stats.packets;
 875                         bytes = p->stats.bytes;
 876                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 877                 rx_b += bytes;
 878                 rx_p += packets;
 879                 rx_buf += p->rx_stats.alloc_buff_failed;
 880                 rx_page += p->rx_stats.alloc_page_failed;
 881         }
 882         rcu_read_unlock();
 883         vsi->tx_restart = tx_restart;
 884         vsi->tx_busy = tx_busy;
 885         vsi->tx_linearize = tx_linearize;
 886         vsi->tx_force_wb = tx_force_wb;
 887         vsi->tx_lost_interrupt = tx_lost_interrupt;
 888         vsi->rx_page_failed = rx_page;
 889         vsi->rx_buf_failed = rx_buf;
 890
 891         ns->rx_packets = rx_p;
 892         ns->rx_bytes = rx_b;
 893         ns->tx_packets = tx_p;
 894         ns->tx_bytes = tx_b;
 895
 896         /* update netdev stats from eth stats */
 897         i40e_update_eth_stats(vsi);
 898         ons->tx_errors = oes->tx_errors;
 899         ns->tx_errors = es->tx_errors;
 900         ons->multicast = oes->rx_multicast;
 901         ns->multicast = es->rx_multicast;
 902         ons->rx_dropped = oes->rx_discards;
 903         ns->rx_dropped = es->rx_discards;
 904         ons->tx_dropped = oes->tx_discards;
 905         ns->tx_dropped = es->tx_discards;
 906
 907         /* pull in a couple PF stats if this is the main vsi */
 908         if (vsi == pf->vsi[pf->lan_vsi]) {
 909                 ns->rx_crc_errors = pf->stats.crc_errors;
 910                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 911                 ns->rx_length_errors = pf->stats.rx_length_errors;
 912         }
 913 }
 914
 915 /**
 916  * i40e_update_pf_stats - Update the PF statistics counters.
 917  * @pf: the PF to be updated
 918  **/
 919 static void i40e_update_pf_stats(struct i40e_pf *pf)
 920 {
 921         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 922         struct i40e_hw_port_stats *nsd = &pf->stats;
 923         struct i40e_hw *hw = &pf->hw;
 924         u32 val;
 925         int i;
 926
 927         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 928                            I40E_GLPRT_GORCL(hw->port),
 929                            pf->stat_offsets_loaded,
 930                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 931         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 932                            I40E_GLPRT_GOTCL(hw->port),
 933                            pf->stat_offsets_loaded,
 934                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 935         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 936                            pf->stat_offsets_loaded,
 937                            &osd->eth.rx_discards,
 938                            &nsd->eth.rx_discards);
 939         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 940                            I40E_GLPRT_UPRCL(hw->port),
 941                            pf->stat_offsets_loaded,
 942                            &osd->eth.rx_unicast,
 943                            &nsd->eth.rx_unicast);
 944         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 945                            I40E_GLPRT_MPRCL(hw->port),
 946                            pf->stat_offsets_loaded,
 947                            &osd->eth.rx_multicast,
 948                            &nsd->eth.rx_multicast);
 949         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 950                            I40E_GLPRT_BPRCL(hw->port),
 951                            pf->stat_offsets_loaded,
 952                            &osd->eth.rx_broadcast,
 953                            &nsd->eth.rx_broadcast);
 954         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 955                            I40E_GLPRT_UPTCL(hw->port),
 956                            pf->stat_offsets_loaded,
 957                            &osd->eth.tx_unicast,
 958                            &nsd->eth.tx_unicast);
 959         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 960                            I40E_GLPRT_MPTCL(hw->port),
 961                            pf->stat_offsets_loaded,
 962                            &osd->eth.tx_multicast,
 963                            &nsd->eth.tx_multicast);
 964         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 965                            I40E_GLPRT_BPTCL(hw->port),
 966                            pf->stat_offsets_loaded,
 967                            &osd->eth.tx_broadcast,
 968                            &nsd->eth.tx_broadcast);
 969
 970         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 971                            pf->stat_offsets_loaded,
 972                            &osd->tx_dropped_link_down,
 973                            &nsd->tx_dropped_link_down);
 974
 975         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 976                            pf->stat_offsets_loaded,
 977                            &osd->crc_errors, &nsd->crc_errors);
 978
 979         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 980                            pf->stat_offsets_loaded,
 981                            &osd->illegal_bytes, &nsd->illegal_bytes);
 982
 983         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 984                            pf->stat_offsets_loaded,
 985                            &osd->mac_local_faults,
 986                            &nsd->mac_local_faults);
 987         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 988                            pf->stat_offsets_loaded,
 989                            &osd->mac_remote_faults,
 990                            &nsd->mac_remote_faults);
 991
 992         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 993                            pf->stat_offsets_loaded,
 994                            &osd->rx_length_errors,
 995                            &nsd->rx_length_errors);
 996
 997         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 998                            pf->stat_offsets_loaded,
 999                            &osd->link_xon_rx, &nsd->link_xon_rx);
1000         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
1001                            pf->stat_offsets_loaded,
1002                            &osd->link_xon_tx, &nsd->link_xon_tx);
1003         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1004                            pf->stat_offsets_loaded,
1005                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
1006         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1007                            pf->stat_offsets_loaded,
1008                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
1009
1010         for (i = 0; i < 8; i++) {
1011                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1012                                    pf->stat_offsets_loaded,
1013                                    &osd->priority_xoff_rx[i],
1014                                    &nsd->priority_xoff_rx[i]);
1015                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1016                                    pf->stat_offsets_loaded,
1017                                    &osd->priority_xon_rx[i],
1018                                    &nsd->priority_xon_rx[i]);
1019                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1020                                    pf->stat_offsets_loaded,
1021                                    &osd->priority_xon_tx[i],
1022                                    &nsd->priority_xon_tx[i]);
1023                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1024                                    pf->stat_offsets_loaded,
1025                                    &osd->priority_xoff_tx[i],
1026                                    &nsd->priority_xoff_tx[i]);
1027                 i40e_stat_update32(hw,
1028                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1029                                    pf->stat_offsets_loaded,
1030                                    &osd->priority_xon_2_xoff[i],
1031                                    &nsd->priority_xon_2_xoff[i]);
1032         }
1033
1034         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1035                            I40E_GLPRT_PRC64L(hw->port),
1036                            pf->stat_offsets_loaded,
1037                            &osd->rx_size_64, &nsd->rx_size_64);
1038         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1039                            I40E_GLPRT_PRC127L(hw->port),
1040                            pf->stat_offsets_loaded,
1041                            &osd->rx_size_127, &nsd->rx_size_127);
1042         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1043                            I40E_GLPRT_PRC255L(hw->port),
1044                            pf->stat_offsets_loaded,
1045                            &osd->rx_size_255, &nsd->rx_size_255);
1046         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1047                            I40E_GLPRT_PRC511L(hw->port),
1048                            pf->stat_offsets_loaded,
1049                            &osd->rx_size_511, &nsd->rx_size_511);
1050         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1051                            I40E_GLPRT_PRC1023L(hw->port),
1052                            pf->stat_offsets_loaded,
1053                            &osd->rx_size_1023, &nsd->rx_size_1023);
1054         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1055                            I40E_GLPRT_PRC1522L(hw->port),
1056                            pf->stat_offsets_loaded,
1057                            &osd->rx_size_1522, &nsd->rx_size_1522);
1058         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1059                            I40E_GLPRT_PRC9522L(hw->port),
1060                            pf->stat_offsets_loaded,
1061                            &osd->rx_size_big, &nsd->rx_size_big);
1062
1063         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1064                            I40E_GLPRT_PTC64L(hw->port),
1065                            pf->stat_offsets_loaded,
1066                            &osd->tx_size_64, &nsd->tx_size_64);
1067         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1068                            I40E_GLPRT_PTC127L(hw->port),
1069                            pf->stat_offsets_loaded,
1070                            &osd->tx_size_127, &nsd->tx_size_127);
1071         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1072                            I40E_GLPRT_PTC255L(hw->port),
1073                            pf->stat_offsets_loaded,
1074                            &osd->tx_size_255, &nsd->tx_size_255);
1075         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1076                            I40E_GLPRT_PTC511L(hw->port),
1077                            pf->stat_offsets_loaded,
1078                            &osd->tx_size_511, &nsd->tx_size_511);
1079         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1080                            I40E_GLPRT_PTC1023L(hw->port),
1081                            pf->stat_offsets_loaded,
1082                            &osd->tx_size_1023, &nsd->tx_size_1023);
1083         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1084                            I40E_GLPRT_PTC1522L(hw->port),
1085                            pf->stat_offsets_loaded,
1086                            &osd->tx_size_1522, &nsd->tx_size_1522);
1087         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1088                            I40E_GLPRT_PTC9522L(hw->port),
1089                            pf->stat_offsets_loaded,
1090                            &osd->tx_size_big, &nsd->tx_size_big);
1091
1092         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1093                            pf->stat_offsets_loaded,
1094                            &osd->rx_undersize, &nsd->rx_undersize);
1095         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1096                            pf->stat_offsets_loaded,
1097                            &osd->rx_fragments, &nsd->rx_fragments);
1098         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1099                            pf->stat_offsets_loaded,
1100                            &osd->rx_oversize, &nsd->rx_oversize);
1101         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1102                            pf->stat_offsets_loaded,
1103                            &osd->rx_jabber, &nsd->rx_jabber);
1104
1105         /* FDIR stats */
1106         i40e_stat_update32(hw,
1107                            I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(pf->hw.pf_id)),
1108                            pf->stat_offsets_loaded,
1109                            &osd->fd_atr_match, &nsd->fd_atr_match);
1110         i40e_stat_update32(hw,
1111                            I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(pf->hw.pf_id)),
1112                            pf->stat_offsets_loaded,
1113                            &osd->fd_sb_match, &nsd->fd_sb_match);
1114         i40e_stat_update32(hw,
1115                       I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id)),
1116                       pf->stat_offsets_loaded,
1117                       &osd->fd_atr_tunnel_match, &nsd->fd_atr_tunnel_match);
1118
1119         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1120         nsd->tx_lpi_status =
1121                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1122                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1123         nsd->rx_lpi_status =
1124                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1125                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1126         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1127                            pf->stat_offsets_loaded,
1128                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1129         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1130                            pf->stat_offsets_loaded,
1131                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1132
1133         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1134             !(pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED))
1135                 nsd->fd_sb_status = true;
1136         else
1137                 nsd->fd_sb_status = false;
1138
1139         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1140             !(pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
1141                 nsd->fd_atr_status = true;
1142         else
1143                 nsd->fd_atr_status = false;
1144
1145         pf->stat_offsets_loaded = true;
1146 }
1147
1148 /**
1149  * i40e_update_stats - Update the various statistics counters.
1150  * @vsi: the VSI to be updated
1151  *
1152  * Update the various stats for this VSI and its related entities.
1153  **/
1154 void i40e_update_stats(struct i40e_vsi *vsi)
1155 {
1156         struct i40e_pf *pf = vsi->back;
1157
1158         if (vsi == pf->vsi[pf->lan_vsi])
1159                 i40e_update_pf_stats(pf);
1160
1161         i40e_update_vsi_stats(vsi);
1162 #ifdef I40E_FCOE
1163         i40e_update_fcoe_stats(vsi);
1164 #endif
1165 }
1166
1167 /**
1168  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1169  * @vsi: the VSI to be searched
1170  * @macaddr: the MAC address
1171  * @vlan: the vlan
1172  * @is_vf: make sure its a VF filter, else doesn't matter
1173  * @is_netdev: make sure its a netdev filter, else doesn't matter
1174  *
1175  * Returns ptr to the filter object or NULL
1176  **/
1177 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1178                                                 u8 *macaddr, s16 vlan,
1179                                                 bool is_vf, bool is_netdev)
1180 {
1181         struct i40e_mac_filter *f;
1182
1183         if (!vsi || !macaddr)
1184                 return NULL;
1185
1186         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1187                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1188                     (vlan == f->vlan)    &&
1189                     (!is_vf || f->is_vf) &&
1190                     (!is_netdev || f->is_netdev))
1191                         return f;
1192         }
1193         return NULL;
1194 }
1195
1196 /**
1197  * i40e_find_mac - Find a mac addr in the macvlan filters list
1198  * @vsi: the VSI to be searched
1199  * @macaddr: the MAC address we are searching for
1200  * @is_vf: make sure its a VF filter, else doesn't matter
1201  * @is_netdev: make sure its a netdev filter, else doesn't matter
1202  *
1203  * Returns the first filter with the provided MAC address or NULL if
1204  * MAC address was not found
1205  **/
1206 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, u8 *macaddr,
1207                                       bool is_vf, bool is_netdev)
1208 {
1209         struct i40e_mac_filter *f;
1210
1211         if (!vsi || !macaddr)
1212                 return NULL;
1213
1214         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1215                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1216                     (!is_vf || f->is_vf) &&
1217                     (!is_netdev || f->is_netdev))
1218                         return f;
1219         }
1220         return NULL;
1221 }
1222
1223 /**
1224  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1225  * @vsi: the VSI to be searched
1226  *
1227  * Returns true if VSI is in vlan mode or false otherwise
1228  **/
1229 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1230 {
1231         struct i40e_mac_filter *f;
1232
1233         /* Only -1 for all the filters denotes not in vlan mode
1234          * so we have to go through all the list in order to make sure
1235          */
1236         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1237                 if (f->vlan >= 0 || vsi->info.pvid)
1238                         return true;
1239         }
1240
1241         return false;
1242 }
1243
1244 /**
1245  * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans
1246  * @vsi: the VSI to be searched
1247  * @macaddr: the mac address to be filtered
1248  * @is_vf: true if it is a VF
1249  * @is_netdev: true if it is a netdev
1250  *
1251  * Goes through all the macvlan filters and adds a
1252  * macvlan filter for each unique vlan that already exists
1253  *
1254  * Returns first filter found on success, else NULL
1255  **/
1256 struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi, u8 *macaddr,
1257                                              bool is_vf, bool is_netdev)
1258 {
1259         struct i40e_mac_filter *f;
1260
1261         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1262                 if (vsi->info.pvid)
1263                         f->vlan = le16_to_cpu(vsi->info.pvid);
1264                 if (!i40e_find_filter(vsi, macaddr, f->vlan,
1265                                       is_vf, is_netdev)) {
1266                         if (!i40e_add_filter(vsi, macaddr, f->vlan,
1267                                              is_vf, is_netdev))
1268                                 return NULL;
1269                 }
1270         }
1271
1272         return list_first_entry_or_null(&vsi->mac_filter_list,
1273                                         struct i40e_mac_filter, list);
1274 }
1275
1276 /**
1277  * i40e_del_mac_all_vlan - Remove a MAC filter from all VLANS
1278  * @vsi: the VSI to be searched
1279  * @macaddr: the mac address to be removed
1280  * @is_vf: true if it is a VF
1281  * @is_netdev: true if it is a netdev
1282  *
1283  * Removes a given MAC address from a VSI, regardless of VLAN
1284  *
1285  * Returns 0 for success, or error
1286  **/
1287 int i40e_del_mac_all_vlan(struct i40e_vsi *vsi, u8 *macaddr,
1288                           bool is_vf, bool is_netdev)
1289 {
1290         struct i40e_mac_filter *f = NULL;
1291         int changed = 0;
1292
1293         WARN(!spin_is_locked(&vsi->mac_filter_list_lock),
1294              "Missing mac_filter_list_lock\n");
1295         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1296                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1297                     (is_vf == f->is_vf) &&
1298                     (is_netdev == f->is_netdev)) {
1299                         f->counter--;
1300                         f->changed = true;
1301                         changed = 1;
1302                 }
1303         }
1304         if (changed) {
1305                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1306                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1307                 return 0;
1308         }
1309         return -ENOENT;
1310 }
1311
1312 /**
1313  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1314  * @vsi: the PF Main VSI - inappropriate for any other VSI
1315  * @macaddr: the MAC address
1316  *
1317  * Some older firmware configurations set up a default promiscuous VLAN
1318  * filter that needs to be removed.
1319  **/
1320 static int i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1321 {
1322         struct i40e_aqc_remove_macvlan_element_data element;
1323         struct i40e_pf *pf = vsi->back;
1324         i40e_status ret;
1325
1326         /* Only appropriate for the PF main VSI */
1327         if (vsi->type != I40E_VSI_MAIN)
1328                 return -EINVAL;
1329
1330         memset(&element, 0, sizeof(element));
1331         ether_addr_copy(element.mac_addr, macaddr);
1332         element.vlan_tag = 0;
1333         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1334                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1335         ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1336         if (ret)
1337                 return -ENOENT;
1338
1339         return 0;
1340 }
1341
1342 /**
1343  * i40e_add_filter - Add a mac/vlan filter to the VSI
1344  * @vsi: the VSI to be searched
1345  * @macaddr: the MAC address
1346  * @vlan: the vlan
1347  * @is_vf: make sure its a VF filter, else doesn't matter
1348  * @is_netdev: make sure its a netdev filter, else doesn't matter
1349  *
1350  * Returns ptr to the filter object or NULL when no memory available.
1351  *
1352  * NOTE: This function is expected to be called with mac_filter_list_lock
1353  * being held.
1354  **/
1355 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1356                                         u8 *macaddr, s16 vlan,
1357                                         bool is_vf, bool is_netdev)
1358 {
1359         struct i40e_mac_filter *f;
1360
1361         if (!vsi || !macaddr)
1362                 return NULL;
1363
1364         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1365         if (!f) {
1366                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1367                 if (!f)
1368                         goto add_filter_out;
1369
1370                 ether_addr_copy(f->macaddr, macaddr);
1371                 f->vlan = vlan;
1372                 f->changed = true;
1373
1374                 INIT_LIST_HEAD(&f->list);
1375                 list_add_tail(&f->list, &vsi->mac_filter_list);
1376         }
1377
1378         /* increment counter and add a new flag if needed */
1379         if (is_vf) {
1380                 if (!f->is_vf) {
1381                         f->is_vf = true;
1382                         f->counter++;
1383                 }
1384         } else if (is_netdev) {
1385                 if (!f->is_netdev) {
1386                         f->is_netdev = true;
1387                         f->counter++;
1388                 }
1389         } else {
1390                 f->counter++;
1391         }
1392
1393         /* changed tells sync_filters_subtask to
1394          * push the filter down to the firmware
1395          */
1396         if (f->changed) {
1397                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1398                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1399         }
1400
1401 add_filter_out:
1402         return f;
1403 }
1404
1405 /**
1406  * i40e_del_filter - Remove a mac/vlan filter from the VSI
1407  * @vsi: the VSI to be searched
1408  * @macaddr: the MAC address
1409  * @vlan: the vlan
1410  * @is_vf: make sure it's a VF filter, else doesn't matter
1411  * @is_netdev: make sure it's a netdev filter, else doesn't matter
1412  *
1413  * NOTE: This function is expected to be called with mac_filter_list_lock
1414  * being held.
1415  **/
1416 void i40e_del_filter(struct i40e_vsi *vsi,
1417                      u8 *macaddr, s16 vlan,
1418                      bool is_vf, bool is_netdev)
1419 {
1420         struct i40e_mac_filter *f;
1421
1422         if (!vsi || !macaddr)
1423                 return;
1424
1425         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1426         if (!f || f->counter == 0)
1427                 return;
1428
1429         if (is_vf) {
1430                 if (f->is_vf) {
1431                         f->is_vf = false;
1432                         f->counter--;
1433                 }
1434         } else if (is_netdev) {
1435                 if (f->is_netdev) {
1436                         f->is_netdev = false;
1437                         f->counter--;
1438                 }
1439         } else {
1440                 /* make sure we don't remove a filter in use by VF or netdev */
1441                 int min_f = 0;
1442
1443                 min_f += (f->is_vf ? 1 : 0);
1444                 min_f += (f->is_netdev ? 1 : 0);
1445
1446                 if (f->counter > min_f)
1447                         f->counter--;
1448         }
1449
1450         /* counter == 0 tells sync_filters_subtask to
1451          * remove the filter from the firmware's list
1452          */
1453         if (f->counter == 0) {
1454                 f->changed = true;
1455                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1456                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1457         }
1458 }
1459
1460 /**
1461  * i40e_set_mac - NDO callback to set mac address
1462  * @netdev: network interface device structure
1463  * @p: pointer to an address structure
1464  *
1465  * Returns 0 on success, negative on failure
1466  **/
1467 #ifdef I40E_FCOE
1468 int i40e_set_mac(struct net_device *netdev, void *p)
1469 #else
1470 static int i40e_set_mac(struct net_device *netdev, void *p)
1471 #endif
1472 {
1473         struct i40e_netdev_priv *np = netdev_priv(netdev);
1474         struct i40e_vsi *vsi = np->vsi;
1475         struct i40e_pf *pf = vsi->back;
1476         struct i40e_hw *hw = &pf->hw;
1477         struct sockaddr *addr = p;
1478         struct i40e_mac_filter *f;
1479
1480         if (!is_valid_ether_addr(addr->sa_data))
1481                 return -EADDRNOTAVAIL;
1482
1483         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1484                 netdev_info(netdev, "already using mac address %pM\n",
1485                             addr->sa_data);
1486                 return 0;
1487         }
1488
1489         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1490             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1491                 return -EADDRNOTAVAIL;
1492
1493         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1494                 netdev_info(netdev, "returning to hw mac address %pM\n",
1495                             hw->mac.addr);
1496         else
1497                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1498
1499         if (vsi->type == I40E_VSI_MAIN) {
1500                 i40e_status ret;
1501
1502                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1503                                                 I40E_AQC_WRITE_TYPE_LAA_WOL,
1504                                                 addr->sa_data, NULL);
1505                 if (ret) {
1506                         netdev_info(netdev,
1507                                     "Addr change for Main VSI failed: %d\n",
1508                                     ret);
1509                         return -EADDRNOTAVAIL;
1510                 }
1511         }
1512
1513         if (ether_addr_equal(netdev->dev_addr, hw->mac.addr)) {
1514                 struct i40e_aqc_remove_macvlan_element_data element;
1515
1516                 memset(&element, 0, sizeof(element));
1517                 ether_addr_copy(element.mac_addr, netdev->dev_addr);
1518                 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1519                 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1520         } else {
1521                 spin_lock_bh(&vsi->mac_filter_list_lock);
1522                 i40e_del_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
1523                                 false, false);
1524                 spin_unlock_bh(&vsi->mac_filter_list_lock);
1525         }
1526
1527         if (ether_addr_equal(addr->sa_data, hw->mac.addr)) {
1528                 struct i40e_aqc_add_macvlan_element_data element;
1529
1530                 memset(&element, 0, sizeof(element));
1531                 ether_addr_copy(element.mac_addr, hw->mac.addr);
1532                 element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH);
1533                 i40e_aq_add_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1534         } else {
1535                 spin_lock_bh(&vsi->mac_filter_list_lock);
1536                 f = i40e_add_filter(vsi, addr->sa_data, I40E_VLAN_ANY,
1537                                     false, false);
1538                 if (f)
1539                         f->is_laa = true;
1540                 spin_unlock_bh(&vsi->mac_filter_list_lock);
1541         }
1542
1543         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1544
1545         /* schedule our worker thread which will take care of
1546          * applying the new filter changes
1547          */
1548         i40e_service_event_schedule(vsi->back);
1549         return 0;
1550 }
1551
1552 /**
1553  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1554  * @vsi: the VSI being setup
1555  * @ctxt: VSI context structure
1556  * @enabled_tc: Enabled TCs bitmap
1557  * @is_add: True if called before Add VSI
1558  *
1559  * Setup VSI queue mapping for enabled traffic classes.
1560  **/
1561 #ifdef I40E_FCOE
1562 void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1563                               struct i40e_vsi_context *ctxt,
1564                               u8 enabled_tc,
1565                               bool is_add)
1566 #else
1567 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1568                                      struct i40e_vsi_context *ctxt,
1569                                      u8 enabled_tc,
1570                                      bool is_add)
1571 #endif
1572 {
1573         struct i40e_pf *pf = vsi->back;
1574         u16 sections = 0;
1575         u8 netdev_tc = 0;
1576         u16 numtc = 0;
1577         u16 qcount;
1578         u8 offset;
1579         u16 qmap;
1580         int i;
1581         u16 num_tc_qps = 0;
1582
1583         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1584         offset = 0;
1585
1586         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1587                 /* Find numtc from enabled TC bitmap */
1588                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1589                         if (enabled_tc & BIT(i)) /* TC is enabled */
1590                                 numtc++;
1591                 }
1592                 if (!numtc) {
1593                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1594                         numtc = 1;
1595                 }
1596         } else {
1597                 /* At least TC0 is enabled in case of non-DCB case */
1598                 numtc = 1;
1599         }
1600
1601         vsi->tc_config.numtc = numtc;
1602         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1603         /* Number of queues per enabled TC */
1604         /* In MFP case we can have a much lower count of MSIx
1605          * vectors available and so we need to lower the used
1606          * q count.
1607          */
1608         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1609                 qcount = min_t(int, vsi->alloc_queue_pairs, pf->num_lan_msix);
1610         else
1611                 qcount = vsi->alloc_queue_pairs;
1612         num_tc_qps = qcount / numtc;
1613         num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf));
1614
1615         /* Setup queue offset/count for all TCs for given VSI */
1616         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1617                 /* See if the given TC is enabled for the given VSI */
1618                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1619                         /* TC is enabled */
1620                         int pow, num_qps;
1621
1622                         switch (vsi->type) {
1623                         case I40E_VSI_MAIN:
1624                                 qcount = min_t(int, pf->alloc_rss_size,
1625                                                num_tc_qps);
1626                                 break;
1627 #ifdef I40E_FCOE
1628                         case I40E_VSI_FCOE:
1629                                 qcount = num_tc_qps;
1630                                 break;
1631 #endif
1632                         case I40E_VSI_FDIR:
1633                         case I40E_VSI_SRIOV:
1634                         case I40E_VSI_VMDQ2:
1635                         default:
1636                                 qcount = num_tc_qps;
1637                                 WARN_ON(i != 0);
1638                                 break;
1639                         }
1640                         vsi->tc_config.tc_info[i].qoffset = offset;
1641                         vsi->tc_config.tc_info[i].qcount = qcount;
1642
1643                         /* find the next higher power-of-2 of num queue pairs */
1644                         num_qps = qcount;
1645                         pow = 0;
1646                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1647                                 pow++;
1648                                 num_qps >>= 1;
1649                         }
1650
1651                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1652                         qmap =
1653                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1654                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1655
1656                         offset += qcount;
1657                 } else {
1658                         /* TC is not enabled so set the offset to
1659                          * default queue and allocate one queue
1660                          * for the given TC.
1661                          */
1662                         vsi->tc_config.tc_info[i].qoffset = 0;
1663                         vsi->tc_config.tc_info[i].qcount = 1;
1664                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1665
1666                         qmap = 0;
1667                 }
1668                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1669         }
1670
1671         /* Set actual Tx/Rx queue pairs */
1672         vsi->num_queue_pairs = offset;
1673         if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1674                 if (vsi->req_queue_pairs > 0)
1675                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1676                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1677                         vsi->num_queue_pairs = pf->num_lan_msix;
1678         }
1679
1680         /* Scheduler section valid can only be set for ADD VSI */
1681         if (is_add) {
1682                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1683
1684                 ctxt->info.up_enable_bits = enabled_tc;
1685         }
1686         if (vsi->type == I40E_VSI_SRIOV) {
1687                 ctxt->info.mapping_flags |=
1688                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1689                 for (i = 0; i < vsi->num_queue_pairs; i++)
1690                         ctxt->info.queue_mapping[i] =
1691                                                cpu_to_le16(vsi->base_queue + i);
1692         } else {
1693                 ctxt->info.mapping_flags |=
1694                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1695                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1696         }
1697         ctxt->info.valid_sections |= cpu_to_le16(sections);
1698 }
1699
1700 /**
1701  * i40e_set_rx_mode - NDO callback to set the netdev filters
1702  * @netdev: network interface device structure
1703  **/
1704 #ifdef I40E_FCOE
1705 void i40e_set_rx_mode(struct net_device *netdev)
1706 #else
1707 static void i40e_set_rx_mode(struct net_device *netdev)
1708 #endif
1709 {
1710         struct i40e_netdev_priv *np = netdev_priv(netdev);
1711         struct i40e_mac_filter *f, *ftmp;
1712         struct i40e_vsi *vsi = np->vsi;
1713         struct netdev_hw_addr *uca;
1714         struct netdev_hw_addr *mca;
1715         struct netdev_hw_addr *ha;
1716
1717         spin_lock_bh(&vsi->mac_filter_list_lock);
1718
1719         /* add addr if not already in the filter list */
1720         netdev_for_each_uc_addr(uca, netdev) {
1721                 if (!i40e_find_mac(vsi, uca->addr, false, true)) {
1722                         if (i40e_is_vsi_in_vlan(vsi))
1723                                 i40e_put_mac_in_vlan(vsi, uca->addr,
1724                                                      false, true);
1725                         else
1726                                 i40e_add_filter(vsi, uca->addr, I40E_VLAN_ANY,
1727                                                 false, true);
1728                 }
1729         }
1730
1731         netdev_for_each_mc_addr(mca, netdev) {
1732                 if (!i40e_find_mac(vsi, mca->addr, false, true)) {
1733                         if (i40e_is_vsi_in_vlan(vsi))
1734                                 i40e_put_mac_in_vlan(vsi, mca->addr,
1735                                                      false, true);
1736                         else
1737                                 i40e_add_filter(vsi, mca->addr, I40E_VLAN_ANY,
1738                                                 false, true);
1739                 }
1740         }
1741
1742         /* remove filter if not in netdev list */
1743         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1744
1745                 if (!f->is_netdev)
1746                         continue;
1747
1748                 netdev_for_each_mc_addr(mca, netdev)
1749                         if (ether_addr_equal(mca->addr, f->macaddr))
1750                                 goto bottom_of_search_loop;
1751
1752                 netdev_for_each_uc_addr(uca, netdev)
1753                         if (ether_addr_equal(uca->addr, f->macaddr))
1754                                 goto bottom_of_search_loop;
1755
1756                 for_each_dev_addr(netdev, ha)
1757                         if (ether_addr_equal(ha->addr, f->macaddr))
1758                                 goto bottom_of_search_loop;
1759
1760                 /* f->macaddr wasn't found in uc, mc, or ha list so delete it */
1761                 i40e_del_filter(vsi, f->macaddr, I40E_VLAN_ANY, false, true);
1762
1763 bottom_of_search_loop:
1764                 continue;
1765         }
1766         spin_unlock_bh(&vsi->mac_filter_list_lock);
1767
1768         /* check for other flag changes */
1769         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1770                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1771                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1772         }
1773
1774         /* schedule our worker thread which will take care of
1775          * applying the new filter changes
1776          */
1777         i40e_service_event_schedule(vsi->back);
1778 }
1779
1780 /**
1781  * i40e_mac_filter_entry_clone - Clones a MAC filter entry
1782  * @src: source MAC filter entry to be clones
1783  *
1784  * Returns the pointer to newly cloned MAC filter entry or NULL
1785  * in case of error
1786  **/
1787 static struct i40e_mac_filter *i40e_mac_filter_entry_clone(
1788                                         struct i40e_mac_filter *src)
1789 {
1790         struct i40e_mac_filter *f;
1791
1792         f = kzalloc(sizeof(*f), GFP_ATOMIC);
1793         if (!f)
1794                 return NULL;
1795         *f = *src;
1796
1797         INIT_LIST_HEAD(&f->list);
1798
1799         return f;
1800 }
1801
1802 /**
1803  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
1804  * @vsi: pointer to vsi struct
1805  * @from: Pointer to list which contains MAC filter entries - changes to
1806  *        those entries needs to be undone.
1807  *
1808  * MAC filter entries from list were slated to be removed from device.
1809  **/
1810 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
1811                                          struct list_head *from)
1812 {
1813         struct i40e_mac_filter *f, *ftmp;
1814
1815         list_for_each_entry_safe(f, ftmp, from, list) {
1816                 f->changed = true;
1817                 /* Move the element back into MAC filter list*/
1818                 list_move_tail(&f->list, &vsi->mac_filter_list);
1819         }
1820 }
1821
1822 /**
1823  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
1824  * @vsi: pointer to vsi struct
1825  *
1826  * MAC filter entries from list were slated to be added from device.
1827  **/
1828 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi)
1829 {
1830         struct i40e_mac_filter *f, *ftmp;
1831
1832         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1833                 if (!f->changed && f->counter)
1834                         f->changed = true;
1835         }
1836 }
1837
1838 /**
1839  * i40e_cleanup_add_list - Deletes the element from add list and release
1840  *                      memory
1841  * @add_list: Pointer to list which contains MAC filter entries
1842  **/
1843 static void i40e_cleanup_add_list(struct list_head *add_list)
1844 {
1845         struct i40e_mac_filter *f, *ftmp;
1846
1847         list_for_each_entry_safe(f, ftmp, add_list, list) {
1848                 list_del(&f->list);
1849                 kfree(f);
1850         }
1851 }
1852
1853 /**
1854  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
1855  * @vsi: ptr to the VSI
1856  *
1857  * Push any outstanding VSI filter changes through the AdminQ.
1858  *
1859  * Returns 0 or error value
1860  **/
1861 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
1862 {
1863         struct list_head tmp_del_list, tmp_add_list;
1864         struct i40e_mac_filter *f, *ftmp, *fclone;
1865         bool promisc_forced_on = false;
1866         bool add_happened = false;
1867         int filter_list_len = 0;
1868         u32 changed_flags = 0;
1869         i40e_status aq_ret = 0;
1870         bool err_cond = false;
1871         int retval = 0;
1872         struct i40e_pf *pf;
1873         int num_add = 0;
1874         int num_del = 0;
1875         int aq_err = 0;
1876         u16 cmd_flags;
1877
1878         /* empty array typed pointers, kcalloc later */
1879         struct i40e_aqc_add_macvlan_element_data *add_list;
1880         struct i40e_aqc_remove_macvlan_element_data *del_list;
1881
1882         while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state))
1883                 usleep_range(1000, 2000);
1884         pf = vsi->back;
1885
1886         if (vsi->netdev) {
1887                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
1888                 vsi->current_netdev_flags = vsi->netdev->flags;
1889         }
1890
1891         INIT_LIST_HEAD(&tmp_del_list);
1892         INIT_LIST_HEAD(&tmp_add_list);
1893
1894         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
1895                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
1896
1897                 spin_lock_bh(&vsi->mac_filter_list_lock);
1898                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1899                         if (!f->changed)
1900                                 continue;
1901
1902                         if (f->counter != 0)
1903                                 continue;
1904                         f->changed = false;
1905
1906                         /* Move the element into temporary del_list */
1907                         list_move_tail(&f->list, &tmp_del_list);
1908                 }
1909
1910                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1911                         if (!f->changed)
1912                                 continue;
1913
1914                         if (f->counter == 0)
1915                                 continue;
1916                         f->changed = false;
1917
1918                         /* Clone MAC filter entry and add into temporary list */
1919                         fclone = i40e_mac_filter_entry_clone(f);
1920                         if (!fclone) {
1921                                 err_cond = true;
1922                                 break;
1923                         }
1924                         list_add_tail(&fclone->list, &tmp_add_list);
1925                 }
1926
1927                 /* if failed to clone MAC filter entry - undo */
1928                 if (err_cond) {
1929                         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
1930                         i40e_undo_add_filter_entries(vsi);
1931                 }
1932                 spin_unlock_bh(&vsi->mac_filter_list_lock);
1933
1934                 if (err_cond) {
1935                         i40e_cleanup_add_list(&tmp_add_list);
1936                         retval = -ENOMEM;
1937                         goto out;
1938                 }
1939         }
1940
1941         /* Now process 'del_list' outside the lock */
1942         if (!list_empty(&tmp_del_list)) {
1943                 int del_list_size;
1944
1945                 filter_list_len = pf->hw.aq.asq_buf_size /
1946                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
1947                 del_list_size = filter_list_len *
1948                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
1949                 del_list = kzalloc(del_list_size, GFP_ATOMIC);
1950                 if (!del_list) {
1951                         i40e_cleanup_add_list(&tmp_add_list);
1952
1953                         /* Undo VSI's MAC filter entry element updates */
1954                         spin_lock_bh(&vsi->mac_filter_list_lock);
1955                         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
1956                         i40e_undo_add_filter_entries(vsi);
1957                         spin_unlock_bh(&vsi->mac_filter_list_lock);
1958                         retval = -ENOMEM;
1959                         goto out;
1960                 }
1961
1962                 list_for_each_entry_safe(f, ftmp, &tmp_del_list, list) {
1963                         cmd_flags = 0;
1964
1965                         /* add to delete list */
1966                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
1967                         del_list[num_del].vlan_tag =
1968                                 cpu_to_le16((u16)(f->vlan ==
1969                                             I40E_VLAN_ANY ? 0 : f->vlan));
1970
1971                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1972                         del_list[num_del].flags = cmd_flags;
1973                         num_del++;
1974
1975                         /* flush a full buffer */
1976                         if (num_del == filter_list_len) {
1977                                 aq_ret = i40e_aq_remove_macvlan(&pf->hw,
1978                                                                 vsi->seid,
1979                                                                 del_list,
1980                                                                 num_del,
1981                                                                 NULL);
1982                                 aq_err = pf->hw.aq.asq_last_status;
1983                                 num_del = 0;
1984                                 memset(del_list, 0, del_list_size);
1985
1986                                 if (aq_ret && aq_err != I40E_AQ_RC_ENOENT) {
1987                                         retval = -EIO;
1988                                         dev_err(&pf->pdev->dev,
1989                                                 "ignoring delete macvlan error, err %s, aq_err %s while flushing a full buffer\n",
1990                                                 i40e_stat_str(&pf->hw, aq_ret),
1991                                                 i40e_aq_str(&pf->hw, aq_err));
1992                                 }
1993                         }
1994                         /* Release memory for MAC filter entries which were
1995                          * synced up with HW.
1996                          */
1997                         list_del(&f->list);
1998                         kfree(f);
1999                 }
2000
2001                 if (num_del) {
2002                         aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid,
2003                                                         del_list, num_del,
2004                                                         NULL);
2005                         aq_err = pf->hw.aq.asq_last_status;
2006                         num_del = 0;
2007
2008                         if (aq_ret && aq_err != I40E_AQ_RC_ENOENT)
2009                                 dev_info(&pf->pdev->dev,
2010                                          "ignoring delete macvlan error, err %s aq_err %s\n",
2011                                          i40e_stat_str(&pf->hw, aq_ret),
2012                                          i40e_aq_str(&pf->hw, aq_err));
2013                 }
2014
2015                 kfree(del_list);
2016                 del_list = NULL;
2017         }
2018
2019         if (!list_empty(&tmp_add_list)) {
2020                 int add_list_size;
2021
2022                 /* do all the adds now */
2023                 filter_list_len = pf->hw.aq.asq_buf_size /
2024                                sizeof(struct i40e_aqc_add_macvlan_element_data),
2025                 add_list_size = filter_list_len *
2026                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2027                 add_list = kzalloc(add_list_size, GFP_ATOMIC);
2028                 if (!add_list) {
2029                         /* Purge element from temporary lists */
2030                         i40e_cleanup_add_list(&tmp_add_list);
2031
2032                         /* Undo add filter entries from VSI MAC filter list */
2033                         spin_lock_bh(&vsi->mac_filter_list_lock);
2034                         i40e_undo_add_filter_entries(vsi);
2035                         spin_unlock_bh(&vsi->mac_filter_list_lock);
2036                         retval = -ENOMEM;
2037                         goto out;
2038                 }
2039
2040                 list_for_each_entry_safe(f, ftmp, &tmp_add_list, list) {
2041
2042                         add_happened = true;
2043                         cmd_flags = 0;
2044
2045                         /* add to add array */
2046                         ether_addr_copy(add_list[num_add].mac_addr, f->macaddr);
2047                         add_list[num_add].vlan_tag =
2048                                 cpu_to_le16(
2049                                  (u16)(f->vlan == I40E_VLAN_ANY ? 0 : f->vlan));
2050                         add_list[num_add].queue_number = 0;
2051
2052                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2053                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2054                         num_add++;
2055
2056                         /* flush a full buffer */
2057                         if (num_add == filter_list_len) {
2058                                 aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
2059                                                              add_list, num_add,
2060                                                              NULL);
2061                                 aq_err = pf->hw.aq.asq_last_status;
2062                                 num_add = 0;
2063
2064                                 if (aq_ret)
2065                                         break;
2066                                 memset(add_list, 0, add_list_size);
2067                         }
2068                         /* Entries from tmp_add_list were cloned from MAC
2069                          * filter list, hence clean those cloned entries
2070                          */
2071                         list_del(&f->list);
2072                         kfree(f);
2073                 }
2074
2075                 if (num_add) {
2076                         aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
2077                                                      add_list, num_add, NULL);
2078                         aq_err = pf->hw.aq.asq_last_status;
2079                         num_add = 0;
2080                 }
2081                 kfree(add_list);
2082                 add_list = NULL;
2083
2084                 if (add_happened && aq_ret && aq_err != I40E_AQ_RC_EINVAL) {
2085                         retval = i40e_aq_rc_to_posix(aq_ret, aq_err);
2086                         dev_info(&pf->pdev->dev,
2087                                  "add filter failed, err %s aq_err %s\n",
2088                                  i40e_stat_str(&pf->hw, aq_ret),
2089                                  i40e_aq_str(&pf->hw, aq_err));
2090                         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOSPC) &&
2091                             !test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
2092                                       &vsi->state)) {
2093                                 promisc_forced_on = true;
2094                                 set_bit(__I40E_FILTER_OVERFLOW_PROMISC,
2095                                         &vsi->state);
2096                                 dev_info(&pf->pdev->dev, "promiscuous mode forced on\n");
2097                         }
2098                 }
2099         }
2100
2101         /* check for changes in promiscuous modes */
2102         if (changed_flags & IFF_ALLMULTI) {
2103                 bool cur_multipromisc;
2104
2105                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2106                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2107                                                                vsi->seid,
2108                                                                cur_multipromisc,
2109                                                                NULL);
2110                 if (aq_ret) {
2111                         retval = i40e_aq_rc_to_posix(aq_ret,
2112                                                      pf->hw.aq.asq_last_status);
2113                         dev_info(&pf->pdev->dev,
2114                                  "set multi promisc failed, err %s aq_err %s\n",
2115                                  i40e_stat_str(&pf->hw, aq_ret),
2116                                  i40e_aq_str(&pf->hw,
2117                                              pf->hw.aq.asq_last_status));
2118                 }
2119         }
2120         if ((changed_flags & IFF_PROMISC) || promisc_forced_on) {
2121                 bool cur_promisc;
2122
2123                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2124                                test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
2125                                         &vsi->state));
2126                 if ((vsi->type == I40E_VSI_MAIN) &&
2127                     (pf->lan_veb != I40E_NO_VEB) &&
2128                     !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2129                         /* set defport ON for Main VSI instead of true promisc
2130                          * this way we will get all unicast/multicast and VLAN
2131                          * promisc behavior but will not get VF or VMDq traffic
2132                          * replicated on the Main VSI.
2133                          */
2134                         if (pf->cur_promisc != cur_promisc) {
2135                                 pf->cur_promisc = cur_promisc;
2136                                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
2137                         }
2138                 } else {
2139                         aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2140                                                           &vsi->back->hw,
2141                                                           vsi->seid,
2142                                                           cur_promisc, NULL);
2143                         if (aq_ret) {
2144                                 retval =
2145                                 i40e_aq_rc_to_posix(aq_ret,
2146                                                     pf->hw.aq.asq_last_status);
2147                                 dev_info(&pf->pdev->dev,
2148                                          "set unicast promisc failed, err %d, aq_err %d\n",
2149                                          aq_ret, pf->hw.aq.asq_last_status);
2150                         }
2151                         aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2152                                                           &vsi->back->hw,
2153                                                           vsi->seid,
2154                                                           cur_promisc, NULL);
2155                         if (aq_ret) {
2156                                 retval =
2157                                 i40e_aq_rc_to_posix(aq_ret,
2158                                                     pf->hw.aq.asq_last_status);
2159                                 dev_info(&pf->pdev->dev,
2160                                          "set multicast promisc failed, err %d, aq_err %d\n",
2161                                          aq_ret, pf->hw.aq.asq_last_status);
2162                         }
2163                 }
2164                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
2165                                                    vsi->seid,
2166                                                    cur_promisc, NULL);
2167                 if (aq_ret) {
2168                         retval = i40e_aq_rc_to_posix(aq_ret,
2169                                                      pf->hw.aq.asq_last_status);
2170                         dev_info(&pf->pdev->dev,
2171                                  "set brdcast promisc failed, err %s, aq_err %s\n",
2172                                  i40e_stat_str(&pf->hw, aq_ret),
2173                                  i40e_aq_str(&pf->hw,
2174                                              pf->hw.aq.asq_last_status));
2175                 }
2176         }
2177 out:
2178         /* if something went wrong then set the changed flag so we try again */
2179         if (retval)
2180                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2181
2182         clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
2183         return retval;
2184 }
2185
2186 /**
2187  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2188  * @pf: board private structure
2189  **/
2190 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2191 {
2192         int v;
2193
2194         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
2195                 return;
2196         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
2197
2198         for (v = 0; v < pf->num_alloc_vsi; v++) {
2199                 if (pf->vsi[v] &&
2200                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2201                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2202
2203                         if (ret) {
2204                                 /* come back and try again later */
2205                                 pf->flags |= I40E_FLAG_FILTER_SYNC;
2206                                 break;
2207                         }
2208                 }
2209         }
2210 }
2211
2212 /**
2213  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2214  * @netdev: network interface device structure
2215  * @new_mtu: new value for maximum frame size
2216  *
2217  * Returns 0 on success, negative on failure
2218  **/
2219 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2220 {
2221         struct i40e_netdev_priv *np = netdev_priv(netdev);
2222         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2223         struct i40e_vsi *vsi = np->vsi;
2224
2225         /* MTU < 68 is an error and causes problems on some kernels */
2226         if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER))
2227                 return -EINVAL;
2228
2229         netdev_info(netdev, "changing MTU from %d to %d\n",
2230                     netdev->mtu, new_mtu);
2231         netdev->mtu = new_mtu;
2232         if (netif_running(netdev))
2233                 i40e_vsi_reinit_locked(vsi);
2234         i40e_notify_client_of_l2_param_changes(vsi);
2235         return 0;
2236 }
2237
2238 /**
2239  * i40e_ioctl - Access the hwtstamp interface
2240  * @netdev: network interface device structure
2241  * @ifr: interface request data
2242  * @cmd: ioctl command
2243  **/
2244 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2245 {
2246         struct i40e_netdev_priv *np = netdev_priv(netdev);
2247         struct i40e_pf *pf = np->vsi->back;
2248
2249         switch (cmd) {
2250         case SIOCGHWTSTAMP:
2251                 return i40e_ptp_get_ts_config(pf, ifr);
2252         case SIOCSHWTSTAMP:
2253                 return i40e_ptp_set_ts_config(pf, ifr);
2254         default:
2255                 return -EOPNOTSUPP;
2256         }
2257 }
2258
2259 /**
2260  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2261  * @vsi: the vsi being adjusted
2262  **/
2263 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2264 {
2265         struct i40e_vsi_context ctxt;
2266         i40e_status ret;
2267
2268         if ((vsi->info.valid_sections &
2269              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2270             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2271                 return;  /* already enabled */
2272
2273         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2274         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2275                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2276
2277         ctxt.seid = vsi->seid;
2278         ctxt.info = vsi->info;
2279         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2280         if (ret) {
2281                 dev_info(&vsi->back->pdev->dev,
2282                          "update vlan stripping failed, err %s aq_err %s\n",
2283                          i40e_stat_str(&vsi->back->hw, ret),
2284                          i40e_aq_str(&vsi->back->hw,
2285                                      vsi->back->hw.aq.asq_last_status));
2286         }
2287 }
2288
2289 /**
2290  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2291  * @vsi: the vsi being adjusted
2292  **/
2293 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2294 {
2295         struct i40e_vsi_context ctxt;
2296         i40e_status ret;
2297
2298         if ((vsi->info.valid_sections &
2299              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2300             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2301              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2302                 return;  /* already disabled */
2303
2304         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2305         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2306                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2307
2308         ctxt.seid = vsi->seid;
2309         ctxt.info = vsi->info;
2310         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2311         if (ret) {
2312                 dev_info(&vsi->back->pdev->dev,
2313                          "update vlan stripping failed, err %s aq_err %s\n",
2314                          i40e_stat_str(&vsi->back->hw, ret),
2315                          i40e_aq_str(&vsi->back->hw,
2316                                      vsi->back->hw.aq.asq_last_status));
2317         }
2318 }
2319
2320 /**
2321  * i40e_vlan_rx_register - Setup or shutdown vlan offload
2322  * @netdev: network interface to be adjusted
2323  * @features: netdev features to test if VLAN offload is enabled or not
2324  **/
2325 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
2326 {
2327         struct i40e_netdev_priv *np = netdev_priv(netdev);
2328         struct i40e_vsi *vsi = np->vsi;
2329
2330         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2331                 i40e_vlan_stripping_enable(vsi);
2332         else
2333                 i40e_vlan_stripping_disable(vsi);
2334 }
2335
2336 /**
2337  * i40e_vsi_add_vlan - Add vsi membership for given vlan
2338  * @vsi: the vsi being configured
2339  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2340  **/
2341 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid)
2342 {
2343         struct i40e_mac_filter *f, *add_f;
2344         bool is_netdev, is_vf;
2345
2346         is_vf = (vsi->type == I40E_VSI_SRIOV);
2347         is_netdev = !!(vsi->netdev);
2348
2349         /* Locked once because all functions invoked below iterates list*/
2350         spin_lock_bh(&vsi->mac_filter_list_lock);
2351
2352         if (is_netdev) {
2353                 add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, vid,
2354                                         is_vf, is_netdev);
2355                 if (!add_f) {
2356                         dev_info(&vsi->back->pdev->dev,
2357                                  "Could not add vlan filter %d for %pM\n",
2358                                  vid, vsi->netdev->dev_addr);
2359                         spin_unlock_bh(&vsi->mac_filter_list_lock);
2360                         return -ENOMEM;
2361                 }
2362         }
2363
2364         list_for_each_entry(f, &vsi->mac_filter_list, list) {
2365                 add_f = i40e_add_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
2366                 if (!add_f) {
2367                         dev_info(&vsi->back->pdev->dev,
2368                                  "Could not add vlan filter %d for %pM\n",
2369                                  vid, f->macaddr);
2370                         spin_unlock_bh(&vsi->mac_filter_list_lock);
2371                         return -ENOMEM;
2372                 }
2373         }
2374
2375         /* Now if we add a vlan tag, make sure to check if it is the first
2376          * tag (i.e. a "tag" -1 does exist) and if so replace the -1 "tag"
2377          * with 0, so we now accept untagged and specified tagged traffic
2378          * (and not any taged and untagged)
2379          */
2380         if (vid > 0) {
2381                 if (is_netdev && i40e_find_filter(vsi, vsi->netdev->dev_addr,
2382                                                   I40E_VLAN_ANY,
2383                                                   is_vf, is_netdev)) {
2384                         i40e_del_filter(vsi, vsi->netdev->dev_addr,
2385                                         I40E_VLAN_ANY, is_vf, is_netdev);
2386                         add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, 0,
2387                                                 is_vf, is_netdev);
2388                         if (!add_f) {
2389                                 dev_info(&vsi->back->pdev->dev,
2390                                          "Could not add filter 0 for %pM\n",
2391                                          vsi->netdev->dev_addr);
2392                                 spin_unlock_bh(&vsi->mac_filter_list_lock);
2393                                 return -ENOMEM;
2394                         }
2395                 }
2396         }
2397
2398         /* Do not assume that I40E_VLAN_ANY should be reset to VLAN 0 */
2399         if (vid > 0 && !vsi->info.pvid) {
2400                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
2401                         if (!i40e_find_filter(vsi, f->macaddr, I40E_VLAN_ANY,
2402                                               is_vf, is_netdev))
2403                                 continue;
2404                         i40e_del_filter(vsi, f->macaddr, I40E_VLAN_ANY,
2405                                         is_vf, is_netdev);
2406                         add_f = i40e_add_filter(vsi, f->macaddr,
2407                                                 0, is_vf, is_netdev);
2408                         if (!add_f) {
2409                                 dev_info(&vsi->back->pdev->dev,
2410                                          "Could not add filter 0 for %pM\n",
2411                                         f->macaddr);
2412                                 spin_unlock_bh(&vsi->mac_filter_list_lock);
2413                                 return -ENOMEM;
2414                         }
2415                 }
2416         }
2417
2418         spin_unlock_bh(&vsi->mac_filter_list_lock);
2419
2420         /* schedule our worker thread which will take care of
2421          * applying the new filter changes
2422          */
2423         i40e_service_event_schedule(vsi->back);
2424         return 0;
2425 }
2426
2427 /**
2428  * i40e_vsi_kill_vlan - Remove vsi membership for given vlan
2429  * @vsi: the vsi being configured
2430  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2431  *
2432  * Return: 0 on success or negative otherwise
2433  **/
2434 int i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid)
2435 {
2436         struct net_device *netdev = vsi->netdev;
2437         struct i40e_mac_filter *f, *add_f;
2438         bool is_vf, is_netdev;
2439         int filter_count = 0;
2440
2441         is_vf = (vsi->type == I40E_VSI_SRIOV);
2442         is_netdev = !!(netdev);
2443
2444         /* Locked once because all functions invoked below iterates list */
2445         spin_lock_bh(&vsi->mac_filter_list_lock);
2446
2447         if (is_netdev)
2448                 i40e_del_filter(vsi, netdev->dev_addr, vid, is_vf, is_netdev);
2449
2450         list_for_each_entry(f, &vsi->mac_filter_list, list)
2451                 i40e_del_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
2452
2453         /* go through all the filters for this VSI and if there is only
2454          * vid == 0 it means there are no other filters, so vid 0 must
2455          * be replaced with -1. This signifies that we should from now
2456          * on accept any traffic (with any tag present, or untagged)
2457          */
2458         list_for_each_entry(f, &vsi->mac_filter_list, list) {
2459                 if (is_netdev) {
2460                         if (f->vlan &&
2461                             ether_addr_equal(netdev->dev_addr, f->macaddr))
2462                                 filter_count++;
2463                 }
2464
2465                 if (f->vlan)
2466                         filter_count++;
2467         }
2468
2469         if (!filter_count && is_netdev) {
2470                 i40e_del_filter(vsi, netdev->dev_addr, 0, is_vf, is_netdev);
2471                 f = i40e_add_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
2472                                     is_vf, is_netdev);
2473                 if (!f) {
2474                         dev_info(&vsi->back->pdev->dev,
2475                                  "Could not add filter %d for %pM\n",
2476                                  I40E_VLAN_ANY, netdev->dev_addr);
2477                         spin_unlock_bh(&vsi->mac_filter_list_lock);
2478                         return -ENOMEM;
2479                 }
2480         }
2481
2482         if (!filter_count) {
2483                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
2484                         i40e_del_filter(vsi, f->macaddr, 0, is_vf, is_netdev);
2485                         add_f = i40e_add_filter(vsi, f->macaddr, I40E_VLAN_ANY,
2486                                                 is_vf, is_netdev);
2487                         if (!add_f) {
2488                                 dev_info(&vsi->back->pdev->dev,
2489                                          "Could not add filter %d for %pM\n",
2490                                          I40E_VLAN_ANY, f->macaddr);
2491                                 spin_unlock_bh(&vsi->mac_filter_list_lock);
2492                                 return -ENOMEM;
2493                         }
2494                 }
2495         }
2496
2497         spin_unlock_bh(&vsi->mac_filter_list_lock);
2498
2499         /* schedule our worker thread which will take care of
2500          * applying the new filter changes
2501          */
2502         i40e_service_event_schedule(vsi->back);
2503         return 0;
2504 }
2505
2506 /**
2507  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2508  * @netdev: network interface to be adjusted
2509  * @vid: vlan id to be added
2510  *
2511  * net_device_ops implementation for adding vlan ids
2512  **/
2513 #ifdef I40E_FCOE
2514 int i40e_vlan_rx_add_vid(struct net_device *netdev,
2515                          __always_unused __be16 proto, u16 vid)
2516 #else
2517 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2518                                 __always_unused __be16 proto, u16 vid)
2519 #endif
2520 {
2521         struct i40e_netdev_priv *np = netdev_priv(netdev);
2522         struct i40e_vsi *vsi = np->vsi;
2523         int ret = 0;
2524
2525         if (vid > 4095)
2526                 return -EINVAL;
2527
2528         netdev_info(netdev, "adding %pM vid=%d\n", netdev->dev_addr, vid);
2529
2530         /* If the network stack called us with vid = 0 then
2531          * it is asking to receive priority tagged packets with
2532          * vlan id 0.  Our HW receives them by default when configured
2533          * to receive untagged packets so there is no need to add an
2534          * extra filter for vlan 0 tagged packets.
2535          */
2536         if (vid)
2537                 ret = i40e_vsi_add_vlan(vsi, vid);
2538
2539         if (!ret && (vid < VLAN_N_VID))
2540                 set_bit(vid, vsi->active_vlans);
2541
2542         return ret;
2543 }
2544
2545 /**
2546  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2547  * @netdev: network interface to be adjusted
2548  * @vid: vlan id to be removed
2549  *
2550  * net_device_ops implementation for removing vlan ids
2551  **/
2552 #ifdef I40E_FCOE
2553 int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2554                           __always_unused __be16 proto, u16 vid)
2555 #else
2556 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2557                                  __always_unused __be16 proto, u16 vid)
2558 #endif
2559 {
2560         struct i40e_netdev_priv *np = netdev_priv(netdev);
2561         struct i40e_vsi *vsi = np->vsi;
2562
2563         netdev_info(netdev, "removing %pM vid=%d\n", netdev->dev_addr, vid);
2564
2565         /* return code is ignored as there is nothing a user
2566          * can do about failure to remove and a log message was
2567          * already printed from the other function
2568          */
2569         i40e_vsi_kill_vlan(vsi, vid);
2570
2571         clear_bit(vid, vsi->active_vlans);
2572
2573         return 0;
2574 }
2575
2576 /**
2577  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2578  * @vsi: the vsi being brought back up
2579  **/
2580 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2581 {
2582         u16 vid;
2583
2584         if (!vsi->netdev)
2585                 return;
2586
2587         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2588
2589         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2590                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2591                                      vid);
2592 }
2593
2594 /**
2595  * i40e_vsi_add_pvid - Add pvid for the VSI
2596  * @vsi: the vsi being adjusted
2597  * @vid: the vlan id to set as a PVID
2598  **/
2599 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2600 {
2601         struct i40e_vsi_context ctxt;
2602         i40e_status ret;
2603
2604         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2605         vsi->info.pvid = cpu_to_le16(vid);
2606         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2607                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2608                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2609
2610         ctxt.seid = vsi->seid;
2611         ctxt.info = vsi->info;
2612         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2613         if (ret) {
2614                 dev_info(&vsi->back->pdev->dev,
2615                          "add pvid failed, err %s aq_err %s\n",
2616                          i40e_stat_str(&vsi->back->hw, ret),
2617                          i40e_aq_str(&vsi->back->hw,
2618                                      vsi->back->hw.aq.asq_last_status));
2619                 return -ENOENT;
2620         }
2621
2622         return 0;
2623 }
2624
2625 /**
2626  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2627  * @vsi: the vsi being adjusted
2628  *
2629  * Just use the vlan_rx_register() service to put it back to normal
2630  **/
2631 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2632 {
2633         i40e_vlan_stripping_disable(vsi);
2634
2635         vsi->info.pvid = 0;
2636 }
2637
2638 /**
2639  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2640  * @vsi: ptr to the VSI
2641  *
2642  * If this function returns with an error, then it's possible one or
2643  * more of the rings is populated (while the rest are not).  It is the
2644  * callers duty to clean those orphaned rings.
2645  *
2646  * Return 0 on success, negative on failure
2647  **/
2648 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2649 {
2650         int i, err = 0;
2651
2652         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2653                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2654
2655         return err;
2656 }
2657
2658 /**
2659  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2660  * @vsi: ptr to the VSI
2661  *
2662  * Free VSI's transmit software resources
2663  **/
2664 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2665 {
2666         int i;
2667
2668         if (!vsi->tx_rings)
2669                 return;
2670
2671         for (i = 0; i < vsi->num_queue_pairs; i++)
2672                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2673                         i40e_free_tx_resources(vsi->tx_rings[i]);
2674 }
2675
2676 /**
2677  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2678  * @vsi: ptr to the VSI
2679  *
2680  * If this function returns with an error, then it's possible one or
2681  * more of the rings is populated (while the rest are not).  It is the
2682  * callers duty to clean those orphaned rings.
2683  *
2684  * Return 0 on success, negative on failure
2685  **/
2686 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2687 {
2688         int i, err = 0;
2689
2690         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2691                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2692 #ifdef I40E_FCOE
2693         i40e_fcoe_setup_ddp_resources(vsi);
2694 #endif
2695         return err;
2696 }
2697
2698 /**
2699  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2700  * @vsi: ptr to the VSI
2701  *
2702  * Free all receive software resources
2703  **/
2704 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2705 {
2706         int i;
2707
2708         if (!vsi->rx_rings)
2709                 return;
2710
2711         for (i = 0; i < vsi->num_queue_pairs; i++)
2712                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2713                         i40e_free_rx_resources(vsi->rx_rings[i]);
2714 #ifdef I40E_FCOE
2715         i40e_fcoe_free_ddp_resources(vsi);
2716 #endif
2717 }
2718
2719 /**
2720  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
2721  * @ring: The Tx ring to configure
2722  *
2723  * This enables/disables XPS for a given Tx descriptor ring
2724  * based on the TCs enabled for the VSI that ring belongs to.
2725  **/
2726 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
2727 {
2728         struct i40e_vsi *vsi = ring->vsi;
2729         cpumask_var_t mask;
2730
2731         if (!ring->q_vector || !ring->netdev)
2732                 return;
2733
2734         /* Single TC mode enable XPS */
2735         if (vsi->tc_config.numtc <= 1) {
2736                 if (!test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
2737                         netif_set_xps_queue(ring->netdev,
2738                                             &ring->q_vector->affinity_mask,
2739                                             ring->queue_index);
2740         } else if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
2741                 /* Disable XPS to allow selection based on TC */
2742                 bitmap_zero(cpumask_bits(mask), nr_cpumask_bits);
2743                 netif_set_xps_queue(ring->netdev, mask, ring->queue_index);
2744                 free_cpumask_var(mask);
2745         }
2746
2747         /* schedule our worker thread which will take care of
2748          * applying the new filter changes
2749          */
2750         i40e_service_event_schedule(vsi->back);
2751 }
2752
2753 /**
2754  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2755  * @ring: The Tx ring to configure
2756  *
2757  * Configure the Tx descriptor ring in the HMC context.
2758  **/
2759 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2760 {
2761         struct i40e_vsi *vsi = ring->vsi;
2762         u16 pf_q = vsi->base_queue + ring->queue_index;
2763         struct i40e_hw *hw = &vsi->back->hw;
2764         struct i40e_hmc_obj_txq tx_ctx;
2765         i40e_status err = 0;
2766         u32 qtx_ctl = 0;
2767
2768         /* some ATR related tx ring init */
2769         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
2770                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2771                 ring->atr_count = 0;
2772         } else {
2773                 ring->atr_sample_rate = 0;
2774         }
2775
2776         /* configure XPS */
2777         i40e_config_xps_tx_ring(ring);
2778
2779         /* clear the context structure first */
2780         memset(&tx_ctx, 0, sizeof(tx_ctx));
2781
2782         tx_ctx.new_context = 1;
2783         tx_ctx.base = (ring->dma / 128);
2784         tx_ctx.qlen = ring->count;
2785         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
2786                                                I40E_FLAG_FD_ATR_ENABLED));
2787 #ifdef I40E_FCOE
2788         tx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE);
2789 #endif
2790         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2791         /* FDIR VSI tx ring can still use RS bit and writebacks */
2792         if (vsi->type != I40E_VSI_FDIR)
2793                 tx_ctx.head_wb_ena = 1;
2794         tx_ctx.head_wb_addr = ring->dma +
2795                               (ring->count * sizeof(struct i40e_tx_desc));
2796
2797         /* As part of VSI creation/update, FW allocates certain
2798          * Tx arbitration queue sets for each TC enabled for
2799          * the VSI. The FW returns the handles to these queue
2800          * sets as part of the response buffer to Add VSI,
2801          * Update VSI, etc. AQ commands. It is expected that
2802          * these queue set handles be associated with the Tx
2803          * queues by the driver as part of the TX queue context
2804          * initialization. This has to be done regardless of
2805          * DCB as by default everything is mapped to TC0.
2806          */
2807         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2808         tx_ctx.rdylist_act = 0;
2809
2810         /* clear the context in the HMC */
2811         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2812         if (err) {
2813                 dev_info(&vsi->back->pdev->dev,
2814                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2815                          ring->queue_index, pf_q, err);
2816                 return -ENOMEM;
2817         }
2818
2819         /* set the context in the HMC */
2820         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2821         if (err) {
2822                 dev_info(&vsi->back->pdev->dev,
2823                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2824                          ring->queue_index, pf_q, err);
2825                 return -ENOMEM;
2826         }
2827
2828         /* Now associate this queue with this PCI function */
2829         if (vsi->type == I40E_VSI_VMDQ2) {
2830                 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
2831                 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
2832                            I40E_QTX_CTL_VFVM_INDX_MASK;
2833         } else {
2834                 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2835         }
2836
2837         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2838                     I40E_QTX_CTL_PF_INDX_MASK);
2839         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2840         i40e_flush(hw);
2841
2842         /* cache tail off for easier writes later */
2843         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2844
2845         return 0;
2846 }
2847
2848 /**
2849  * i40e_configure_rx_ring - Configure a receive ring context
2850  * @ring: The Rx ring to configure
2851  *
2852  * Configure the Rx descriptor ring in the HMC context.
2853  **/
2854 static int i40e_configure_rx_ring(struct i40e_ring *ring)
2855 {
2856         struct i40e_vsi *vsi = ring->vsi;
2857         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
2858         u16 pf_q = vsi->base_queue + ring->queue_index;
2859         struct i40e_hw *hw = &vsi->back->hw;
2860         struct i40e_hmc_obj_rxq rx_ctx;
2861         i40e_status err = 0;
2862
2863         ring->state = 0;
2864
2865         /* clear the context structure first */
2866         memset(&rx_ctx, 0, sizeof(rx_ctx));
2867
2868         ring->rx_buf_len = vsi->rx_buf_len;
2869         ring->rx_hdr_len = vsi->rx_hdr_len;
2870
2871         rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
2872         rx_ctx.hbuff = ring->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;
2873
2874         rx_ctx.base = (ring->dma / 128);
2875         rx_ctx.qlen = ring->count;
2876
2877         if (vsi->back->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED) {
2878                 set_ring_16byte_desc_enabled(ring);
2879                 rx_ctx.dsize = 0;
2880         } else {
2881                 rx_ctx.dsize = 1;
2882         }
2883
2884         rx_ctx.dtype = vsi->dtype;
2885         if (vsi->dtype) {
2886                 set_ring_ps_enabled(ring);
2887                 rx_ctx.hsplit_0 = I40E_RX_SPLIT_L2      |
2888                                   I40E_RX_SPLIT_IP      |
2889                                   I40E_RX_SPLIT_TCP_UDP |
2890                                   I40E_RX_SPLIT_SCTP;
2891         } else {
2892                 rx_ctx.hsplit_0 = 0;
2893         }
2894
2895         rx_ctx.rxmax = min_t(u16, vsi->max_frame,
2896                                   (chain_len * ring->rx_buf_len));
2897         if (hw->revision_id == 0)
2898                 rx_ctx.lrxqthresh = 0;
2899         else
2900                 rx_ctx.lrxqthresh = 2;
2901         rx_ctx.crcstrip = 1;
2902         rx_ctx.l2tsel = 1;
2903         /* this controls whether VLAN is stripped from inner headers */
2904         rx_ctx.showiv = 0;
2905 #ifdef I40E_FCOE
2906         rx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE);
2907 #endif
2908         /* set the prefena field to 1 because the manual says to */
2909         rx_ctx.prefena = 1;
2910
2911         /* clear the context in the HMC */
2912         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
2913         if (err) {
2914                 dev_info(&vsi->back->pdev->dev,
2915                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2916                          ring->queue_index, pf_q, err);
2917                 return -ENOMEM;
2918         }
2919
2920         /* set the context in the HMC */
2921         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
2922         if (err) {
2923                 dev_info(&vsi->back->pdev->dev,
2924                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2925                          ring->queue_index, pf_q, err);
2926                 return -ENOMEM;
2927         }
2928
2929         /* cache tail for quicker writes, and clear the reg before use */
2930         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
2931         writel(0, ring->tail);
2932
2933         if (ring_is_ps_enabled(ring)) {
2934                 i40e_alloc_rx_headers(ring);
2935                 i40e_alloc_rx_buffers_ps(ring, I40E_DESC_UNUSED(ring));
2936         } else {
2937                 i40e_alloc_rx_buffers_1buf(ring, I40E_DESC_UNUSED(ring));
2938         }
2939
2940         return 0;
2941 }
2942
2943 /**
2944  * i40e_vsi_configure_tx - Configure the VSI for Tx
2945  * @vsi: VSI structure describing this set of rings and resources
2946  *
2947  * Configure the Tx VSI for operation.
2948  **/
2949 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
2950 {
2951         int err = 0;
2952         u16 i;
2953
2954         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
2955                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
2956
2957         return err;
2958 }
2959
2960 /**
2961  * i40e_vsi_configure_rx - Configure the VSI for Rx
2962  * @vsi: the VSI being configured
2963  *
2964  * Configure the Rx VSI for operation.
2965  **/
2966 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
2967 {
2968         int err = 0;
2969         u16 i;
2970
2971         if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN))
2972                 vsi->max_frame = vsi->netdev->mtu + ETH_HLEN
2973                                + ETH_FCS_LEN + VLAN_HLEN;
2974         else
2975                 vsi->max_frame = I40E_RXBUFFER_2048;
2976
2977         /* figure out correct receive buffer length */
2978         switch (vsi->back->flags & (I40E_FLAG_RX_1BUF_ENABLED |
2979                                     I40E_FLAG_RX_PS_ENABLED)) {
2980         case I40E_FLAG_RX_1BUF_ENABLED:
2981                 vsi->rx_hdr_len = 0;
2982                 vsi->rx_buf_len = vsi->max_frame;
2983                 vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
2984                 break;
2985         case I40E_FLAG_RX_PS_ENABLED:
2986                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2987                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2988                 vsi->dtype = I40E_RX_DTYPE_HEADER_SPLIT;
2989                 break;
2990         default:
2991                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2992                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2993                 vsi->dtype = I40E_RX_DTYPE_SPLIT_ALWAYS;
2994                 break;
2995         }
2996
2997 #ifdef I40E_FCOE
2998         /* setup rx buffer for FCoE */
2999         if ((vsi->type == I40E_VSI_FCOE) &&
3000             (vsi->back->flags & I40E_FLAG_FCOE_ENABLED)) {
3001                 vsi->rx_hdr_len = 0;
3002                 vsi->rx_buf_len = I40E_RXBUFFER_3072;
3003                 vsi->max_frame = I40E_RXBUFFER_3072;
3004                 vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
3005         }
3006
3007 #endif /* I40E_FCOE */
3008         /* round up for the chip's needs */
3009         vsi->rx_hdr_len = ALIGN(vsi->rx_hdr_len,
3010                                 BIT_ULL(I40E_RXQ_CTX_HBUFF_SHIFT));
3011         vsi->rx_buf_len = ALIGN(vsi->rx_buf_len,
3012                                 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3013
3014         /* set up individual rings */
3015         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3016                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3017
3018         return err;
3019 }
3020
3021 /**
3022  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3023  * @vsi: ptr to the VSI
3024  **/
3025 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3026 {
3027         struct i40e_ring *tx_ring, *rx_ring;
3028         u16 qoffset, qcount;
3029         int i, n;
3030
3031         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3032                 /* Reset the TC information */
3033                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3034                         rx_ring = vsi->rx_rings[i];
3035                         tx_ring = vsi->tx_rings[i];
3036                         rx_ring->dcb_tc = 0;
3037                         tx_ring->dcb_tc = 0;
3038                 }
3039         }
3040
3041         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3042                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3043                         continue;
3044
3045                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3046                 qcount = vsi->tc_config.tc_info[n].qcount;
3047                 for (i = qoffset; i < (qoffset + qcount); i++) {
3048                         rx_ring = vsi->rx_rings[i];
3049                         tx_ring = vsi->tx_rings[i];
3050                         rx_ring->dcb_tc = n;
3051                         tx_ring->dcb_tc = n;
3052                 }
3053         }
3054 }
3055
3056 /**
3057  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3058  * @vsi: ptr to the VSI
3059  **/
3060 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3061 {
3062         if (vsi->netdev)
3063                 i40e_set_rx_mode(vsi->netdev);
3064 }
3065
3066 /**
3067  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3068  * @vsi: Pointer to the targeted VSI
3069  *
3070  * This function replays the hlist on the hw where all the SB Flow Director
3071  * filters were saved.
3072  **/
3073 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3074 {
3075         struct i40e_fdir_filter *filter;
3076         struct i40e_pf *pf = vsi->back;
3077         struct hlist_node *node;
3078
3079         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3080                 return;
3081
3082         hlist_for_each_entry_safe(filter, node,
3083                                   &pf->fdir_filter_list, fdir_node) {
3084                 i40e_add_del_fdir(vsi, filter, true);
3085         }
3086 }
3087
3088 /**
3089  * i40e_vsi_configure - Set up the VSI for action
3090  * @vsi: the VSI being configured
3091  **/
3092 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3093 {
3094         int err;
3095
3096         i40e_set_vsi_rx_mode(vsi);
3097         i40e_restore_vlan(vsi);
3098         i40e_vsi_config_dcb_rings(vsi);
3099         err = i40e_vsi_configure_tx(vsi);
3100         if (!err)
3101                 err = i40e_vsi_configure_rx(vsi);
3102
3103         return err;
3104 }
3105
3106 /**
3107  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3108  * @vsi: the VSI being configured
3109  **/
3110 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3111 {
3112         struct i40e_pf *pf = vsi->back;
3113         struct i40e_hw *hw = &pf->hw;
3114         u16 vector;
3115         int i, q;
3116         u32 qp;
3117
3118         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3119          * and PFINT_LNKLSTn registers, e.g.:
3120          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3121          */
3122         qp = vsi->base_queue;
3123         vector = vsi->base_vector;
3124         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3125                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3126
3127                 q_vector->itr_countdown = ITR_COUNTDOWN_START;
3128                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting);
3129                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
3130                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3131                      q_vector->rx.itr);
3132                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting);
3133                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
3134                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3135                      q_vector->tx.itr);
3136                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3137                      INTRL_USEC_TO_REG(vsi->int_rate_limit));
3138
3139                 /* Linked list for the queuepairs assigned to this vector */
3140                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3141                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3142                         u32 val;
3143
3144                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3145                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3146                               (vector      << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3147                               (qp          << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3148                               (I40E_QUEUE_TYPE_TX
3149                                       << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3150
3151                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3152
3153                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3154                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)  |
3155                               (vector      << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3156                               ((qp+1)      << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)|
3157                               (I40E_QUEUE_TYPE_RX
3158                                       << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3159
3160                         /* Terminate the linked list */
3161                         if (q == (q_vector->num_ringpairs - 1))
3162                                 val |= (I40E_QUEUE_END_OF_LIST
3163                                            << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3164
3165                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3166                         qp++;
3167                 }
3168         }
3169
3170         i40e_flush(hw);
3171 }
3172
3173 /**
3174  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3175  * @hw: ptr to the hardware info
3176  **/
3177 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3178 {
3179         struct i40e_hw *hw = &pf->hw;
3180         u32 val;
3181
3182         /* clear things first */
3183         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3184         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3185
3186         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3187               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3188               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3189               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3190               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3191               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3192               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3193               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3194
3195         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3196                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3197
3198         if (pf->flags & I40E_FLAG_PTP)
3199                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3200
3201         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3202
3203         /* SW_ITR_IDX = 0, but don't change INTENA */
3204         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3205                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3206
3207         /* OTHER_ITR_IDX = 0 */
3208         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3209 }
3210
3211 /**
3212  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3213  * @vsi: the VSI being configured
3214  **/
3215 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3216 {
3217         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3218         struct i40e_pf *pf = vsi->back;
3219         struct i40e_hw *hw = &pf->hw;
3220         u32 val;
3221
3222         /* set the ITR configuration */
3223         q_vector->itr_countdown = ITR_COUNTDOWN_START;
3224         q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting);
3225         q_vector->rx.latency_range = I40E_LOW_LATENCY;
3226         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
3227         q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting);
3228         q_vector->tx.latency_range = I40E_LOW_LATENCY;
3229         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
3230
3231         i40e_enable_misc_int_causes(pf);
3232
3233         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3234         wr32(hw, I40E_PFINT_LNKLST0, 0);
3235
3236         /* Associate the queue pair to the vector and enable the queue int */
3237         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                  |
3238               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3239               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3240
3241         wr32(hw, I40E_QINT_RQCTL(0), val);
3242
3243         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3244               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3245               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3246
3247         wr32(hw, I40E_QINT_TQCTL(0), val);
3248         i40e_flush(hw);
3249 }
3250
3251 /**
3252  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3253  * @pf: board private structure
3254  **/
3255 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3256 {
3257         struct i40e_hw *hw = &pf->hw;
3258
3259         wr32(hw, I40E_PFINT_DYN_CTL0,
3260              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3261         i40e_flush(hw);
3262 }
3263
3264 /**
3265  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3266  * @pf: board private structure
3267  * @clearpba: true when all pending interrupt events should be cleared
3268  **/
3269 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba)
3270 {
3271         struct i40e_hw *hw = &pf->hw;
3272         u32 val;
3273
3274         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3275               (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) |
3276               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3277
3278         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3279         i40e_flush(hw);
3280 }
3281
3282 /**
3283  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3284  * @irq: interrupt number
3285  * @data: pointer to a q_vector
3286  **/
3287 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3288 {
3289         struct i40e_q_vector *q_vector = data;
3290
3291         if (!q_vector->tx.ring && !q_vector->rx.ring)
3292                 return IRQ_HANDLED;
3293
3294         napi_schedule_irqoff(&q_vector->napi);
3295
3296         return IRQ_HANDLED;
3297 }
3298
3299 /**
3300  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3301  * @vsi: the VSI being configured
3302  * @basename: name for the vector
3303  *
3304  * Allocates MSI-X vectors and requests interrupts from the kernel.
3305  **/
3306 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3307 {
3308         int q_vectors = vsi->num_q_vectors;
3309         struct i40e_pf *pf = vsi->back;
3310         int base = vsi->base_vector;
3311         int rx_int_idx = 0;
3312         int tx_int_idx = 0;
3313         int vector, err;
3314
3315         for (vector = 0; vector < q_vectors; vector++) {
3316                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3317
3318                 if (q_vector->tx.ring && q_vector->rx.ring) {
3319                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3320                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3321                         tx_int_idx++;
3322                 } else if (q_vector->rx.ring) {
3323                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3324                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3325                 } else if (q_vector->tx.ring) {
3326                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3327                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3328                 } else {
3329                         /* skip this unused q_vector */
3330                         continue;
3331                 }
3332                 err = request_irq(pf->msix_entries[base + vector].vector,
3333                                   vsi->irq_handler,
3334                                   0,
3335                                   q_vector->name,
3336                                   q_vector);
3337                 if (err) {
3338                         dev_info(&pf->pdev->dev,
3339                                  "MSIX request_irq failed, error: %d\n", err);
3340                         goto free_queue_irqs;
3341                 }
3342                 /* assign the mask for this irq */
3343                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
3344                                       &q_vector->affinity_mask);
3345         }
3346
3347         vsi->irqs_ready = true;
3348         return 0;
3349
3350 free_queue_irqs:
3351         while (vector) {
3352                 vector--;
3353                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
3354                                       NULL);
3355                 free_irq(pf->msix_entries[base + vector].vector,
3356                          &(vsi->q_vectors[vector]));
3357         }
3358         return err;
3359 }
3360
3361 /**
3362  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3363  * @vsi: the VSI being un-configured
3364  **/
3365 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3366 {
3367         struct i40e_pf *pf = vsi->back;
3368         struct i40e_hw *hw = &pf->hw;
3369         int base = vsi->base_vector;
3370         int i;
3371
3372         for (i = 0; i < vsi->num_queue_pairs; i++) {
3373                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
3374                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
3375         }
3376
3377         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3378                 for (i = vsi->base_vector;
3379                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3380                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3381
3382                 i40e_flush(hw);
3383                 for (i = 0; i < vsi->num_q_vectors; i++)
3384                         synchronize_irq(pf->msix_entries[i + base].vector);
3385         } else {
3386                 /* Legacy and MSI mode - this stops all interrupt handling */
3387                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3388                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3389                 i40e_flush(hw);
3390                 synchronize_irq(pf->pdev->irq);
3391         }
3392 }
3393
3394 /**
3395  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3396  * @vsi: the VSI being configured
3397  **/
3398 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3399 {
3400         struct i40e_pf *pf = vsi->back;
3401         int i;
3402
3403         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3404                 for (i = 0; i < vsi->num_q_vectors; i++)
3405                         i40e_irq_dynamic_enable(vsi, i);
3406         } else {
3407                 i40e_irq_dynamic_enable_icr0(pf, true);
3408         }
3409
3410         i40e_flush(&pf->hw);
3411         return 0;
3412 }
3413
3414 /**
3415  * i40e_stop_misc_vector - Stop the vector that handles non-queue events
3416  * @pf: board private structure
3417  **/
3418 static void i40e_stop_misc_vector(struct i40e_pf *pf)
3419 {
3420         /* Disable ICR 0 */
3421         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3422         i40e_flush(&pf->hw);
3423 }
3424
3425 /**
3426  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3427  * @irq: interrupt number
3428  * @data: pointer to a q_vector
3429  *
3430  * This is the handler used for all MSI/Legacy interrupts, and deals
3431  * with both queue and non-queue interrupts.  This is also used in
3432  * MSIX mode to handle the non-queue interrupts.
3433  **/
3434 static irqreturn_t i40e_intr(int irq, void *data)
3435 {
3436         struct i40e_pf *pf = (struct i40e_pf *)data;
3437         struct i40e_hw *hw = &pf->hw;
3438         irqreturn_t ret = IRQ_NONE;
3439         u32 icr0, icr0_remaining;
3440         u32 val, ena_mask;
3441
3442         icr0 = rd32(hw, I40E_PFINT_ICR0);
3443         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3444
3445         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3446         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3447                 goto enable_intr;
3448
3449         /* if interrupt but no bits showing, must be SWINT */
3450         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3451             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3452                 pf->sw_int_count++;
3453
3454         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3455             (ena_mask & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3456                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3457                 icr0 &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3458                 dev_info(&pf->pdev->dev, "cleared PE_CRITERR\n");
3459         }
3460
3461         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3462         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3463                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3464                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3465
3466                 /* We do not have a way to disarm Queue causes while leaving
3467                  * interrupt enabled for all other causes, ideally
3468                  * interrupt should be disabled while we are in NAPI but
3469                  * this is not a performance path and napi_schedule()
3470                  * can deal with rescheduling.
3471                  */
3472                 if (!test_bit(__I40E_DOWN, &pf->state))
3473                         napi_schedule_irqoff(&q_vector->napi);
3474         }
3475
3476         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3477                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3478                 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
3479                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3480         }
3481
3482         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3483                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3484                 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
3485         }
3486
3487         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3488                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3489                 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state);
3490         }
3491
3492         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3493                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
3494                         set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
3495                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3496                 val = rd32(hw, I40E_GLGEN_RSTAT);
3497                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3498                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3499                 if (val == I40E_RESET_CORER) {
3500                         pf->corer_count++;
3501                 } else if (val == I40E_RESET_GLOBR) {
3502                         pf->globr_count++;
3503                 } else if (val == I40E_RESET_EMPR) {
3504                         pf->empr_count++;
3505                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state);
3506                 }
3507         }
3508
3509         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3510                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3511                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3512                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
3513                          rd32(hw, I40E_PFHMC_ERRORINFO),
3514                          rd32(hw, I40E_PFHMC_ERRORDATA));
3515         }
3516
3517         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3518                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3519
3520                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
3521                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3522                         i40e_ptp_tx_hwtstamp(pf);
3523                 }
3524         }
3525
3526         /* If a critical error is pending we have no choice but to reset the
3527          * device.
3528          * Report and mask out any remaining unexpected interrupts.
3529          */
3530         icr0_remaining = icr0 & ena_mask;
3531         if (icr0_remaining) {
3532                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
3533                          icr0_remaining);
3534                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
3535                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
3536                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
3537                         dev_info(&pf->pdev->dev, "device will be reset\n");
3538                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
3539                         i40e_service_event_schedule(pf);
3540                 }
3541                 ena_mask &= ~icr0_remaining;
3542         }
3543         ret = IRQ_HANDLED;
3544
3545 enable_intr:
3546         /* re-enable interrupt causes */
3547         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
3548         if (!test_bit(__I40E_DOWN, &pf->state)) {
3549                 i40e_service_event_schedule(pf);
3550                 i40e_irq_dynamic_enable_icr0(pf, false);
3551         }
3552
3553         return ret;
3554 }
3555
3556 /**
3557  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
3558  * @tx_ring:  tx ring to clean
3559  * @budget:   how many cleans we're allowed
3560  *
3561  * Returns true if there's any budget left (e.g. the clean is finished)
3562  **/
3563 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
3564 {
3565         struct i40e_vsi *vsi = tx_ring->vsi;
3566         u16 i = tx_ring->next_to_clean;
3567         struct i40e_tx_buffer *tx_buf;
3568         struct i40e_tx_desc *tx_desc;
3569
3570         tx_buf = &tx_ring->tx_bi[i];
3571         tx_desc = I40E_TX_DESC(tx_ring, i);
3572         i -= tx_ring->count;
3573
3574         do {
3575                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
3576
3577                 /* if next_to_watch is not set then there is no work pending */
3578                 if (!eop_desc)
3579                         break;
3580
3581                 /* prevent any other reads prior to eop_desc */
3582                 read_barrier_depends();
3583
3584                 /* if the descriptor isn't done, no work yet to do */
3585                 if (!(eop_desc->cmd_type_offset_bsz &
3586                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
3587                         break;
3588
3589                 /* clear next_to_watch to prevent false hangs */
3590                 tx_buf->next_to_watch = NULL;
3591
3592                 tx_desc->buffer_addr = 0;
3593                 tx_desc->cmd_type_offset_bsz = 0;
3594                 /* move past filter desc */
3595                 tx_buf++;
3596                 tx_desc++;
3597                 i++;
3598                 if (unlikely(!i)) {
3599                         i -= tx_ring->count;
3600                         tx_buf = tx_ring->tx_bi;
3601                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3602                 }
3603                 /* unmap skb header data */
3604                 dma_unmap_single(tx_ring->dev,
3605                                  dma_unmap_addr(tx_buf, dma),
3606                                  dma_unmap_len(tx_buf, len),
3607                                  DMA_TO_DEVICE);
3608                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
3609                         kfree(tx_buf->raw_buf);
3610
3611                 tx_buf->raw_buf = NULL;
3612                 tx_buf->tx_flags = 0;
3613                 tx_buf->next_to_watch = NULL;
3614                 dma_unmap_len_set(tx_buf, len, 0);
3615                 tx_desc->buffer_addr = 0;
3616                 tx_desc->cmd_type_offset_bsz = 0;
3617
3618                 /* move us past the eop_desc for start of next FD desc */
3619                 tx_buf++;
3620                 tx_desc++;
3621                 i++;
3622                 if (unlikely(!i)) {
3623                         i -= tx_ring->count;
3624                         tx_buf = tx_ring->tx_bi;
3625                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3626                 }
3627
3628                 /* update budget accounting */
3629                 budget--;
3630         } while (likely(budget));
3631
3632         i += tx_ring->count;
3633         tx_ring->next_to_clean = i;
3634
3635         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
3636                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
3637
3638         return budget > 0;
3639 }
3640
3641 /**
3642  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
3643  * @irq: interrupt number
3644  * @data: pointer to a q_vector
3645  **/
3646 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
3647 {
3648         struct i40e_q_vector *q_vector = data;
3649         struct i40e_vsi *vsi;
3650
3651         if (!q_vector->tx.ring)
3652                 return IRQ_HANDLED;
3653
3654         vsi = q_vector->tx.ring->vsi;
3655         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
3656
3657         return IRQ_HANDLED;
3658 }
3659
3660 /**
3661  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
3662  * @vsi: the VSI being configured
3663  * @v_idx: vector index
3664  * @qp_idx: queue pair index
3665  **/
3666 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
3667 {
3668         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3669         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
3670         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
3671
3672         tx_ring->q_vector = q_vector;
3673         tx_ring->next = q_vector->tx.ring;
3674         q_vector->tx.ring = tx_ring;
3675         q_vector->tx.count++;
3676
3677         rx_ring->q_vector = q_vector;
3678         rx_ring->next = q_vector->rx.ring;
3679         q_vector->rx.ring = rx_ring;
3680         q_vector->rx.count++;
3681 }
3682
3683 /**
3684  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
3685  * @vsi: the VSI being configured
3686  *
3687  * This function maps descriptor rings to the queue-specific vectors
3688  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
3689  * one vector per queue pair, but on a constrained vector budget, we
3690  * group the queue pairs as "efficiently" as possible.
3691  **/
3692 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
3693 {
3694         int qp_remaining = vsi->num_queue_pairs;
3695         int q_vectors = vsi->num_q_vectors;
3696         int num_ringpairs;
3697         int v_start = 0;
3698         int qp_idx = 0;
3699
3700         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
3701          * group them so there are multiple queues per vector.
3702          * It is also important to go through all the vectors available to be
3703          * sure that if we don't use all the vectors, that the remaining vectors
3704          * are cleared. This is especially important when decreasing the
3705          * number of queues in use.
3706          */
3707         for (; v_start < q_vectors; v_start++) {
3708                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
3709
3710                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
3711
3712                 q_vector->num_ringpairs = num_ringpairs;
3713
3714                 q_vector->rx.count = 0;
3715                 q_vector->tx.count = 0;
3716                 q_vector->rx.ring = NULL;
3717                 q_vector->tx.ring = NULL;
3718
3719                 while (num_ringpairs--) {
3720                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
3721                         qp_idx++;
3722                         qp_remaining--;
3723                 }
3724         }
3725 }
3726
3727 /**
3728  * i40e_vsi_request_irq - Request IRQ from the OS
3729  * @vsi: the VSI being configured
3730  * @basename: name for the vector
3731  **/
3732 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
3733 {
3734         struct i40e_pf *pf = vsi->back;
3735         int err;
3736
3737         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3738                 err = i40e_vsi_request_irq_msix(vsi, basename);
3739         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
3740                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
3741                                   pf->int_name, pf);
3742         else
3743                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
3744                                   pf->int_name, pf);
3745
3746         if (err)
3747                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
3748
3749         return err;
3750 }
3751
3752 #ifdef CONFIG_NET_POLL_CONTROLLER
3753 /**
3754  * i40e_netpoll - A Polling 'interrupt' handler
3755  * @netdev: network interface device structure
3756  *
3757  * This is used by netconsole to send skbs without having to re-enable
3758  * interrupts.  It's not called while the normal interrupt routine is executing.
3759  **/
3760 #ifdef I40E_FCOE
3761 void i40e_netpoll(struct net_device *netdev)
3762 #else
3763 static void i40e_netpoll(struct net_device *netdev)
3764 #endif
3765 {
3766         struct i40e_netdev_priv *np = netdev_priv(netdev);
3767         struct i40e_vsi *vsi = np->vsi;
3768         struct i40e_pf *pf = vsi->back;
3769         int i;
3770
3771         /* if interface is down do nothing */
3772         if (test_bit(__I40E_DOWN, &vsi->state))
3773                 return;
3774
3775         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3776                 for (i = 0; i < vsi->num_q_vectors; i++)
3777                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
3778         } else {
3779                 i40e_intr(pf->pdev->irq, netdev);
3780         }
3781 }
3782 #endif
3783
3784 /**
3785  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
3786  * @pf: the PF being configured
3787  * @pf_q: the PF queue
3788  * @enable: enable or disable state of the queue
3789  *
3790  * This routine will wait for the given Tx queue of the PF to reach the
3791  * enabled or disabled state.
3792  * Returns -ETIMEDOUT in case of failing to reach the requested state after
3793  * multiple retries; else will return 0 in case of success.
3794  **/
3795 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
3796 {
3797         int i;
3798         u32 tx_reg;
3799
3800         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
3801                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
3802                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3803                         break;
3804
3805                 usleep_range(10, 20);
3806         }
3807         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
3808                 return -ETIMEDOUT;
3809
3810         return 0;
3811 }
3812
3813 /**
3814  * i40e_vsi_control_tx - Start or stop a VSI's rings
3815  * @vsi: the VSI being configured
3816  * @enable: start or stop the rings
3817  **/
3818 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
3819 {
3820         struct i40e_pf *pf = vsi->back;
3821         struct i40e_hw *hw = &pf->hw;
3822         int i, j, pf_q, ret = 0;
3823         u32 tx_reg;
3824
3825         pf_q = vsi->base_queue;
3826         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3827
3828                 /* warn the TX unit of coming changes */
3829                 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
3830                 if (!enable)
3831                         usleep_range(10, 20);
3832
3833                 for (j = 0; j < 50; j++) {
3834                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
3835                         if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
3836                             ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
3837                                 break;
3838                         usleep_range(1000, 2000);
3839                 }
3840                 /* Skip if the queue is already in the requested state */
3841                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3842                         continue;
3843
3844                 /* turn on/off the queue */
3845                 if (enable) {
3846                         wr32(hw, I40E_QTX_HEAD(pf_q), 0);
3847                         tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
3848                 } else {
3849                         tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3850                 }
3851
3852                 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
3853                 /* No waiting for the Tx queue to disable */
3854                 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state))
3855                         continue;
3856
3857                 /* wait for the change to finish */
3858                 ret = i40e_pf_txq_wait(pf, pf_q, enable);
3859                 if (ret) {
3860                         dev_info(&pf->pdev->dev,
3861                                  "VSI seid %d Tx ring %d %sable timeout\n",
3862                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
3863                         break;
3864                 }
3865         }
3866
3867         if (hw->revision_id == 0)
3868                 mdelay(50);
3869         return ret;
3870 }
3871
3872 /**
3873  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
3874  * @pf: the PF being configured
3875  * @pf_q: the PF queue
3876  * @enable: enable or disable state of the queue
3877  *
3878  * This routine will wait for the given Rx queue of the PF to reach the
3879  * enabled or disabled state.
3880  * Returns -ETIMEDOUT in case of failing to reach the requested state after
3881  * multiple retries; else will return 0 in case of success.
3882  **/
3883 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
3884 {
3885         int i;
3886         u32 rx_reg;
3887
3888         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
3889                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
3890                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3891                         break;
3892
3893                 usleep_range(10, 20);
3894         }
3895         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
3896                 return -ETIMEDOUT;
3897
3898         return 0;
3899 }
3900
3901 /**
3902  * i40e_vsi_control_rx - Start or stop a VSI's rings
3903  * @vsi: the VSI being configured
3904  * @enable: start or stop the rings
3905  **/
3906 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
3907 {
3908         struct i40e_pf *pf = vsi->back;
3909         struct i40e_hw *hw = &pf->hw;
3910         int i, j, pf_q, ret = 0;
3911         u32 rx_reg;
3912
3913         pf_q = vsi->base_queue;
3914         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3915                 for (j = 0; j < 50; j++) {
3916                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
3917                         if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
3918                             ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
3919                                 break;
3920                         usleep_range(1000, 2000);
3921                 }
3922
3923                 /* Skip if the queue is already in the requested state */
3924                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3925                         continue;
3926
3927                 /* turn on/off the queue */
3928                 if (enable)
3929                         rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
3930                 else
3931                         rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3932                 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
3933                 /* No waiting for the Tx queue to disable */
3934                 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state))
3935                         continue;
3936
3937                 /* wait for the change to finish */
3938                 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
3939                 if (ret) {
3940                         dev_info(&pf->pdev->dev,
3941                                  "VSI seid %d Rx ring %d %sable timeout\n",
3942                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
3943                         break;
3944                 }
3945         }
3946
3947         return ret;
3948 }
3949
3950 /**
3951  * i40e_vsi_control_rings - Start or stop a VSI's rings
3952  * @vsi: the VSI being configured
3953  * @enable: start or stop the rings
3954  **/
3955 int i40e_vsi_control_rings(struct i40e_vsi *vsi, bool request)
3956 {
3957         int ret = 0;
3958
3959         /* do rx first for enable and last for disable */
3960         if (request) {
3961                 ret = i40e_vsi_control_rx(vsi, request);
3962                 if (ret)
3963                         return ret;
3964                 ret = i40e_vsi_control_tx(vsi, request);
3965         } else {
3966                 /* Ignore return value, we need to shutdown whatever we can */
3967                 i40e_vsi_control_tx(vsi, request);
3968                 i40e_vsi_control_rx(vsi, request);
3969         }
3970
3971         return ret;
3972 }
3973
3974 /**
3975  * i40e_vsi_free_irq - Free the irq association with the OS
3976  * @vsi: the VSI being configured
3977  **/
3978 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
3979 {
3980         struct i40e_pf *pf = vsi->back;
3981         struct i40e_hw *hw = &pf->hw;
3982         int base = vsi->base_vector;
3983         u32 val, qp;
3984         int i;
3985
3986         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3987                 if (!vsi->q_vectors)
3988                         return;
3989
3990                 if (!vsi->irqs_ready)
3991                         return;
3992
3993                 vsi->irqs_ready = false;
3994                 for (i = 0; i < vsi->num_q_vectors; i++) {
3995                         u16 vector = i + base;
3996
3997                         /* free only the irqs that were actually requested */
3998                         if (!vsi->q_vectors[i] ||
3999                             !vsi->q_vectors[i]->num_ringpairs)
4000                                 continue;
4001
4002                         /* clear the affinity_mask in the IRQ descriptor */
4003                         irq_set_affinity_hint(pf->msix_entries[vector].vector,
4004                                               NULL);
4005                         free_irq(pf->msix_entries[vector].vector,
4006                                  vsi->q_vectors[i]);
4007
4008                         /* Tear down the interrupt queue link list
4009                          *
4010                          * We know that they come in pairs and always
4011                          * the Rx first, then the Tx.  To clear the
4012                          * link list, stick the EOL value into the
4013                          * next_q field of the registers.
4014                          */
4015                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4016                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4017                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4018                         val |= I40E_QUEUE_END_OF_LIST
4019                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4020                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4021
4022                         while (qp != I40E_QUEUE_END_OF_LIST) {
4023                                 u32 next;
4024
4025                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4026
4027                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4028                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4029                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4030                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4031
4032                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4033                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4034
4035                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4036
4037                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4038
4039                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4040                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4041
4042                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4043                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4044                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4045                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4046
4047                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4048                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4049
4050                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4051                                 qp = next;
4052                         }
4053                 }
4054         } else {
4055                 free_irq(pf->pdev->irq, pf);
4056
4057                 val = rd32(hw, I40E_PFINT_LNKLST0);
4058                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4059                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4060                 val |= I40E_QUEUE_END_OF_LIST
4061                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4062                 wr32(hw, I40E_PFINT_LNKLST0, val);
4063
4064                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4065                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4066                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4067                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4068                          I40E_QINT_RQCTL_INTEVENT_MASK);
4069
4070                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4071                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4072
4073                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4074
4075                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4076
4077                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4078                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4079                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4080                          I40E_QINT_TQCTL_INTEVENT_MASK);
4081
4082                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4083                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4084
4085                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4086         }
4087 }
4088
4089 /**
4090  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4091  * @vsi: the VSI being configured
4092  * @v_idx: Index of vector to be freed
4093  *
4094  * This function frees the memory allocated to the q_vector.  In addition if
4095  * NAPI is enabled it will delete any references to the NAPI struct prior
4096  * to freeing the q_vector.
4097  **/
4098 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4099 {
4100         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4101         struct i40e_ring *ring;
4102
4103         if (!q_vector)
4104                 return;
4105
4106         /* disassociate q_vector from rings */
4107         i40e_for_each_ring(ring, q_vector->tx)
4108                 ring->q_vector = NULL;
4109
4110         i40e_for_each_ring(ring, q_vector->rx)
4111                 ring->q_vector = NULL;
4112
4113         /* only VSI w/ an associated netdev is set up w/ NAPI */
4114         if (vsi->netdev)
4115                 netif_napi_del(&q_vector->napi);
4116
4117         vsi->q_vectors[v_idx] = NULL;
4118
4119         kfree_rcu(q_vector, rcu);
4120 }
4121
4122 /**
4123  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4124  * @vsi: the VSI being un-configured
4125  *
4126  * This frees the memory allocated to the q_vectors and
4127  * deletes references to the NAPI struct.
4128  **/
4129 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4130 {
4131         int v_idx;
4132
4133         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4134                 i40e_free_q_vector(vsi, v_idx);
4135 }
4136
4137 /**
4138  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4139  * @pf: board private structure
4140  **/
4141 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4142 {
4143         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4144         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4145                 pci_disable_msix(pf->pdev);
4146                 kfree(pf->msix_entries);
4147                 pf->msix_entries = NULL;
4148                 kfree(pf->irq_pile);
4149                 pf->irq_pile = NULL;
4150         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4151                 pci_disable_msi(pf->pdev);
4152         }
4153         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4154 }
4155
4156 /**
4157  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4158  * @pf: board private structure
4159  *
4160  * We go through and clear interrupt specific resources and reset the structure
4161  * to pre-load conditions
4162  **/
4163 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4164 {
4165         int i;
4166
4167         i40e_stop_misc_vector(pf);
4168         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
4169                 synchronize_irq(pf->msix_entries[0].vector);
4170                 free_irq(pf->msix_entries[0].vector, pf);
4171         }
4172
4173         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4174                       I40E_IWARP_IRQ_PILE_ID);
4175
4176         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4177         for (i = 0; i < pf->num_alloc_vsi; i++)
4178                 if (pf->vsi[i])
4179                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4180         i40e_reset_interrupt_capability(pf);
4181 }
4182
4183 /**
4184  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4185  * @vsi: the VSI being configured
4186  **/
4187 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4188 {
4189         int q_idx;
4190
4191         if (!vsi->netdev)
4192                 return;
4193
4194         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
4195                 napi_enable(&vsi->q_vectors[q_idx]->napi);
4196 }
4197
4198 /**
4199  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4200  * @vsi: the VSI being configured
4201  **/
4202 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4203 {
4204         int q_idx;
4205
4206         if (!vsi->netdev)
4207                 return;
4208
4209         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
4210                 napi_disable(&vsi->q_vectors[q_idx]->napi);
4211 }
4212
4213 /**
4214  * i40e_vsi_close - Shut down a VSI
4215  * @vsi: the vsi to be quelled
4216  **/
4217 static void i40e_vsi_close(struct i40e_vsi *vsi)
4218 {
4219         bool reset = false;
4220
4221         if (!test_and_set_bit(__I40E_DOWN, &vsi->state))
4222                 i40e_down(vsi);
4223         i40e_vsi_free_irq(vsi);
4224         i40e_vsi_free_tx_resources(vsi);
4225         i40e_vsi_free_rx_resources(vsi);
4226         vsi->current_netdev_flags = 0;
4227         if (test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
4228                 reset = true;
4229         i40e_notify_client_of_netdev_close(vsi, reset);
4230 }
4231
4232 /**
4233  * i40e_quiesce_vsi - Pause a given VSI
4234  * @vsi: the VSI being paused
4235  **/
4236 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4237 {
4238         if (test_bit(__I40E_DOWN, &vsi->state))
4239                 return;
4240
4241         /* No need to disable FCoE VSI when Tx suspended */
4242         if ((test_bit(__I40E_PORT_TX_SUSPENDED, &vsi->back->state)) &&
4243             vsi->type == I40E_VSI_FCOE) {
4244                 dev_dbg(&vsi->back->pdev->dev,
4245                          "VSI seid %d skipping FCoE VSI disable\n", vsi->seid);
4246                 return;
4247         }
4248
4249         set_bit(__I40E_NEEDS_RESTART, &vsi->state);
4250         if (vsi->netdev && netif_running(vsi->netdev))
4251                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4252         else
4253                 i40e_vsi_close(vsi);
4254 }
4255
4256 /**
4257  * i40e_unquiesce_vsi - Resume a given VSI
4258  * @vsi: the VSI being resumed
4259  **/
4260 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4261 {
4262         if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state))
4263                 return;
4264
4265         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
4266         if (vsi->netdev && netif_running(vsi->netdev))
4267                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4268         else
4269                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4270 }
4271
4272 /**
4273  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4274  * @pf: the PF
4275  **/
4276 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4277 {
4278         int v;
4279
4280         for (v = 0; v < pf->num_alloc_vsi; v++) {
4281                 if (pf->vsi[v])
4282                         i40e_quiesce_vsi(pf->vsi[v]);
4283         }
4284 }
4285
4286 /**
4287  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4288  * @pf: the PF
4289  **/
4290 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4291 {
4292         int v;
4293
4294         for (v = 0; v < pf->num_alloc_vsi; v++) {
4295                 if (pf->vsi[v])
4296                         i40e_unquiesce_vsi(pf->vsi[v]);
4297         }
4298 }
4299
4300 #ifdef CONFIG_I40E_DCB
4301 /**
4302  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4303  * @vsi: the VSI being configured
4304  *
4305  * This function waits for the given VSI's queues to be disabled.
4306  **/
4307 static int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4308 {
4309         struct i40e_pf *pf = vsi->back;
4310         int i, pf_q, ret;
4311
4312         pf_q = vsi->base_queue;
4313         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4314                 /* Check and wait for the disable status of the queue */
4315                 ret = i40e_pf_txq_wait(pf, pf_q, false);
4316                 if (ret) {
4317                         dev_info(&pf->pdev->dev,
4318                                  "VSI seid %d Tx ring %d disable timeout\n",
4319                                  vsi->seid, pf_q);
4320                         return ret;
4321                 }
4322         }
4323
4324         pf_q = vsi->base_queue;
4325         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4326                 /* Check and wait for the disable status of the queue */
4327                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4328                 if (ret) {
4329                         dev_info(&pf->pdev->dev,
4330                                  "VSI seid %d Rx ring %d disable timeout\n",
4331                                  vsi->seid, pf_q);
4332                         return ret;
4333                 }
4334         }
4335
4336         return 0;
4337 }
4338
4339 /**
4340  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
4341  * @pf: the PF
4342  *
4343  * This function waits for the queues to be in disabled state for all the
4344  * VSIs that are managed by this PF.
4345  **/
4346 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4347 {
4348         int v, ret = 0;
4349
4350         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4351                 /* No need to wait for FCoE VSI queues */
4352                 if (pf->vsi[v] && pf->vsi[v]->type != I40E_VSI_FCOE) {
4353                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
4354                         if (ret)
4355                                 break;
4356                 }
4357         }
4358
4359         return ret;
4360 }
4361
4362 #endif
4363
4364 /**
4365  * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue
4366  * @q_idx: TX queue number
4367  * @vsi: Pointer to VSI struct
4368  *
4369  * This function checks specified queue for given VSI. Detects hung condition.
4370  * Sets hung bit since it is two step process. Before next run of service task
4371  * if napi_poll runs, it reset 'hung' bit for respective q_vector. If not,
4372  * hung condition remain unchanged and during subsequent run, this function
4373  * issues SW interrupt to recover from hung condition.
4374  **/
4375 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi)
4376 {
4377         struct i40e_ring *tx_ring = NULL;
4378         struct i40e_pf  *pf;
4379         u32 head, val, tx_pending_hw;
4380         int i;
4381
4382         pf = vsi->back;
4383
4384         /* now that we have an index, find the tx_ring struct */
4385         for (i = 0; i < vsi->num_queue_pairs; i++) {
4386                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
4387                         if (q_idx == vsi->tx_rings[i]->queue_index) {
4388                                 tx_ring = vsi->tx_rings[i];
4389                                 break;
4390                         }
4391                 }
4392         }
4393
4394         if (!tx_ring)
4395                 return;
4396
4397         /* Read interrupt register */
4398         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4399                 val = rd32(&pf->hw,
4400                            I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
4401                                                tx_ring->vsi->base_vector - 1));
4402         else
4403                 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
4404
4405         head = i40e_get_head(tx_ring);
4406
4407         tx_pending_hw = i40e_get_tx_pending(tx_ring, false);
4408
4409         /* HW is done executing descriptors, updated HEAD write back,
4410          * but SW hasn't processed those descriptors. If interrupt is
4411          * not generated from this point ON, it could result into
4412          * dev_watchdog detecting timeout on those netdev_queue,
4413          * hence proactively trigger SW interrupt.
4414          */
4415         if (tx_pending_hw && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) {
4416                 /* NAPI Poll didn't run and clear since it was set */
4417                 if (test_and_clear_bit(I40E_Q_VECTOR_HUNG_DETECT,
4418                                        &tx_ring->q_vector->hung_detected)) {
4419                         netdev_info(vsi->netdev, "VSI_seid %d, Hung TX queue %d, tx_pending_hw: %d, NTC:0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x\n",
4420                                     vsi->seid, q_idx, tx_pending_hw,
4421                                     tx_ring->next_to_clean, head,
4422                                     tx_ring->next_to_use,
4423                                     readl(tx_ring->tail));
4424                         netdev_info(vsi->netdev, "VSI_seid %d, Issuing force_wb for TX queue %d, Interrupt Reg: 0x%x\n",
4425                                     vsi->seid, q_idx, val);
4426                         i40e_force_wb(vsi, tx_ring->q_vector);
4427                 } else {
4428                         /* First Chance - detected possible hung */
4429                         set_bit(I40E_Q_VECTOR_HUNG_DETECT,
4430                                 &tx_ring->q_vector->hung_detected);
4431                 }
4432         }
4433
4434         /* This is the case where we have interrupts missing,
4435          * so the tx_pending in HW will most likely be 0, but we
4436          * will have tx_pending in SW since the WB happened but the
4437          * interrupt got lost.
4438          */
4439         if ((!tx_pending_hw) && i40e_get_tx_pending(tx_ring, true) &&
4440             (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) {
4441                 if (napi_reschedule(&tx_ring->q_vector->napi))
4442                         tx_ring->tx_stats.tx_lost_interrupt++;
4443         }
4444 }
4445
4446 /**
4447  * i40e_detect_recover_hung - Function to detect and recover hung_queues
4448  * @pf:  pointer to PF struct
4449  *
4450  * LAN VSI has netdev and netdev has TX queues. This function is to check
4451  * each of those TX queues if they are hung, trigger recovery by issuing
4452  * SW interrupt.
4453  **/
4454 static void i40e_detect_recover_hung(struct i40e_pf *pf)
4455 {
4456         struct net_device *netdev;
4457         struct i40e_vsi *vsi;
4458         int i;
4459
4460         /* Only for LAN VSI */
4461         vsi = pf->vsi[pf->lan_vsi];
4462
4463         if (!vsi)
4464                 return;
4465
4466         /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */
4467         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
4468             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
4469                 return;
4470
4471         /* Make sure type is MAIN VSI */
4472         if (vsi->type != I40E_VSI_MAIN)
4473                 return;
4474
4475         netdev = vsi->netdev;
4476         if (!netdev)
4477                 return;
4478
4479         /* Bail out if netif_carrier is not OK */
4480         if (!netif_carrier_ok(netdev))
4481                 return;
4482
4483         /* Go thru' TX queues for netdev */
4484         for (i = 0; i < netdev->num_tx_queues; i++) {
4485                 struct netdev_queue *q;
4486
4487                 q = netdev_get_tx_queue(netdev, i);
4488                 if (q)
4489                         i40e_detect_recover_hung_queue(i, vsi);
4490         }
4491 }
4492
4493 /**
4494  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
4495  * @pf: pointer to PF
4496  *
4497  * Get TC map for ISCSI PF type that will include iSCSI TC
4498  * and LAN TC.
4499  **/
4500 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
4501 {
4502         struct i40e_dcb_app_priority_table app;
4503         struct i40e_hw *hw = &pf->hw;
4504         u8 enabled_tc = 1; /* TC0 is always enabled */
4505         u8 tc, i;
4506         /* Get the iSCSI APP TLV */
4507         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4508
4509         for (i = 0; i < dcbcfg->numapps; i++) {
4510                 app = dcbcfg->app[i];
4511                 if (app.selector == I40E_APP_SEL_TCPIP &&
4512                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
4513                         tc = dcbcfg->etscfg.prioritytable[app.priority];
4514                         enabled_tc |= BIT(tc);
4515                         break;
4516                 }
4517         }
4518
4519         return enabled_tc;
4520 }
4521
4522 /**
4523  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
4524  * @dcbcfg: the corresponding DCBx configuration structure
4525  *
4526  * Return the number of TCs from given DCBx configuration
4527  **/
4528 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
4529 {
4530         u8 num_tc = 0;
4531         int i;
4532
4533         /* Scan the ETS Config Priority Table to find
4534          * traffic class enabled for a given priority
4535          * and use the traffic class index to get the
4536          * number of traffic classes enabled
4537          */
4538         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
4539                 if (dcbcfg->etscfg.prioritytable[i] > num_tc)
4540                         num_tc = dcbcfg->etscfg.prioritytable[i];
4541         }
4542
4543         /* Traffic class index starts from zero so
4544          * increment to return the actual count
4545          */
4546         return num_tc + 1;
4547 }
4548
4549 /**
4550  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
4551  * @dcbcfg: the corresponding DCBx configuration structure
4552  *
4553  * Query the current DCB configuration and return the number of
4554  * traffic classes enabled from the given DCBX config
4555  **/
4556 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
4557 {
4558         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
4559         u8 enabled_tc = 1;
4560         u8 i;
4561
4562         for (i = 0; i < num_tc; i++)
4563                 enabled_tc |= BIT(i);
4564
4565         return enabled_tc;
4566 }
4567
4568 /**
4569  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
4570  * @pf: PF being queried
4571  *
4572  * Return number of traffic classes enabled for the given PF
4573  **/
4574 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
4575 {
4576         struct i40e_hw *hw = &pf->hw;
4577         u8 i, enabled_tc;
4578         u8 num_tc = 0;
4579         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4580
4581         /* If DCB is not enabled then always in single TC */
4582         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4583                 return 1;
4584
4585         /* SFP mode will be enabled for all TCs on port */
4586         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4587                 return i40e_dcb_get_num_tc(dcbcfg);
4588
4589         /* MFP mode return count of enabled TCs for this PF */
4590         if (pf->hw.func_caps.iscsi)
4591                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
4592         else
4593                 return 1; /* Only TC0 */
4594
4595         /* At least have TC0 */
4596         enabled_tc = (enabled_tc ? enabled_tc : 0x1);
4597         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4598                 if (enabled_tc & BIT(i))
4599                         num_tc++;
4600         }
4601         return num_tc;
4602 }
4603
4604 /**
4605  * i40e_pf_get_default_tc - Get bitmap for first enabled TC
4606  * @pf: PF being queried
4607  *
4608  * Return a bitmap for first enabled traffic class for this PF.
4609  **/
4610 static u8 i40e_pf_get_default_tc(struct i40e_pf *pf)
4611 {
4612         u8 enabled_tc = pf->hw.func_caps.enabled_tcmap;
4613         u8 i = 0;
4614
4615         if (!enabled_tc)
4616                 return 0x1; /* TC0 */
4617
4618         /* Find the first enabled TC */
4619         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4620                 if (enabled_tc & BIT(i))
4621                         break;
4622         }
4623
4624         return BIT(i);
4625 }
4626
4627 /**
4628  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
4629  * @pf: PF being queried
4630  *
4631  * Return a bitmap for enabled traffic classes for this PF.
4632  **/
4633 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
4634 {
4635         /* If DCB is not enabled for this PF then just return default TC */
4636         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4637                 return i40e_pf_get_default_tc(pf);
4638
4639         /* SFP mode we want PF to be enabled for all TCs */
4640         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4641                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
4642
4643         /* MFP enabled and iSCSI PF type */
4644         if (pf->hw.func_caps.iscsi)
4645                 return i40e_get_iscsi_tc_map(pf);
4646         else
4647                 return i40e_pf_get_default_tc(pf);
4648 }
4649
4650 /**
4651  * i40e_vsi_get_bw_info - Query VSI BW Information
4652  * @vsi: the VSI being queried
4653  *
4654  * Returns 0 on success, negative value on failure
4655  **/
4656 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
4657 {
4658         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
4659         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
4660         struct i40e_pf *pf = vsi->back;
4661         struct i40e_hw *hw = &pf->hw;
4662         i40e_status ret;
4663         u32 tc_bw_max;
4664         int i;
4665
4666         /* Get the VSI level BW configuration */
4667         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4668         if (ret) {
4669                 dev_info(&pf->pdev->dev,
4670                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
4671                          i40e_stat_str(&pf->hw, ret),
4672                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4673                 return -EINVAL;
4674         }
4675
4676         /* Get the VSI level BW configuration per TC */
4677         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
4678                                                NULL);
4679         if (ret) {
4680                 dev_info(&pf->pdev->dev,
4681                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
4682                          i40e_stat_str(&pf->hw, ret),
4683                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4684                 return -EINVAL;
4685         }
4686
4687         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
4688                 dev_info(&pf->pdev->dev,
4689                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
4690                          bw_config.tc_valid_bits,
4691                          bw_ets_config.tc_valid_bits);
4692                 /* Still continuing */
4693         }
4694
4695         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
4696         vsi->bw_max_quanta = bw_config.max_bw;
4697         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
4698                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
4699         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4700                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
4701                 vsi->bw_ets_limit_credits[i] =
4702                                         le16_to_cpu(bw_ets_config.credits[i]);
4703                 /* 3 bits out of 4 for each TC */
4704                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
4705         }
4706
4707         return 0;
4708 }
4709
4710 /**
4711  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
4712  * @vsi: the VSI being configured
4713  * @enabled_tc: TC bitmap
4714  * @bw_credits: BW shared credits per TC
4715  *
4716  * Returns 0 on success, negative value on failure
4717  **/
4718 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
4719                                        u8 *bw_share)
4720 {
4721         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
4722         i40e_status ret;
4723         int i;
4724
4725         bw_data.tc_valid_bits = enabled_tc;
4726         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4727                 bw_data.tc_bw_credits[i] = bw_share[i];
4728
4729         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
4730                                        NULL);
4731         if (ret) {
4732                 dev_info(&vsi->back->pdev->dev,
4733                          "AQ command Config VSI BW allocation per TC failed = %d\n",
4734                          vsi->back->hw.aq.asq_last_status);
4735                 return -EINVAL;
4736         }
4737
4738         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4739                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
4740
4741         return 0;
4742 }
4743
4744 /**
4745  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
4746  * @vsi: the VSI being configured
4747  * @enabled_tc: TC map to be enabled
4748  *
4749  **/
4750 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
4751 {
4752         struct net_device *netdev = vsi->netdev;
4753         struct i40e_pf *pf = vsi->back;
4754         struct i40e_hw *hw = &pf->hw;
4755         u8 netdev_tc = 0;
4756         int i;
4757         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4758
4759         if (!netdev)
4760                 return;
4761
4762         if (!enabled_tc) {
4763                 netdev_reset_tc(netdev);
4764                 return;
4765         }
4766
4767         /* Set up actual enabled TCs on the VSI */
4768         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
4769                 return;
4770
4771         /* set per TC queues for the VSI */
4772         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4773                 /* Only set TC queues for enabled tcs
4774                  *
4775                  * e.g. For a VSI that has TC0 and TC3 enabled the
4776                  * enabled_tc bitmap would be 0x00001001; the driver
4777                  * will set the numtc for netdev as 2 that will be
4778                  * referenced by the netdev layer as TC 0 and 1.
4779                  */
4780                 if (vsi->tc_config.enabled_tc & BIT(i))
4781                         netdev_set_tc_queue(netdev,
4782                                         vsi->tc_config.tc_info[i].netdev_tc,
4783                                         vsi->tc_config.tc_info[i].qcount,
4784                                         vsi->tc_config.tc_info[i].qoffset);
4785         }
4786
4787         /* Assign UP2TC map for the VSI */
4788         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
4789                 /* Get the actual TC# for the UP */
4790                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
4791                 /* Get the mapped netdev TC# for the UP */
4792                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
4793                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
4794         }
4795 }
4796
4797 /**
4798  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
4799  * @vsi: the VSI being configured
4800  * @ctxt: the ctxt buffer returned from AQ VSI update param command
4801  **/
4802 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
4803                                       struct i40e_vsi_context *ctxt)
4804 {
4805         /* copy just the sections touched not the entire info
4806          * since not all sections are valid as returned by
4807          * update vsi params
4808          */
4809         vsi->info.mapping_flags = ctxt->info.mapping_flags;
4810         memcpy(&vsi->info.queue_mapping,
4811                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
4812         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
4813                sizeof(vsi->info.tc_mapping));
4814 }
4815
4816 /**
4817  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
4818  * @vsi: VSI to be configured
4819  * @enabled_tc: TC bitmap
4820  *
4821  * This configures a particular VSI for TCs that are mapped to the
4822  * given TC bitmap. It uses default bandwidth share for TCs across
4823  * VSIs to configure TC for a particular VSI.
4824  *
4825  * NOTE:
4826  * It is expected that the VSI queues have been quisced before calling
4827  * this function.
4828  **/
4829 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
4830 {
4831         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
4832         struct i40e_vsi_context ctxt;
4833         int ret = 0;
4834         int i;
4835
4836         /* Check if enabled_tc is same as existing or new TCs */
4837         if (vsi->tc_config.enabled_tc == enabled_tc)
4838                 return ret;
4839
4840         /* Enable ETS TCs with equal BW Share for now across all VSIs */
4841         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4842                 if (enabled_tc & BIT(i))
4843                         bw_share[i] = 1;
4844         }
4845
4846         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
4847         if (ret) {
4848                 dev_info(&vsi->back->pdev->dev,
4849                          "Failed configuring TC map %d for VSI %d\n",
4850                          enabled_tc, vsi->seid);
4851                 goto out;
4852         }
4853
4854         /* Update Queue Pairs Mapping for currently enabled UPs */
4855         ctxt.seid = vsi->seid;
4856         ctxt.pf_num = vsi->back->hw.pf_id;
4857         ctxt.vf_num = 0;
4858         ctxt.uplink_seid = vsi->uplink_seid;
4859         ctxt.info = vsi->info;
4860         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
4861
4862         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
4863                 ctxt.info.valid_sections |=
4864                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
4865                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
4866         }
4867
4868         /* Update the VSI after updating the VSI queue-mapping information */
4869         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
4870         if (ret) {
4871                 dev_info(&vsi->back->pdev->dev,
4872                          "Update vsi tc config failed, err %s aq_err %s\n",
4873                          i40e_stat_str(&vsi->back->hw, ret),
4874                          i40e_aq_str(&vsi->back->hw,
4875                                      vsi->back->hw.aq.asq_last_status));
4876                 goto out;
4877         }
4878         /* update the local VSI info with updated queue map */
4879         i40e_vsi_update_queue_map(vsi, &ctxt);
4880         vsi->info.valid_sections = 0;
4881
4882         /* Update current VSI BW information */
4883         ret = i40e_vsi_get_bw_info(vsi);
4884         if (ret) {
4885                 dev_info(&vsi->back->pdev->dev,
4886                          "Failed updating vsi bw info, err %s aq_err %s\n",
4887                          i40e_stat_str(&vsi->back->hw, ret),
4888                          i40e_aq_str(&vsi->back->hw,
4889                                      vsi->back->hw.aq.asq_last_status));
4890                 goto out;
4891         }
4892
4893         /* Update the netdev TC setup */
4894         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
4895 out:
4896         return ret;
4897 }
4898
4899 /**
4900  * i40e_veb_config_tc - Configure TCs for given VEB
4901  * @veb: given VEB
4902  * @enabled_tc: TC bitmap
4903  *
4904  * Configures given TC bitmap for VEB (switching) element
4905  **/
4906 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
4907 {
4908         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
4909         struct i40e_pf *pf = veb->pf;
4910         int ret = 0;
4911         int i;
4912
4913         /* No TCs or already enabled TCs just return */
4914         if (!enabled_tc || veb->enabled_tc == enabled_tc)
4915                 return ret;
4916
4917         bw_data.tc_valid_bits = enabled_tc;
4918         /* bw_data.absolute_credits is not set (relative) */
4919
4920         /* Enable ETS TCs with equal BW Share for now */
4921         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4922                 if (enabled_tc & BIT(i))
4923                         bw_data.tc_bw_share_credits[i] = 1;
4924         }
4925
4926         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
4927                                                    &bw_data, NULL);
4928         if (ret) {
4929                 dev_info(&pf->pdev->dev,
4930                          "VEB bw config failed, err %s aq_err %s\n",
4931                          i40e_stat_str(&pf->hw, ret),
4932                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4933                 goto out;
4934         }
4935
4936         /* Update the BW information */
4937         ret = i40e_veb_get_bw_info(veb);
4938         if (ret) {
4939                 dev_info(&pf->pdev->dev,
4940                          "Failed getting veb bw config, err %s aq_err %s\n",
4941                          i40e_stat_str(&pf->hw, ret),
4942                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4943         }
4944
4945 out:
4946         return ret;
4947 }
4948
4949 #ifdef CONFIG_I40E_DCB
4950 /**
4951  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
4952  * @pf: PF struct
4953  *
4954  * Reconfigure VEB/VSIs on a given PF; it is assumed that
4955  * the caller would've quiesce all the VSIs before calling
4956  * this function
4957  **/
4958 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
4959 {
4960         u8 tc_map = 0;
4961         int ret;
4962         u8 v;
4963
4964         /* Enable the TCs available on PF to all VEBs */
4965         tc_map = i40e_pf_get_tc_map(pf);
4966         for (v = 0; v < I40E_MAX_VEB; v++) {
4967                 if (!pf->veb[v])
4968                         continue;
4969                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
4970                 if (ret) {
4971                         dev_info(&pf->pdev->dev,
4972                                  "Failed configuring TC for VEB seid=%d\n",
4973                                  pf->veb[v]->seid);
4974                         /* Will try to configure as many components */
4975                 }
4976         }
4977
4978         /* Update each VSI */
4979         for (v = 0; v < pf->num_alloc_vsi; v++) {
4980                 if (!pf->vsi[v])
4981                         continue;
4982
4983                 /* - Enable all TCs for the LAN VSI
4984 #ifdef I40E_FCOE
4985                  * - For FCoE VSI only enable the TC configured
4986                  *   as per the APP TLV
4987 #endif
4988                  * - For all others keep them at TC0 for now
4989                  */
4990                 if (v == pf->lan_vsi)
4991                         tc_map = i40e_pf_get_tc_map(pf);
4992                 else
4993                         tc_map = i40e_pf_get_default_tc(pf);
4994 #ifdef I40E_FCOE
4995                 if (pf->vsi[v]->type == I40E_VSI_FCOE)
4996                         tc_map = i40e_get_fcoe_tc_map(pf);
4997 #endif /* #ifdef I40E_FCOE */
4998
4999                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
5000                 if (ret) {
5001                         dev_info(&pf->pdev->dev,
5002                                  "Failed configuring TC for VSI seid=%d\n",
5003                                  pf->vsi[v]->seid);
5004                         /* Will try to configure as many components */
5005                 } else {
5006                         /* Re-configure VSI vectors based on updated TC map */
5007                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
5008                         if (pf->vsi[v]->netdev)
5009                                 i40e_dcbnl_set_all(pf->vsi[v]);
5010                 }
5011                 i40e_notify_client_of_l2_param_changes(pf->vsi[v]);
5012         }
5013 }
5014
5015 /**
5016  * i40e_resume_port_tx - Resume port Tx
5017  * @pf: PF struct
5018  *
5019  * Resume a port's Tx and issue a PF reset in case of failure to
5020  * resume.
5021  **/
5022 static int i40e_resume_port_tx(struct i40e_pf *pf)
5023 {
5024         struct i40e_hw *hw = &pf->hw;
5025         int ret;
5026
5027         ret = i40e_aq_resume_port_tx(hw, NULL);
5028         if (ret) {
5029                 dev_info(&pf->pdev->dev,
5030                          "Resume Port Tx failed, err %s aq_err %s\n",
5031                           i40e_stat_str(&pf->hw, ret),
5032                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5033                 /* Schedule PF reset to recover */
5034                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
5035                 i40e_service_event_schedule(pf);
5036         }
5037
5038         return ret;
5039 }
5040
5041 /**
5042  * i40e_init_pf_dcb - Initialize DCB configuration
5043  * @pf: PF being configured
5044  *
5045  * Query the current DCB configuration and cache it
5046  * in the hardware structure
5047  **/
5048 static int i40e_init_pf_dcb(struct i40e_pf *pf)
5049 {
5050         struct i40e_hw *hw = &pf->hw;
5051         int err = 0;
5052
5053         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */
5054         if (pf->flags & I40E_FLAG_NO_DCB_SUPPORT)
5055                 goto out;
5056
5057         /* Get the initial DCB configuration */
5058         err = i40e_init_dcb(hw);
5059         if (!err) {
5060                 /* Device/Function is not DCBX capable */
5061                 if ((!hw->func_caps.dcb) ||
5062                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
5063                         dev_info(&pf->pdev->dev,
5064                                  "DCBX offload is not supported or is disabled for this PF.\n");
5065
5066                         if (pf->flags & I40E_FLAG_MFP_ENABLED)
5067                                 goto out;
5068
5069                 } else {
5070                         /* When status is not DISABLED then DCBX in FW */
5071                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
5072                                        DCB_CAP_DCBX_VER_IEEE;
5073
5074                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
5075                         /* Enable DCB tagging only when more than one TC */
5076                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
5077                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
5078                         dev_dbg(&pf->pdev->dev,
5079                                 "DCBX offload is supported for this PF.\n");
5080                 }
5081         } else {
5082                 dev_info(&pf->pdev->dev,
5083                          "Query for DCB configuration failed, err %s aq_err %s\n",
5084                          i40e_stat_str(&pf->hw, err),
5085                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5086         }
5087
5088 out:
5089         return err;
5090 }
5091 #endif /* CONFIG_I40E_DCB */
5092 #define SPEED_SIZE 14
5093 #define FC_SIZE 8
5094 /**
5095  * i40e_print_link_message - print link up or down
5096  * @vsi: the VSI for which link needs a message
5097  */
5098 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
5099 {
5100         char *speed = "Unknown";
5101         char *fc = "Unknown";
5102
5103         if (vsi->current_isup == isup)
5104                 return;
5105         vsi->current_isup = isup;
5106         if (!isup) {
5107                 netdev_info(vsi->netdev, "NIC Link is Down\n");
5108                 return;
5109         }
5110
5111         /* Warn user if link speed on NPAR enabled partition is not at
5112          * least 10GB
5113          */
5114         if (vsi->back->hw.func_caps.npar_enable &&
5115             (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
5116              vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
5117                 netdev_warn(vsi->netdev,
5118                             "The partition detected link speed that is less than 10Gbps\n");
5119
5120         switch (vsi->back->hw.phy.link_info.link_speed) {
5121         case I40E_LINK_SPEED_40GB:
5122                 speed = "40 G";
5123                 break;
5124         case I40E_LINK_SPEED_20GB:
5125                 speed = "20 G";
5126                 break;
5127         case I40E_LINK_SPEED_10GB:
5128                 speed = "10 G";
5129                 break;
5130         case I40E_LINK_SPEED_1GB:
5131                 speed = "1000 M";
5132                 break;
5133         case I40E_LINK_SPEED_100MB:
5134                 speed = "100 M";
5135                 break;
5136         default:
5137                 break;
5138         }
5139
5140         switch (vsi->back->hw.fc.current_mode) {
5141         case I40E_FC_FULL:
5142                 fc = "RX/TX";
5143                 break;
5144         case I40E_FC_TX_PAUSE:
5145                 fc = "TX";
5146                 break;
5147         case I40E_FC_RX_PAUSE:
5148                 fc = "RX";
5149                 break;
5150         default:
5151                 fc = "None";
5152                 break;
5153         }
5154
5155         netdev_info(vsi->netdev, "NIC Link is Up %sbps Full Duplex, Flow Control: %s\n",
5156                     speed, fc);
5157 }
5158
5159 /**
5160  * i40e_up_complete - Finish the last steps of bringing up a connection
5161  * @vsi: the VSI being configured
5162  **/
5163 static int i40e_up_complete(struct i40e_vsi *vsi)
5164 {
5165         struct i40e_pf *pf = vsi->back;
5166         int err;
5167
5168         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5169                 i40e_vsi_configure_msix(vsi);
5170         else
5171                 i40e_configure_msi_and_legacy(vsi);
5172
5173         /* start rings */
5174         err = i40e_vsi_control_rings(vsi, true);
5175         if (err)
5176                 return err;
5177
5178         clear_bit(__I40E_DOWN, &vsi->state);
5179         i40e_napi_enable_all(vsi);
5180         i40e_vsi_enable_irq(vsi);
5181
5182         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
5183             (vsi->netdev)) {
5184                 i40e_print_link_message(vsi, true);
5185                 netif_tx_start_all_queues(vsi->netdev);
5186                 netif_carrier_on(vsi->netdev);
5187         } else if (vsi->netdev) {
5188                 i40e_print_link_message(vsi, false);
5189                 /* need to check for qualified module here*/
5190                 if ((pf->hw.phy.link_info.link_info &
5191                         I40E_AQ_MEDIA_AVAILABLE) &&
5192                     (!(pf->hw.phy.link_info.an_info &
5193                         I40E_AQ_QUALIFIED_MODULE)))
5194                         netdev_err(vsi->netdev,
5195                                    "the driver failed to link because an unqualified module was detected.");
5196         }
5197
5198         /* replay FDIR SB filters */
5199         if (vsi->type == I40E_VSI_FDIR) {
5200                 /* reset fd counters */
5201                 pf->fd_add_err = pf->fd_atr_cnt = 0;
5202                 if (pf->fd_tcp_rule > 0) {
5203                         pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
5204                         if (I40E_DEBUG_FD & pf->hw.debug_mask)
5205                                 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 exist\n");
5206                         pf->fd_tcp_rule = 0;
5207                 }
5208                 i40e_fdir_filter_restore(vsi);
5209         }
5210
5211         /* On the next run of the service_task, notify any clients of the new
5212          * opened netdev
5213          */
5214         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
5215         i40e_service_event_schedule(pf);
5216
5217         return 0;
5218 }
5219
5220 /**
5221  * i40e_vsi_reinit_locked - Reset the VSI
5222  * @vsi: the VSI being configured
5223  *
5224  * Rebuild the ring structs after some configuration
5225  * has changed, e.g. MTU size.
5226  **/
5227 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
5228 {
5229         struct i40e_pf *pf = vsi->back;
5230
5231         WARN_ON(in_interrupt());
5232         while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state))
5233                 usleep_range(1000, 2000);
5234         i40e_down(vsi);
5235
5236         /* Give a VF some time to respond to the reset.  The
5237          * two second wait is based upon the watchdog cycle in
5238          * the VF driver.
5239          */
5240         if (vsi->type == I40E_VSI_SRIOV)
5241                 msleep(2000);
5242         i40e_up(vsi);
5243         clear_bit(__I40E_CONFIG_BUSY, &pf->state);
5244 }
5245
5246 /**
5247  * i40e_up - Bring the connection back up after being down
5248  * @vsi: the VSI being configured
5249  **/
5250 int i40e_up(struct i40e_vsi *vsi)
5251 {
5252         int err;
5253
5254         err = i40e_vsi_configure(vsi);
5255         if (!err)
5256                 err = i40e_up_complete(vsi);
5257
5258         return err;
5259 }
5260
5261 /**
5262  * i40e_down - Shutdown the connection processing
5263  * @vsi: the VSI being stopped
5264  **/
5265 void i40e_down(struct i40e_vsi *vsi)
5266 {
5267         int i;
5268
5269         /* It is assumed that the caller of this function
5270          * sets the vsi->state __I40E_DOWN bit.
5271          */
5272         if (vsi->netdev) {
5273                 netif_carrier_off(vsi->netdev);
5274                 netif_tx_disable(vsi->netdev);
5275         }
5276         i40e_vsi_disable_irq(vsi);
5277         i40e_vsi_control_rings(vsi, false);
5278         i40e_napi_disable_all(vsi);
5279
5280         for (i = 0; i < vsi->num_queue_pairs; i++) {
5281                 i40e_clean_tx_ring(vsi->tx_rings[i]);
5282                 i40e_clean_rx_ring(vsi->rx_rings[i]);
5283         }
5284 }
5285
5286 /**
5287  * i40e_setup_tc - configure multiple traffic classes
5288  * @netdev: net device to configure
5289  * @tc: number of traffic classes to enable
5290  **/
5291 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
5292 {
5293         struct i40e_netdev_priv *np = netdev_priv(netdev);
5294         struct i40e_vsi *vsi = np->vsi;
5295         struct i40e_pf *pf = vsi->back;
5296         u8 enabled_tc = 0;
5297         int ret = -EINVAL;
5298         int i;
5299
5300         /* Check if DCB enabled to continue */
5301         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5302                 netdev_info(netdev, "DCB is not enabled for adapter\n");
5303                 goto exit;
5304         }
5305
5306         /* Check if MFP enabled */
5307         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
5308                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
5309                 goto exit;
5310         }
5311
5312         /* Check whether tc count is within enabled limit */
5313         if (tc > i40e_pf_get_num_tc(pf)) {
5314                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
5315                 goto exit;
5316         }
5317
5318         /* Generate TC map for number of tc requested */
5319         for (i = 0; i < tc; i++)
5320                 enabled_tc |= BIT(i);
5321
5322         /* Requesting same TC configuration as already enabled */
5323         if (enabled_tc == vsi->tc_config.enabled_tc)
5324                 return 0;
5325
5326         /* Quiesce VSI queues */
5327         i40e_quiesce_vsi(vsi);
5328
5329         /* Configure VSI for enabled TCs */
5330         ret = i40e_vsi_config_tc(vsi, enabled_tc);
5331         if (ret) {
5332                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
5333                             vsi->seid);
5334                 goto exit;
5335         }
5336
5337         /* Unquiesce VSI */
5338         i40e_unquiesce_vsi(vsi);
5339
5340 exit:
5341         return ret;
5342 }
5343
5344 #ifdef I40E_FCOE
5345 int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto,
5346                     struct tc_to_netdev *tc)
5347 #else
5348 static int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto,
5349                            struct tc_to_netdev *tc)
5350 #endif
5351 {
5352         if (handle != TC_H_ROOT || tc->type != TC_SETUP_MQPRIO)
5353                 return -EINVAL;
5354         return i40e_setup_tc(netdev, tc->tc);
5355 }
5356
5357 /**
5358  * i40e_open - Called when a network interface is made active
5359  * @netdev: network interface device structure
5360  *
5361  * The open entry point is called when a network interface is made
5362  * active by the system (IFF_UP).  At this point all resources needed
5363  * for transmit and receive operations are allocated, the interrupt
5364  * handler is registered with the OS, the netdev watchdog subtask is
5365  * enabled, and the stack is notified that the interface is ready.
5366  *
5367  * Returns 0 on success, negative value on failure
5368  **/
5369 int i40e_open(struct net_device *netdev)
5370 {
5371         struct i40e_netdev_priv *np = netdev_priv(netdev);
5372         struct i40e_vsi *vsi = np->vsi;
5373         struct i40e_pf *pf = vsi->back;
5374         int err;
5375
5376         /* disallow open during test or if eeprom is broken */
5377         if (test_bit(__I40E_TESTING, &pf->state) ||
5378             test_bit(__I40E_BAD_EEPROM, &pf->state))
5379                 return -EBUSY;
5380
5381         netif_carrier_off(netdev);
5382
5383         err = i40e_vsi_open(vsi);
5384         if (err)
5385                 return err;
5386
5387         /* configure global TSO hardware offload settings */
5388         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
5389                                                        TCP_FLAG_FIN) >> 16);
5390         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
5391                                                        TCP_FLAG_FIN |
5392                                                        TCP_FLAG_CWR) >> 16);
5393         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
5394
5395 #ifdef CONFIG_I40E_VXLAN
5396         vxlan_get_rx_port(netdev);
5397 #endif
5398 #ifdef CONFIG_I40E_GENEVE
5399         if (pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE)
5400                 geneve_get_rx_port(netdev);
5401 #endif
5402
5403         i40e_notify_client_of_netdev_open(vsi);
5404
5405         return 0;
5406 }
5407
5408 /**
5409  * i40e_vsi_open -
5410  * @vsi: the VSI to open
5411  *
5412  * Finish initialization of the VSI.
5413  *
5414  * Returns 0 on success, negative value on failure
5415  **/
5416 int i40e_vsi_open(struct i40e_vsi *vsi)
5417 {
5418         struct i40e_pf *pf = vsi->back;
5419         char int_name[I40E_INT_NAME_STR_LEN];
5420         int err;
5421
5422         /* allocate descriptors */
5423         err = i40e_vsi_setup_tx_resources(vsi);
5424         if (err)
5425                 goto err_setup_tx;
5426         err = i40e_vsi_setup_rx_resources(vsi);
5427         if (err)
5428                 goto err_setup_rx;
5429
5430         err = i40e_vsi_configure(vsi);
5431         if (err)
5432                 goto err_setup_rx;
5433
5434         if (vsi->netdev) {
5435                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
5436                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
5437                 err = i40e_vsi_request_irq(vsi, int_name);
5438                 if (err)
5439                         goto err_setup_rx;
5440
5441                 /* Notify the stack of the actual queue counts. */
5442                 err = netif_set_real_num_tx_queues(vsi->netdev,
5443                                                    vsi->num_queue_pairs);
5444                 if (err)
5445                         goto err_set_queues;
5446
5447                 err = netif_set_real_num_rx_queues(vsi->netdev,
5448                                                    vsi->num_queue_pairs);
5449                 if (err)
5450                         goto err_set_queues;
5451
5452         } else if (vsi->type == I40E_VSI_FDIR) {
5453                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
5454                          dev_driver_string(&pf->pdev->dev),
5455                          dev_name(&pf->pdev->dev));
5456                 err = i40e_vsi_request_irq(vsi, int_name);
5457
5458         } else {
5459                 err = -EINVAL;
5460                 goto err_setup_rx;
5461         }
5462
5463         err = i40e_up_complete(vsi);
5464         if (err)
5465                 goto err_up_complete;
5466
5467         return 0;
5468
5469 err_up_complete:
5470         i40e_down(vsi);
5471 err_set_queues:
5472         i40e_vsi_free_irq(vsi);
5473 err_setup_rx:
5474         i40e_vsi_free_rx_resources(vsi);
5475 err_setup_tx:
5476         i40e_vsi_free_tx_resources(vsi);
5477         if (vsi == pf->vsi[pf->lan_vsi])
5478                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED));
5479
5480         return err;
5481 }
5482
5483 /**
5484  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
5485  * @pf: Pointer to PF
5486  *
5487  * This function destroys the hlist where all the Flow Director
5488  * filters were saved.
5489  **/
5490 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
5491 {
5492         struct i40e_fdir_filter *filter;
5493         struct hlist_node *node2;
5494
5495         hlist_for_each_entry_safe(filter, node2,
5496                                   &pf->fdir_filter_list, fdir_node) {
5497                 hlist_del(&filter->fdir_node);
5498                 kfree(filter);
5499         }
5500         pf->fdir_pf_active_filters = 0;
5501 }
5502
5503 /**
5504  * i40e_close - Disables a network interface
5505  * @netdev: network interface device structure
5506  *
5507  * The close entry point is called when an interface is de-activated
5508  * by the OS.  The hardware is still under the driver's control, but
5509  * this netdev interface is disabled.
5510  *
5511  * Returns 0, this is not allowed to fail
5512  **/
5513 int i40e_close(struct net_device *netdev)
5514 {
5515         struct i40e_netdev_priv *np = netdev_priv(netdev);
5516         struct i40e_vsi *vsi = np->vsi;
5517
5518         i40e_vsi_close(vsi);
5519
5520         return 0;
5521 }
5522
5523 /**
5524  * i40e_do_reset - Start a PF or Core Reset sequence
5525  * @pf: board private structure
5526  * @reset_flags: which reset is requested
5527  *
5528  * The essential difference in resets is that the PF Reset
5529  * doesn't clear the packet buffers, doesn't reset the PE
5530  * firmware, and doesn't bother the other PFs on the chip.
5531  **/
5532 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags)
5533 {
5534         u32 val;
5535
5536         WARN_ON(in_interrupt());
5537
5538
5539         /* do the biggest reset indicated */
5540         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
5541
5542                 /* Request a Global Reset
5543                  *
5544                  * This will start the chip's countdown to the actual full
5545                  * chip reset event, and a warning interrupt to be sent
5546                  * to all PFs, including the requestor.  Our handler
5547                  * for the warning interrupt will deal with the shutdown
5548                  * and recovery of the switch setup.
5549                  */
5550                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
5551                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5552                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
5553                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5554
5555         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
5556
5557                 /* Request a Core Reset
5558                  *
5559                  * Same as Global Reset, except does *not* include the MAC/PHY
5560                  */
5561                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
5562                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5563                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
5564                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5565                 i40e_flush(&pf->hw);
5566
5567         } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) {
5568
5569                 /* Request a PF Reset
5570                  *
5571                  * Resets only the PF-specific registers
5572                  *
5573                  * This goes directly to the tear-down and rebuild of
5574                  * the switch, since we need to do all the recovery as
5575                  * for the Core Reset.
5576                  */
5577                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
5578                 i40e_handle_reset_warning(pf);
5579
5580         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
5581                 int v;
5582
5583                 /* Find the VSI(s) that requested a re-init */
5584                 dev_info(&pf->pdev->dev,
5585                          "VSI reinit requested\n");
5586                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5587                         struct i40e_vsi *vsi = pf->vsi[v];
5588
5589                         if (vsi != NULL &&
5590                             test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) {
5591                                 i40e_vsi_reinit_locked(pf->vsi[v]);
5592                                 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state);
5593                         }
5594                 }
5595         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
5596                 int v;
5597
5598                 /* Find the VSI(s) that needs to be brought down */
5599                 dev_info(&pf->pdev->dev, "VSI down requested\n");
5600                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5601                         struct i40e_vsi *vsi = pf->vsi[v];
5602
5603                         if (vsi != NULL &&
5604                             test_bit(__I40E_DOWN_REQUESTED, &vsi->state)) {
5605                                 set_bit(__I40E_DOWN, &vsi->state);
5606                                 i40e_down(vsi);
5607                                 clear_bit(__I40E_DOWN_REQUESTED, &vsi->state);
5608                         }
5609                 }
5610         } else {
5611                 dev_info(&pf->pdev->dev,
5612                          "bad reset request 0x%08x\n", reset_flags);
5613         }
5614 }
5615
5616 #ifdef CONFIG_I40E_DCB
5617 /**
5618  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
5619  * @pf: board private structure
5620  * @old_cfg: current DCB config
5621  * @new_cfg: new DCB config
5622  **/
5623 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
5624                             struct i40e_dcbx_config *old_cfg,
5625                             struct i40e_dcbx_config *new_cfg)
5626 {
5627         bool need_reconfig = false;
5628
5629         /* Check if ETS configuration has changed */
5630         if (memcmp(&new_cfg->etscfg,
5631                    &old_cfg->etscfg,
5632                    sizeof(new_cfg->etscfg))) {
5633                 /* If Priority Table has changed reconfig is needed */
5634                 if (memcmp(&new_cfg->etscfg.prioritytable,
5635                            &old_cfg->etscfg.prioritytable,
5636                            sizeof(new_cfg->etscfg.prioritytable))) {
5637                         need_reconfig = true;
5638                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
5639                 }
5640
5641                 if (memcmp(&new_cfg->etscfg.tcbwtable,
5642                            &old_cfg->etscfg.tcbwtable,
5643                            sizeof(new_cfg->etscfg.tcbwtable)))
5644                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
5645
5646                 if (memcmp(&new_cfg->etscfg.tsatable,
5647                            &old_cfg->etscfg.tsatable,
5648                            sizeof(new_cfg->etscfg.tsatable)))
5649                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
5650         }
5651
5652         /* Check if PFC configuration has changed */
5653         if (memcmp(&new_cfg->pfc,
5654                    &old_cfg->pfc,
5655                    sizeof(new_cfg->pfc))) {
5656                 need_reconfig = true;
5657                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
5658         }
5659
5660         /* Check if APP Table has changed */
5661         if (memcmp(&new_cfg->app,
5662                    &old_cfg->app,
5663                    sizeof(new_cfg->app))) {
5664                 need_reconfig = true;
5665                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
5666         }
5667
5668         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
5669         return need_reconfig;
5670 }
5671
5672 /**
5673  * i40e_handle_lldp_event - Handle LLDP Change MIB event
5674  * @pf: board private structure
5675  * @e: event info posted on ARQ
5676  **/
5677 static int i40e_handle_lldp_event(struct i40e_pf *pf,
5678                                   struct i40e_arq_event_info *e)
5679 {
5680         struct i40e_aqc_lldp_get_mib *mib =
5681                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
5682         struct i40e_hw *hw = &pf->hw;
5683         struct i40e_dcbx_config tmp_dcbx_cfg;
5684         bool need_reconfig = false;
5685         int ret = 0;
5686         u8 type;
5687
5688         /* Not DCB capable or capability disabled */
5689         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
5690                 return ret;
5691
5692         /* Ignore if event is not for Nearest Bridge */
5693         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
5694                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
5695         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
5696         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
5697                 return ret;
5698
5699         /* Check MIB Type and return if event for Remote MIB update */
5700         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
5701         dev_dbg(&pf->pdev->dev,
5702                 "LLDP event mib type %s\n", type ? "remote" : "local");
5703         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
5704                 /* Update the remote cached instance and return */
5705                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
5706                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
5707                                 &hw->remote_dcbx_config);
5708                 goto exit;
5709         }
5710
5711         /* Store the old configuration */
5712         tmp_dcbx_cfg = hw->local_dcbx_config;
5713
5714         /* Reset the old DCBx configuration data */
5715         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
5716         /* Get updated DCBX data from firmware */
5717         ret = i40e_get_dcb_config(&pf->hw);
5718         if (ret) {
5719                 dev_info(&pf->pdev->dev,
5720                          "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
5721                          i40e_stat_str(&pf->hw, ret),
5722                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5723                 goto exit;
5724         }
5725
5726         /* No change detected in DCBX configs */
5727         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
5728                     sizeof(tmp_dcbx_cfg))) {
5729                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
5730                 goto exit;
5731         }
5732
5733         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
5734                                                &hw->local_dcbx_config);
5735
5736         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
5737
5738         if (!need_reconfig)
5739                 goto exit;
5740
5741         /* Enable DCB tagging only when more than one TC */
5742         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
5743                 pf->flags |= I40E_FLAG_DCB_ENABLED;
5744         else
5745                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
5746
5747         set_bit(__I40E_PORT_TX_SUSPENDED, &pf->state);
5748         /* Reconfiguration needed quiesce all VSIs */
5749         i40e_pf_quiesce_all_vsi(pf);
5750
5751         /* Changes in configuration update VEB/VSI */
5752         i40e_dcb_reconfigure(pf);
5753
5754         ret = i40e_resume_port_tx(pf);
5755
5756         clear_bit(__I40E_PORT_TX_SUSPENDED, &pf->state);
5757         /* In case of error no point in resuming VSIs */
5758         if (ret)
5759                 goto exit;
5760
5761         /* Wait for the PF's queues to be disabled */
5762         ret = i40e_pf_wait_queues_disabled(pf);
5763         if (ret) {
5764                 /* Schedule PF reset to recover */
5765                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
5766                 i40e_service_event_schedule(pf);
5767         } else {
5768                 i40e_pf_unquiesce_all_vsi(pf);
5769         }
5770
5771 exit:
5772         return ret;
5773 }
5774 #endif /* CONFIG_I40E_DCB */
5775
5776 /**
5777  * i40e_do_reset_safe - Protected reset path for userland calls.
5778  * @pf: board private structure
5779  * @reset_flags: which reset is requested
5780  *
5781  **/
5782 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
5783 {
5784         rtnl_lock();
5785         i40e_do_reset(pf, reset_flags);
5786         rtnl_unlock();
5787 }
5788
5789 /**
5790  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
5791  * @pf: board private structure
5792  * @e: event info posted on ARQ
5793  *
5794  * Handler for LAN Queue Overflow Event generated by the firmware for PF
5795  * and VF queues
5796  **/
5797 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
5798                                            struct i40e_arq_event_info *e)
5799 {
5800         struct i40e_aqc_lan_overflow *data =
5801                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
5802         u32 queue = le32_to_cpu(data->prtdcb_rupto);
5803         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
5804         struct i40e_hw *hw = &pf->hw;
5805         struct i40e_vf *vf;
5806         u16 vf_id;
5807
5808         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
5809                 queue, qtx_ctl);
5810
5811         /* Queue belongs to VF, find the VF and issue VF reset */
5812         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
5813             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
5814                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
5815                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
5816                 vf_id -= hw->func_caps.vf_base_id;
5817                 vf = &pf->vf[vf_id];
5818                 i40e_vc_notify_vf_reset(vf);
5819                 /* Allow VF to process pending reset notification */
5820                 msleep(20);
5821                 i40e_reset_vf(vf, false);
5822         }
5823 }
5824
5825 /**
5826  * i40e_service_event_complete - Finish up the service event
5827  * @pf: board private structure
5828  **/
5829 static void i40e_service_event_complete(struct i40e_pf *pf)
5830 {
5831         WARN_ON(!test_bit(__I40E_SERVICE_SCHED, &pf->state));
5832
5833         /* flush memory to make sure state is correct before next watchog */
5834         smp_mb__before_atomic();
5835         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
5836 }
5837
5838 /**
5839  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
5840  * @pf: board private structure
5841  **/
5842 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
5843 {
5844         u32 val, fcnt_prog;
5845
5846         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
5847         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
5848         return fcnt_prog;
5849 }
5850
5851 /**
5852  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
5853  * @pf: board private structure
5854  **/
5855 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
5856 {
5857         u32 val, fcnt_prog;
5858
5859         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
5860         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
5861                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
5862                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
5863         return fcnt_prog;
5864 }
5865
5866 /**
5867  * i40e_get_global_fd_count - Get total FD filters programmed on device
5868  * @pf: board private structure
5869  **/
5870 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
5871 {
5872         u32 val, fcnt_prog;
5873
5874         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
5875         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
5876                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
5877                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
5878         return fcnt_prog;
5879 }
5880
5881 /**
5882  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
5883  * @pf: board private structure
5884  **/
5885 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
5886 {
5887         struct i40e_fdir_filter *filter;
5888         u32 fcnt_prog, fcnt_avail;
5889         struct hlist_node *node;
5890
5891         if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state))
5892                 return;
5893
5894         /* Check if, FD SB or ATR was auto disabled and if there is enough room
5895          * to re-enable
5896          */
5897         fcnt_prog = i40e_get_global_fd_count(pf);
5898         fcnt_avail = pf->fdir_pf_filter_count;
5899         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
5900             (pf->fd_add_err == 0) ||
5901             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) {
5902                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
5903                     (pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) {
5904                         pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED;
5905                         if (I40E_DEBUG_FD & pf->hw.debug_mask)
5906                                 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
5907                 }
5908         }
5909         /* Wait for some more space to be available to turn on ATR */
5910         if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) {
5911                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
5912                     (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) {
5913                         pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
5914                         if (I40E_DEBUG_FD & pf->hw.debug_mask)
5915                                 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table now\n");
5916                 }
5917         }
5918
5919         /* if hw had a problem adding a filter, delete it */
5920         if (pf->fd_inv > 0) {
5921                 hlist_for_each_entry_safe(filter, node,
5922                                           &pf->fdir_filter_list, fdir_node) {
5923                         if (filter->fd_id == pf->fd_inv) {
5924                                 hlist_del(&filter->fdir_node);
5925                                 kfree(filter);
5926                                 pf->fdir_pf_active_filters--;
5927                         }
5928                 }
5929         }
5930 }
5931
5932 #define I40E_MIN_FD_FLUSH_INTERVAL 10
5933 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
5934 /**
5935  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
5936  * @pf: board private structure
5937  **/
5938 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
5939 {
5940         unsigned long min_flush_time;
5941         int flush_wait_retry = 50;
5942         bool disable_atr = false;
5943         int fd_room;
5944         int reg;
5945
5946         if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)))
5947                 return;
5948
5949         if (!time_after(jiffies, pf->fd_flush_timestamp +
5950                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
5951                 return;
5952
5953         /* If the flush is happening too quick and we have mostly SB rules we
5954          * should not re-enable ATR for some time.
5955          */
5956         min_flush_time = pf->fd_flush_timestamp +
5957                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
5958         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
5959
5960         if (!(time_after(jiffies, min_flush_time)) &&
5961             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
5962                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
5963                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
5964                 disable_atr = true;
5965         }
5966
5967         pf->fd_flush_timestamp = jiffies;
5968         pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
5969         /* flush all filters */
5970         wr32(&pf->hw, I40E_PFQF_CTL_1,
5971              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
5972         i40e_flush(&pf->hw);
5973         pf->fd_flush_cnt++;
5974         pf->fd_add_err = 0;
5975         do {
5976                 /* Check FD flush status every 5-6msec */
5977                 usleep_range(5000, 6000);
5978                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
5979                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
5980                         break;
5981         } while (flush_wait_retry--);
5982         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
5983                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
5984         } else {
5985                 /* replay sideband filters */
5986                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
5987                 if (!disable_atr)
5988                         pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
5989                 clear_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state);
5990                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
5991                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
5992         }
5993
5994 }
5995
5996 /**
5997  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
5998  * @pf: board private structure
5999  **/
6000 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
6001 {
6002         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
6003 }
6004
6005 /* We can see up to 256 filter programming desc in transit if the filters are
6006  * being applied really fast; before we see the first
6007  * filter miss error on Rx queue 0. Accumulating enough error messages before
6008  * reacting will make sure we don't cause flush too often.
6009  */
6010 #define I40E_MAX_FD_PROGRAM_ERROR 256
6011
6012 /**
6013  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
6014  * @pf: board private structure
6015  **/
6016 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
6017 {
6018
6019         /* if interface is down do nothing */
6020         if (test_bit(__I40E_DOWN, &pf->state))
6021                 return;
6022
6023         if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)))
6024                 return;
6025
6026         if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state))
6027                 i40e_fdir_flush_and_replay(pf);
6028
6029         i40e_fdir_check_and_reenable(pf);
6030
6031 }
6032
6033 /**
6034  * i40e_vsi_link_event - notify VSI of a link event
6035  * @vsi: vsi to be notified
6036  * @link_up: link up or down
6037  **/
6038 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
6039 {
6040         if (!vsi || test_bit(__I40E_DOWN, &vsi->state))
6041                 return;
6042
6043         switch (vsi->type) {
6044         case I40E_VSI_MAIN:
6045 #ifdef I40E_FCOE
6046         case I40E_VSI_FCOE:
6047 #endif
6048                 if (!vsi->netdev || !vsi->netdev_registered)
6049                         break;
6050
6051                 if (link_up) {
6052                         netif_carrier_on(vsi->netdev);
6053                         netif_tx_wake_all_queues(vsi->netdev);
6054                 } else {
6055                         netif_carrier_off(vsi->netdev);
6056                         netif_tx_stop_all_queues(vsi->netdev);
6057                 }
6058                 break;
6059
6060         case I40E_VSI_SRIOV:
6061         case I40E_VSI_VMDQ2:
6062         case I40E_VSI_CTRL:
6063         case I40E_VSI_IWARP:
6064         case I40E_VSI_MIRROR:
6065         default:
6066                 /* there is no notification for other VSIs */
6067                 break;
6068         }
6069 }
6070
6071 /**
6072  * i40e_veb_link_event - notify elements on the veb of a link event
6073  * @veb: veb to be notified
6074  * @link_up: link up or down
6075  **/
6076 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
6077 {
6078         struct i40e_pf *pf;
6079         int i;
6080
6081         if (!veb || !veb->pf)
6082                 return;
6083         pf = veb->pf;
6084
6085         /* depth first... */
6086         for (i = 0; i < I40E_MAX_VEB; i++)
6087                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
6088                         i40e_veb_link_event(pf->veb[i], link_up);
6089
6090         /* ... now the local VSIs */
6091         for (i = 0; i < pf->num_alloc_vsi; i++)
6092                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
6093                         i40e_vsi_link_event(pf->vsi[i], link_up);
6094 }
6095
6096 /**
6097  * i40e_link_event - Update netif_carrier status
6098  * @pf: board private structure
6099  **/
6100 static void i40e_link_event(struct i40e_pf *pf)
6101 {
6102         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6103         u8 new_link_speed, old_link_speed;
6104         i40e_status status;
6105         bool new_link, old_link;
6106
6107         /* save off old link status information */
6108         pf->hw.phy.link_info_old = pf->hw.phy.link_info;
6109
6110         /* set this to force the get_link_status call to refresh state */
6111         pf->hw.phy.get_link_info = true;
6112
6113         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
6114
6115         status = i40e_get_link_status(&pf->hw, &new_link);
6116         if (status) {
6117                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
6118                         status);
6119                 return;
6120         }
6121
6122         old_link_speed = pf->hw.phy.link_info_old.link_speed;
6123         new_link_speed = pf->hw.phy.link_info.link_speed;
6124
6125         if (new_link == old_link &&
6126             new_link_speed == old_link_speed &&
6127             (test_bit(__I40E_DOWN, &vsi->state) ||
6128              new_link == netif_carrier_ok(vsi->netdev)))
6129                 return;
6130
6131         if (!test_bit(__I40E_DOWN, &vsi->state))
6132                 i40e_print_link_message(vsi, new_link);
6133
6134         /* Notify the base of the switch tree connected to
6135          * the link.  Floating VEBs are not notified.
6136          */
6137         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
6138                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
6139         else
6140                 i40e_vsi_link_event(vsi, new_link);
6141
6142         if (pf->vf)
6143                 i40e_vc_notify_link_state(pf);
6144
6145         if (pf->flags & I40E_FLAG_PTP)
6146                 i40e_ptp_set_increment(pf);
6147 }
6148
6149 /**
6150  * i40e_watchdog_subtask - periodic checks not using event driven response
6151  * @pf: board private structure
6152  **/
6153 static void i40e_watchdog_subtask(struct i40e_pf *pf)
6154 {
6155         int i;
6156
6157         /* if interface is down do nothing */
6158         if (test_bit(__I40E_DOWN, &pf->state) ||
6159             test_bit(__I40E_CONFIG_BUSY, &pf->state))
6160                 return;
6161
6162         /* make sure we don't do these things too often */
6163         if (time_before(jiffies, (pf->service_timer_previous +
6164                                   pf->service_timer_period)))
6165                 return;
6166         pf->service_timer_previous = jiffies;
6167
6168         if (pf->flags & I40E_FLAG_LINK_POLLING_ENABLED)
6169                 i40e_link_event(pf);
6170
6171         /* Update the stats for active netdevs so the network stack
6172          * can look at updated numbers whenever it cares to
6173          */
6174         for (i = 0; i < pf->num_alloc_vsi; i++)
6175                 if (pf->vsi[i] && pf->vsi[i]->netdev)
6176                         i40e_update_stats(pf->vsi[i]);
6177
6178         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
6179                 /* Update the stats for the active switching components */
6180                 for (i = 0; i < I40E_MAX_VEB; i++)
6181                         if (pf->veb[i])
6182                                 i40e_update_veb_stats(pf->veb[i]);
6183         }
6184
6185         i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]);
6186 }
6187
6188 /**
6189  * i40e_reset_subtask - Set up for resetting the device and driver
6190  * @pf: board private structure
6191  **/
6192 static void i40e_reset_subtask(struct i40e_pf *pf)
6193 {
6194         u32 reset_flags = 0;
6195
6196         rtnl_lock();
6197         if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) {
6198                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
6199                 clear_bit(__I40E_REINIT_REQUESTED, &pf->state);
6200         }
6201         if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) {
6202                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
6203                 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
6204         }
6205         if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) {
6206                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
6207                 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
6208         }
6209         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) {
6210                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
6211                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
6212         }
6213         if (test_bit(__I40E_DOWN_REQUESTED, &pf->state)) {
6214                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
6215                 clear_bit(__I40E_DOWN_REQUESTED, &pf->state);
6216         }
6217
6218         /* If there's a recovery already waiting, it takes
6219          * precedence before starting a new reset sequence.
6220          */
6221         if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) {
6222                 i40e_handle_reset_warning(pf);
6223                 goto unlock;
6224         }
6225
6226         /* If we're already down or resetting, just bail */
6227         if (reset_flags &&
6228             !test_bit(__I40E_DOWN, &pf->state) &&
6229             !test_bit(__I40E_CONFIG_BUSY, &pf->state))
6230                 i40e_do_reset(pf, reset_flags);
6231
6232 unlock:
6233         rtnl_unlock();
6234 }
6235
6236 /**
6237  * i40e_handle_link_event - Handle link event
6238  * @pf: board private structure
6239  * @e: event info posted on ARQ
6240  **/
6241 static void i40e_handle_link_event(struct i40e_pf *pf,
6242                                    struct i40e_arq_event_info *e)
6243 {
6244         struct i40e_aqc_get_link_status *status =
6245                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
6246
6247         /* Do a new status request to re-enable LSE reporting
6248          * and load new status information into the hw struct
6249          * This completely ignores any state information
6250          * in the ARQ event info, instead choosing to always
6251          * issue the AQ update link status command.
6252          */
6253         i40e_link_event(pf);
6254
6255         /* check for unqualified module, if link is down */
6256         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
6257             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
6258             (!(status->link_info & I40E_AQ_LINK_UP)))
6259                 dev_err(&pf->pdev->dev,
6260                         "The driver failed to link because an unqualified module was detected.\n");
6261 }
6262
6263 /**
6264  * i40e_clean_adminq_subtask - Clean the AdminQ rings
6265  * @pf: board private structure
6266  **/
6267 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
6268 {
6269         struct i40e_arq_event_info event;
6270         struct i40e_hw *hw = &pf->hw;
6271         u16 pending, i = 0;
6272         i40e_status ret;
6273         u16 opcode;
6274         u32 oldval;
6275         u32 val;
6276
6277         /* Do not run clean AQ when PF reset fails */
6278         if (test_bit(__I40E_RESET_FAILED, &pf->state))
6279                 return;
6280
6281         /* check for error indications */
6282         val = rd32(&pf->hw, pf->hw.aq.arq.len);
6283         oldval = val;
6284         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
6285                 if (hw->debug_mask & I40E_DEBUG_AQ)
6286                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
6287                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
6288         }
6289         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
6290                 if (hw->debug_mask & I40E_DEBUG_AQ)
6291                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
6292                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
6293                 pf->arq_overflows++;
6294         }
6295         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
6296                 if (hw->debug_mask & I40E_DEBUG_AQ)
6297                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
6298                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
6299         }
6300         if (oldval != val)
6301                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
6302
6303         val = rd32(&pf->hw, pf->hw.aq.asq.len);
6304         oldval = val;
6305         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
6306                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6307                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
6308                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
6309         }
6310         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
6311                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6312                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
6313                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
6314         }
6315         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
6316                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6317                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
6318                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
6319         }
6320         if (oldval != val)
6321                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
6322
6323         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
6324         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
6325         if (!event.msg_buf)
6326                 return;
6327
6328         do {
6329                 ret = i40e_clean_arq_element(hw, &event, &pending);
6330                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
6331                         break;
6332                 else if (ret) {
6333                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
6334                         break;
6335                 }
6336
6337                 opcode = le16_to_cpu(event.desc.opcode);
6338                 switch (opcode) {
6339
6340                 case i40e_aqc_opc_get_link_status:
6341                         i40e_handle_link_event(pf, &event);
6342                         break;
6343                 case i40e_aqc_opc_send_msg_to_pf:
6344                         ret = i40e_vc_process_vf_msg(pf,
6345                                         le16_to_cpu(event.desc.retval),
6346                                         le32_to_cpu(event.desc.cookie_high),
6347                                         le32_to_cpu(event.desc.cookie_low),
6348                                         event.msg_buf,
6349                                         event.msg_len);
6350                         break;
6351                 case i40e_aqc_opc_lldp_update_mib:
6352                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
6353 #ifdef CONFIG_I40E_DCB
6354                         rtnl_lock();
6355                         ret = i40e_handle_lldp_event(pf, &event);
6356                         rtnl_unlock();
6357 #endif /* CONFIG_I40E_DCB */
6358                         break;
6359                 case i40e_aqc_opc_event_lan_overflow:
6360                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
6361                         i40e_handle_lan_overflow_event(pf, &event);
6362                         break;
6363                 case i40e_aqc_opc_send_msg_to_peer:
6364                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
6365                         break;
6366                 case i40e_aqc_opc_nvm_erase:
6367                 case i40e_aqc_opc_nvm_update:
6368                 case i40e_aqc_opc_oem_post_update:
6369                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
6370                                    "ARQ NVM operation 0x%04x completed\n",
6371                                    opcode);
6372                         break;
6373                 default:
6374                         dev_info(&pf->pdev->dev,
6375                                  "ARQ: Unknown event 0x%04x ignored\n",
6376                                  opcode);
6377                         break;
6378                 }
6379         } while (pending && (i++ < pf->adminq_work_limit));
6380
6381         clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
6382         /* re-enable Admin queue interrupt cause */
6383         val = rd32(hw, I40E_PFINT_ICR0_ENA);
6384         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
6385         wr32(hw, I40E_PFINT_ICR0_ENA, val);
6386         i40e_flush(hw);
6387
6388         kfree(event.msg_buf);
6389 }
6390
6391 /**
6392  * i40e_verify_eeprom - make sure eeprom is good to use
6393  * @pf: board private structure
6394  **/
6395 static void i40e_verify_eeprom(struct i40e_pf *pf)
6396 {
6397         int err;
6398
6399         err = i40e_diag_eeprom_test(&pf->hw);
6400         if (err) {
6401                 /* retry in case of garbage read */
6402                 err = i40e_diag_eeprom_test(&pf->hw);
6403                 if (err) {
6404                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
6405                                  err);
6406                         set_bit(__I40E_BAD_EEPROM, &pf->state);
6407                 }
6408         }
6409
6410         if (!err && test_bit(__I40E_BAD_EEPROM, &pf->state)) {
6411                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
6412                 clear_bit(__I40E_BAD_EEPROM, &pf->state);
6413         }
6414 }
6415
6416 /**
6417  * i40e_enable_pf_switch_lb
6418  * @pf: pointer to the PF structure
6419  *
6420  * enable switch loop back or die - no point in a return value
6421  **/
6422 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
6423 {
6424         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6425         struct i40e_vsi_context ctxt;
6426         int ret;
6427
6428         ctxt.seid = pf->main_vsi_seid;
6429         ctxt.pf_num = pf->hw.pf_id;
6430         ctxt.vf_num = 0;
6431         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6432         if (ret) {
6433                 dev_info(&pf->pdev->dev,
6434                          "couldn't get PF vsi config, err %s aq_err %s\n",
6435                          i40e_stat_str(&pf->hw, ret),
6436                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6437                 return;
6438         }
6439         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6440         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6441         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6442
6443         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6444         if (ret) {
6445                 dev_info(&pf->pdev->dev,
6446                          "update vsi switch failed, err %s aq_err %s\n",
6447                          i40e_stat_str(&pf->hw, ret),
6448                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6449         }
6450 }
6451
6452 /**
6453  * i40e_disable_pf_switch_lb
6454  * @pf: pointer to the PF structure
6455  *
6456  * disable switch loop back or die - no point in a return value
6457  **/
6458 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
6459 {
6460         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6461         struct i40e_vsi_context ctxt;
6462         int ret;
6463
6464         ctxt.seid = pf->main_vsi_seid;
6465         ctxt.pf_num = pf->hw.pf_id;
6466         ctxt.vf_num = 0;
6467         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6468         if (ret) {
6469                 dev_info(&pf->pdev->dev,
6470                          "couldn't get PF vsi config, err %s aq_err %s\n",
6471                          i40e_stat_str(&pf->hw, ret),
6472                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6473                 return;
6474         }
6475         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6476         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6477         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6478
6479         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6480         if (ret) {
6481                 dev_info(&pf->pdev->dev,
6482                          "update vsi switch failed, err %s aq_err %s\n",
6483                          i40e_stat_str(&pf->hw, ret),
6484                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6485         }
6486 }
6487
6488 /**
6489  * i40e_config_bridge_mode - Configure the HW bridge mode
6490  * @veb: pointer to the bridge instance
6491  *
6492  * Configure the loop back mode for the LAN VSI that is downlink to the
6493  * specified HW bridge instance. It is expected this function is called
6494  * when a new HW bridge is instantiated.
6495  **/
6496 static void i40e_config_bridge_mode(struct i40e_veb *veb)
6497 {
6498         struct i40e_pf *pf = veb->pf;
6499
6500         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
6501                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
6502                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
6503         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
6504                 i40e_disable_pf_switch_lb(pf);
6505         else
6506                 i40e_enable_pf_switch_lb(pf);
6507 }
6508
6509 /**
6510  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
6511  * @veb: pointer to the VEB instance
6512  *
6513  * This is a recursive function that first builds the attached VSIs then
6514  * recurses in to build the next layer of VEB.  We track the connections
6515  * through our own index numbers because the seid's from the HW could
6516  * change across the reset.
6517  **/
6518 static int i40e_reconstitute_veb(struct i40e_veb *veb)
6519 {
6520         struct i40e_vsi *ctl_vsi = NULL;
6521         struct i40e_pf *pf = veb->pf;
6522         int v, veb_idx;
6523         int ret;
6524
6525         /* build VSI that owns this VEB, temporarily attached to base VEB */
6526         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
6527                 if (pf->vsi[v] &&
6528                     pf->vsi[v]->veb_idx == veb->idx &&
6529                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
6530                         ctl_vsi = pf->vsi[v];
6531                         break;
6532                 }
6533         }
6534         if (!ctl_vsi) {
6535                 dev_info(&pf->pdev->dev,
6536                          "missing owner VSI for veb_idx %d\n", veb->idx);
6537                 ret = -ENOENT;
6538                 goto end_reconstitute;
6539         }
6540         if (ctl_vsi != pf->vsi[pf->lan_vsi])
6541                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6542         ret = i40e_add_vsi(ctl_vsi);
6543         if (ret) {
6544                 dev_info(&pf->pdev->dev,
6545                          "rebuild of veb_idx %d owner VSI failed: %d\n",
6546                          veb->idx, ret);
6547                 goto end_reconstitute;
6548         }
6549         i40e_vsi_reset_stats(ctl_vsi);
6550
6551         /* create the VEB in the switch and move the VSI onto the VEB */
6552         ret = i40e_add_veb(veb, ctl_vsi);
6553         if (ret)
6554                 goto end_reconstitute;
6555
6556         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
6557                 veb->bridge_mode = BRIDGE_MODE_VEB;
6558         else
6559                 veb->bridge_mode = BRIDGE_MODE_VEPA;
6560         i40e_config_bridge_mode(veb);
6561
6562         /* create the remaining VSIs attached to this VEB */
6563         for (v = 0; v < pf->num_alloc_vsi; v++) {
6564                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
6565                         continue;
6566
6567                 if (pf->vsi[v]->veb_idx == veb->idx) {
6568                         struct i40e_vsi *vsi = pf->vsi[v];
6569
6570                         vsi->uplink_seid = veb->seid;
6571                         ret = i40e_add_vsi(vsi);
6572                         if (ret) {
6573                                 dev_info(&pf->pdev->dev,
6574                                          "rebuild of vsi_idx %d failed: %d\n",
6575                                          v, ret);
6576                                 goto end_reconstitute;
6577                         }
6578                         i40e_vsi_reset_stats(vsi);
6579                 }
6580         }
6581
6582         /* create any VEBs attached to this VEB - RECURSION */
6583         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
6584                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
6585                         pf->veb[veb_idx]->uplink_seid = veb->seid;
6586                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
6587                         if (ret)
6588                                 break;
6589                 }
6590         }
6591
6592 end_reconstitute:
6593         return ret;
6594 }
6595
6596 /**
6597  * i40e_get_capabilities - get info about the HW
6598  * @pf: the PF struct
6599  **/
6600 static int i40e_get_capabilities(struct i40e_pf *pf)
6601 {
6602         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
6603         u16 data_size;
6604         int buf_len;
6605         int err;
6606
6607         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
6608         do {
6609                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
6610                 if (!cap_buf)
6611                         return -ENOMEM;
6612
6613                 /* this loads the data into the hw struct for us */
6614                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
6615                                             &data_size,
6616                                             i40e_aqc_opc_list_func_capabilities,
6617                                             NULL);
6618                 /* data loaded, buffer no longer needed */
6619                 kfree(cap_buf);
6620
6621                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
6622                         /* retry with a larger buffer */
6623                         buf_len = data_size;
6624                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
6625                         dev_info(&pf->pdev->dev,
6626                                  "capability discovery failed, err %s aq_err %s\n",
6627                                  i40e_stat_str(&pf->hw, err),
6628                                  i40e_aq_str(&pf->hw,
6629                                              pf->hw.aq.asq_last_status));
6630                         return -ENODEV;
6631                 }
6632         } while (err);
6633
6634         if (pf->hw.debug_mask & I40E_DEBUG_USER)
6635                 dev_info(&pf->pdev->dev,
6636                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
6637                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
6638                          pf->hw.func_caps.num_msix_vectors,
6639                          pf->hw.func_caps.num_msix_vectors_vf,
6640                          pf->hw.func_caps.fd_filters_guaranteed,
6641                          pf->hw.func_caps.fd_filters_best_effort,
6642                          pf->hw.func_caps.num_tx_qp,
6643                          pf->hw.func_caps.num_vsis);
6644
6645 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
6646                        + pf->hw.func_caps.num_vfs)
6647         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
6648                 dev_info(&pf->pdev->dev,
6649                          "got num_vsis %d, setting num_vsis to %d\n",
6650                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
6651                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
6652         }
6653
6654         return 0;
6655 }
6656
6657 static int i40e_vsi_clear(struct i40e_vsi *vsi);
6658
6659 /**
6660  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
6661  * @pf: board private structure
6662  **/
6663 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
6664 {
6665         struct i40e_vsi *vsi;
6666         int i;
6667
6668         /* quick workaround for an NVM issue that leaves a critical register
6669          * uninitialized
6670          */
6671         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
6672                 static const u32 hkey[] = {
6673                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
6674                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
6675                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
6676                         0x95b3a76d};
6677
6678                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
6679                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
6680         }
6681
6682         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
6683                 return;
6684
6685         /* find existing VSI and see if it needs configuring */
6686         vsi = NULL;
6687         for (i = 0; i < pf->num_alloc_vsi; i++) {
6688                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
6689                         vsi = pf->vsi[i];
6690                         break;
6691                 }
6692         }
6693
6694         /* create a new VSI if none exists */
6695         if (!vsi) {
6696                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
6697                                      pf->vsi[pf->lan_vsi]->seid, 0);
6698                 if (!vsi) {
6699                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
6700                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
6701                         return;
6702                 }
6703         }
6704
6705         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
6706 }
6707
6708 /**
6709  * i40e_fdir_teardown - release the Flow Director resources
6710  * @pf: board private structure
6711  **/
6712 static void i40e_fdir_teardown(struct i40e_pf *pf)
6713 {
6714         int i;
6715
6716         i40e_fdir_filter_exit(pf);
6717         for (i = 0; i < pf->num_alloc_vsi; i++) {
6718                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
6719                         i40e_vsi_release(pf->vsi[i]);
6720                         break;
6721                 }
6722         }
6723 }
6724
6725 /**
6726  * i40e_prep_for_reset - prep for the core to reset
6727  * @pf: board private structure
6728  *
6729  * Close up the VFs and other things in prep for PF Reset.
6730   **/
6731 static void i40e_prep_for_reset(struct i40e_pf *pf)
6732 {
6733         struct i40e_hw *hw = &pf->hw;
6734         i40e_status ret = 0;
6735         u32 v;
6736
6737         clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
6738         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
6739                 return;
6740         if (i40e_check_asq_alive(&pf->hw))
6741                 i40e_vc_notify_reset(pf);
6742
6743         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
6744
6745         /* quiesce the VSIs and their queues that are not already DOWN */
6746         i40e_pf_quiesce_all_vsi(pf);
6747
6748         for (v = 0; v < pf->num_alloc_vsi; v++) {
6749                 if (pf->vsi[v])
6750                         pf->vsi[v]->seid = 0;
6751         }
6752
6753         i40e_shutdown_adminq(&pf->hw);
6754
6755         /* call shutdown HMC */
6756         if (hw->hmc.hmc_obj) {
6757                 ret = i40e_shutdown_lan_hmc(hw);
6758                 if (ret)
6759                         dev_warn(&pf->pdev->dev,
6760                                  "shutdown_lan_hmc failed: %d\n", ret);
6761         }
6762 }
6763
6764 /**
6765  * i40e_send_version - update firmware with driver version
6766  * @pf: PF struct
6767  */
6768 static void i40e_send_version(struct i40e_pf *pf)
6769 {
6770         struct i40e_driver_version dv;
6771
6772         dv.major_version = DRV_VERSION_MAJOR;
6773         dv.minor_version = DRV_VERSION_MINOR;
6774         dv.build_version = DRV_VERSION_BUILD;
6775         dv.subbuild_version = 0;
6776         strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
6777         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
6778 }
6779
6780 /**
6781  * i40e_reset_and_rebuild - reset and rebuild using a saved config
6782  * @pf: board private structure
6783  * @reinit: if the Main VSI needs to re-initialized.
6784  **/
6785 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit)
6786 {
6787         struct i40e_hw *hw = &pf->hw;
6788         u8 set_fc_aq_fail = 0;
6789         i40e_status ret;
6790         u32 val;
6791         u32 v;
6792
6793         /* Now we wait for GRST to settle out.
6794          * We don't have to delete the VEBs or VSIs from the hw switch
6795          * because the reset will make them disappear.
6796          */
6797         ret = i40e_pf_reset(hw);
6798         if (ret) {
6799                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
6800                 set_bit(__I40E_RESET_FAILED, &pf->state);
6801                 goto clear_recovery;
6802         }
6803         pf->pfr_count++;
6804
6805         if (test_bit(__I40E_DOWN, &pf->state))
6806                 goto clear_recovery;
6807         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
6808
6809         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
6810         ret = i40e_init_adminq(&pf->hw);
6811         if (ret) {
6812                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
6813                          i40e_stat_str(&pf->hw, ret),
6814                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6815                 goto clear_recovery;
6816         }
6817
6818         /* re-verify the eeprom if we just had an EMP reset */
6819         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state))
6820                 i40e_verify_eeprom(pf);
6821
6822         i40e_clear_pxe_mode(hw);
6823         ret = i40e_get_capabilities(pf);
6824         if (ret)
6825                 goto end_core_reset;
6826
6827         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
6828                                 hw->func_caps.num_rx_qp,
6829                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
6830         if (ret) {
6831                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
6832                 goto end_core_reset;
6833         }
6834         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
6835         if (ret) {
6836                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
6837                 goto end_core_reset;
6838         }
6839
6840 #ifdef CONFIG_I40E_DCB
6841         ret = i40e_init_pf_dcb(pf);
6842         if (ret) {
6843                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
6844                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
6845                 /* Continue without DCB enabled */
6846         }
6847 #endif /* CONFIG_I40E_DCB */
6848 #ifdef I40E_FCOE
6849         i40e_init_pf_fcoe(pf);
6850
6851 #endif
6852         /* do basic switch setup */
6853         ret = i40e_setup_pf_switch(pf, reinit);
6854         if (ret)
6855                 goto end_core_reset;
6856
6857         /* The driver only wants link up/down and module qualification
6858          * reports from firmware.  Note the negative logic.
6859          */
6860         ret = i40e_aq_set_phy_int_mask(&pf->hw,
6861                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
6862                                          I40E_AQ_EVENT_MEDIA_NA |
6863                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
6864         if (ret)
6865                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
6866                          i40e_stat_str(&pf->hw, ret),
6867                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6868
6869         /* make sure our flow control settings are restored */
6870         ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
6871         if (ret)
6872                 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
6873                         i40e_stat_str(&pf->hw, ret),
6874                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6875
6876         /* Rebuild the VSIs and VEBs that existed before reset.
6877          * They are still in our local switch element arrays, so only
6878          * need to rebuild the switch model in the HW.
6879          *
6880          * If there were VEBs but the reconstitution failed, we'll try
6881          * try to recover minimal use by getting the basic PF VSI working.
6882          */
6883         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
6884                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
6885                 /* find the one VEB connected to the MAC, and find orphans */
6886                 for (v = 0; v < I40E_MAX_VEB; v++) {
6887                         if (!pf->veb[v])
6888                                 continue;
6889
6890                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
6891                             pf->veb[v]->uplink_seid == 0) {
6892                                 ret = i40e_reconstitute_veb(pf->veb[v]);
6893
6894                                 if (!ret)
6895                                         continue;
6896
6897                                 /* If Main VEB failed, we're in deep doodoo,
6898                                  * so give up rebuilding the switch and set up
6899                                  * for minimal rebuild of PF VSI.
6900                                  * If orphan failed, we'll report the error
6901                                  * but try to keep going.
6902                                  */
6903                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
6904                                         dev_info(&pf->pdev->dev,
6905                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
6906                                                  ret);
6907                                         pf->vsi[pf->lan_vsi]->uplink_seid
6908                                                                 = pf->mac_seid;
6909                                         break;
6910                                 } else if (pf->veb[v]->uplink_seid == 0) {
6911                                         dev_info(&pf->pdev->dev,
6912                                                  "rebuild of orphan VEB failed: %d\n",
6913                                                  ret);
6914                                 }
6915                         }
6916                 }
6917         }
6918
6919         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
6920                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
6921                 /* no VEB, so rebuild only the Main VSI */
6922                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
6923                 if (ret) {
6924                         dev_info(&pf->pdev->dev,
6925                                  "rebuild of Main VSI failed: %d\n", ret);
6926                         goto end_core_reset;
6927                 }
6928         }
6929
6930         /* Reconfigure hardware for allowing smaller MSS in the case
6931          * of TSO, so that we avoid the MDD being fired and causing
6932          * a reset in the case of small MSS+TSO.
6933          */
6934 #define I40E_REG_MSS          0x000E64DC
6935 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
6936 #define I40E_64BYTE_MSS       0x400000
6937         val = rd32(hw, I40E_REG_MSS);
6938         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
6939                 val &= ~I40E_REG_MSS_MIN_MASK;
6940                 val |= I40E_64BYTE_MSS;
6941                 wr32(hw, I40E_REG_MSS, val);
6942         }
6943
6944         if (pf->flags & I40E_FLAG_RESTART_AUTONEG) {
6945                 msleep(75);
6946                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
6947                 if (ret)
6948                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
6949                                  i40e_stat_str(&pf->hw, ret),
6950                                  i40e_aq_str(&pf->hw,
6951                                              pf->hw.aq.asq_last_status));
6952         }
6953         /* reinit the misc interrupt */
6954         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
6955                 ret = i40e_setup_misc_vector(pf);
6956
6957         /* Add a filter to drop all Flow control frames from any VSI from being
6958          * transmitted. By doing so we stop a malicious VF from sending out
6959          * PAUSE or PFC frames and potentially controlling traffic for other
6960          * PF/VF VSIs.
6961          * The FW can still send Flow control frames if enabled.
6962          */
6963         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
6964                                                        pf->main_vsi_seid);
6965
6966         /* restart the VSIs that were rebuilt and running before the reset */
6967         i40e_pf_unquiesce_all_vsi(pf);
6968
6969         if (pf->num_alloc_vfs) {
6970                 for (v = 0; v < pf->num_alloc_vfs; v++)
6971                         i40e_reset_vf(&pf->vf[v], true);
6972         }
6973
6974         /* tell the firmware that we're starting */
6975         i40e_send_version(pf);
6976
6977 end_core_reset:
6978         clear_bit(__I40E_RESET_FAILED, &pf->state);
6979 clear_recovery:
6980         clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
6981 }
6982
6983 /**
6984  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
6985  * @pf: board private structure
6986  *
6987  * Close up the VFs and other things in prep for a Core Reset,
6988  * then get ready to rebuild the world.
6989  **/
6990 static void i40e_handle_reset_warning(struct i40e_pf *pf)
6991 {
6992         i40e_prep_for_reset(pf);
6993         i40e_reset_and_rebuild(pf, false);
6994 }
6995
6996 /**
6997  * i40e_handle_mdd_event
6998  * @pf: pointer to the PF structure
6999  *
7000  * Called from the MDD irq handler to identify possibly malicious vfs
7001  **/
7002 static void i40e_handle_mdd_event(struct i40e_pf *pf)
7003 {
7004         struct i40e_hw *hw = &pf->hw;
7005         bool mdd_detected = false;
7006         bool pf_mdd_detected = false;
7007         struct i40e_vf *vf;
7008         u32 reg;
7009         int i;
7010
7011         if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state))
7012                 return;
7013
7014         /* find what triggered the MDD event */
7015         reg = rd32(hw, I40E_GL_MDET_TX);
7016         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
7017                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
7018                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
7019                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
7020                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
7021                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
7022                                 I40E_GL_MDET_TX_EVENT_SHIFT;
7023                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
7024                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
7025                                 pf->hw.func_caps.base_queue;
7026                 if (netif_msg_tx_err(pf))
7027                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
7028                                  event, queue, pf_num, vf_num);
7029                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
7030                 mdd_detected = true;
7031         }
7032         reg = rd32(hw, I40E_GL_MDET_RX);
7033         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
7034                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
7035                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
7036                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
7037                                 I40E_GL_MDET_RX_EVENT_SHIFT;
7038                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
7039                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
7040                                 pf->hw.func_caps.base_queue;
7041                 if (netif_msg_rx_err(pf))
7042                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
7043                                  event, queue, func);
7044                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
7045                 mdd_detected = true;
7046         }
7047
7048         if (mdd_detected) {
7049                 reg = rd32(hw, I40E_PF_MDET_TX);
7050                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
7051                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
7052                         dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
7053                         pf_mdd_detected = true;
7054                 }
7055                 reg = rd32(hw, I40E_PF_MDET_RX);
7056                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
7057                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
7058                         dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
7059                         pf_mdd_detected = true;
7060                 }
7061                 /* Queue belongs to the PF, initiate a reset */
7062                 if (pf_mdd_detected) {
7063                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
7064                         i40e_service_event_schedule(pf);
7065                 }
7066         }
7067
7068         /* see if one of the VFs needs its hand slapped */
7069         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
7070                 vf = &(pf->vf[i]);
7071                 reg = rd32(hw, I40E_VP_MDET_TX(i));
7072                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
7073                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
7074                         vf->num_mdd_events++;
7075                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
7076                                  i);
7077                 }
7078
7079                 reg = rd32(hw, I40E_VP_MDET_RX(i));
7080                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
7081                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
7082                         vf->num_mdd_events++;
7083                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
7084                                  i);
7085                 }
7086
7087                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
7088                         dev_info(&pf->pdev->dev,
7089                                  "Too many MDD events on VF %d, disabled\n", i);
7090                         dev_info(&pf->pdev->dev,
7091                                  "Use PF Control I/F to re-enable the VF\n");
7092                         set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states);
7093                 }
7094         }
7095
7096         /* re-enable mdd interrupt cause */
7097         clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
7098         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
7099         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
7100         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
7101         i40e_flush(hw);
7102 }
7103
7104 /**
7105  * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
7106  * @pf: board private structure
7107  **/
7108 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
7109 {
7110 #if IS_ENABLED(CONFIG_VXLAN) || IS_ENABLED(CONFIG_GENEVE)
7111         struct i40e_hw *hw = &pf->hw;
7112         i40e_status ret;
7113         __be16 port;
7114         int i;
7115
7116         if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC))
7117                 return;
7118
7119         pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC;
7120
7121         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7122                 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
7123                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
7124                         port = pf->udp_ports[i].index;
7125                         if (port)
7126                                 ret = i40e_aq_add_udp_tunnel(hw, ntohs(port),
7127                                                      pf->udp_ports[i].type,
7128                                                      NULL, NULL);
7129                         else
7130                                 ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
7131
7132                         if (ret) {
7133                                 dev_dbg(&pf->pdev->dev,
7134                                         "%s %s port %d, index %d failed, err %s aq_err %s\n",
7135                                         pf->udp_ports[i].type ? "vxlan" : "geneve",
7136                                         port ? "add" : "delete",
7137                                         ntohs(port), i,
7138                                         i40e_stat_str(&pf->hw, ret),
7139                                         i40e_aq_str(&pf->hw,
7140                                                     pf->hw.aq.asq_last_status));
7141                                 pf->udp_ports[i].index = 0;
7142                         }
7143                 }
7144         }
7145 #endif
7146 }
7147
7148 /**
7149  * i40e_service_task - Run the driver's async subtasks
7150  * @work: pointer to work_struct containing our data
7151  **/
7152 static void i40e_service_task(struct work_struct *work)
7153 {
7154         struct i40e_pf *pf = container_of(work,
7155                                           struct i40e_pf,
7156                                           service_task);
7157         unsigned long start_time = jiffies;
7158
7159         /* don't bother with service tasks if a reset is in progress */
7160         if (test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
7161                 i40e_service_event_complete(pf);
7162                 return;
7163         }
7164
7165         i40e_detect_recover_hung(pf);
7166         i40e_sync_filters_subtask(pf);
7167         i40e_reset_subtask(pf);
7168         i40e_handle_mdd_event(pf);
7169         i40e_vc_process_vflr_event(pf);
7170         i40e_watchdog_subtask(pf);
7171         i40e_fdir_reinit_subtask(pf);
7172         i40e_client_subtask(pf);
7173         i40e_sync_filters_subtask(pf);
7174         i40e_sync_udp_filters_subtask(pf);
7175         i40e_clean_adminq_subtask(pf);
7176
7177         i40e_service_event_complete(pf);
7178
7179         /* If the tasks have taken longer than one timer cycle or there
7180          * is more work to be done, reschedule the service task now
7181          * rather than wait for the timer to tick again.
7182          */
7183         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
7184             test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state)            ||
7185             test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)               ||
7186             test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state))
7187                 i40e_service_event_schedule(pf);
7188 }
7189
7190 /**
7191  * i40e_service_timer - timer callback
7192  * @data: pointer to PF struct
7193  **/
7194 static void i40e_service_timer(unsigned long data)
7195 {
7196         struct i40e_pf *pf = (struct i40e_pf *)data;
7197
7198         mod_timer(&pf->service_timer,
7199                   round_jiffies(jiffies + pf->service_timer_period));
7200         i40e_service_event_schedule(pf);
7201 }
7202
7203 /**
7204  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
7205  * @vsi: the VSI being configured
7206  **/
7207 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
7208 {
7209         struct i40e_pf *pf = vsi->back;
7210
7211         switch (vsi->type) {
7212         case I40E_VSI_MAIN:
7213                 vsi->alloc_queue_pairs = pf->num_lan_qps;
7214                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7215                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7216                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7217                         vsi->num_q_vectors = pf->num_lan_msix;
7218                 else
7219                         vsi->num_q_vectors = 1;
7220
7221                 break;
7222
7223         case I40E_VSI_FDIR:
7224                 vsi->alloc_queue_pairs = 1;
7225                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
7226                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7227                 vsi->num_q_vectors = 1;
7228                 break;
7229
7230         case I40E_VSI_VMDQ2:
7231                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
7232                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7233                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7234                 vsi->num_q_vectors = pf->num_vmdq_msix;
7235                 break;
7236
7237         case I40E_VSI_SRIOV:
7238                 vsi->alloc_queue_pairs = pf->num_vf_qps;
7239                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7240                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7241                 break;
7242
7243 #ifdef I40E_FCOE
7244         case I40E_VSI_FCOE:
7245                 vsi->alloc_queue_pairs = pf->num_fcoe_qps;
7246                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7247                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7248                 vsi->num_q_vectors = pf->num_fcoe_msix;
7249                 break;
7250
7251 #endif /* I40E_FCOE */
7252         default:
7253                 WARN_ON(1);
7254                 return -ENODATA;
7255         }
7256
7257         return 0;
7258 }
7259
7260 /**
7261  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
7262  * @type: VSI pointer
7263  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
7264  *
7265  * On error: returns error code (negative)
7266  * On success: returns 0
7267  **/
7268 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
7269 {
7270         int size;
7271         int ret = 0;
7272
7273         /* allocate memory for both Tx and Rx ring pointers */
7274         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2;
7275         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
7276         if (!vsi->tx_rings)
7277                 return -ENOMEM;
7278         vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs];
7279
7280         if (alloc_qvectors) {
7281                 /* allocate memory for q_vector pointers */
7282                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
7283                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
7284                 if (!vsi->q_vectors) {
7285                         ret = -ENOMEM;
7286                         goto err_vectors;
7287                 }
7288         }
7289         return ret;
7290
7291 err_vectors:
7292         kfree(vsi->tx_rings);
7293         return ret;
7294 }
7295
7296 /**
7297  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
7298  * @pf: board private structure
7299  * @type: type of VSI
7300  *
7301  * On error: returns error code (negative)
7302  * On success: returns vsi index in PF (positive)
7303  **/
7304 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
7305 {
7306         int ret = -ENODEV;
7307         struct i40e_vsi *vsi;
7308         int vsi_idx;
7309         int i;
7310
7311         /* Need to protect the allocation of the VSIs at the PF level */
7312         mutex_lock(&pf->switch_mutex);
7313
7314         /* VSI list may be fragmented if VSI creation/destruction has
7315          * been happening.  We can afford to do a quick scan to look
7316          * for any free VSIs in the list.
7317          *
7318          * find next empty vsi slot, looping back around if necessary
7319          */
7320         i = pf->next_vsi;
7321         while (i < pf->num_alloc_vsi && pf->vsi[i])
7322                 i++;
7323         if (i >= pf->num_alloc_vsi) {
7324                 i = 0;
7325                 while (i < pf->next_vsi && pf->vsi[i])
7326                         i++;
7327         }
7328
7329         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
7330                 vsi_idx = i;             /* Found one! */
7331         } else {
7332                 ret = -ENODEV;
7333                 goto unlock_pf;  /* out of VSI slots! */
7334         }
7335         pf->next_vsi = ++i;
7336
7337         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
7338         if (!vsi) {
7339                 ret = -ENOMEM;
7340                 goto unlock_pf;
7341         }
7342         vsi->type = type;
7343         vsi->back = pf;
7344         set_bit(__I40E_DOWN, &vsi->state);
7345         vsi->flags = 0;
7346         vsi->idx = vsi_idx;
7347         vsi->int_rate_limit = 0;
7348         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
7349                                 pf->rss_table_size : 64;
7350         vsi->netdev_registered = false;
7351         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
7352         INIT_LIST_HEAD(&vsi->mac_filter_list);
7353         vsi->irqs_ready = false;
7354
7355         ret = i40e_set_num_rings_in_vsi(vsi);
7356         if (ret)
7357                 goto err_rings;
7358
7359         ret = i40e_vsi_alloc_arrays(vsi, true);
7360         if (ret)
7361                 goto err_rings;
7362
7363         /* Setup default MSIX irq handler for VSI */
7364         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
7365
7366         /* Initialize VSI lock */
7367         spin_lock_init(&vsi->mac_filter_list_lock);
7368         pf->vsi[vsi_idx] = vsi;
7369         ret = vsi_idx;
7370         goto unlock_pf;
7371
7372 err_rings:
7373         pf->next_vsi = i - 1;
7374         kfree(vsi);
7375 unlock_pf:
7376         mutex_unlock(&pf->switch_mutex);
7377         return ret;
7378 }
7379
7380 /**
7381  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
7382  * @type: VSI pointer
7383  * @free_qvectors: a bool to specify if q_vectors need to be freed.
7384  *
7385  * On error: returns error code (negative)
7386  * On success: returns 0
7387  **/
7388 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
7389 {
7390         /* free the ring and vector containers */
7391         if (free_qvectors) {
7392                 kfree(vsi->q_vectors);
7393                 vsi->q_vectors = NULL;
7394         }
7395         kfree(vsi->tx_rings);
7396         vsi->tx_rings = NULL;
7397         vsi->rx_rings = NULL;
7398 }
7399
7400 /**
7401  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
7402  * and lookup table
7403  * @vsi: Pointer to VSI structure
7404  */
7405 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
7406 {
7407         if (!vsi)
7408                 return;
7409
7410         kfree(vsi->rss_hkey_user);
7411         vsi->rss_hkey_user = NULL;
7412
7413         kfree(vsi->rss_lut_user);
7414         vsi->rss_lut_user = NULL;
7415 }
7416
7417 /**
7418  * i40e_vsi_clear - Deallocate the VSI provided
7419  * @vsi: the VSI being un-configured
7420  **/
7421 static int i40e_vsi_clear(struct i40e_vsi *vsi)
7422 {
7423         struct i40e_pf *pf;
7424
7425         if (!vsi)
7426                 return 0;
7427
7428         if (!vsi->back)
7429                 goto free_vsi;
7430         pf = vsi->back;
7431
7432         mutex_lock(&pf->switch_mutex);
7433         if (!pf->vsi[vsi->idx]) {
7434                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
7435                         vsi->idx, vsi->idx, vsi, vsi->type);
7436                 goto unlock_vsi;
7437         }
7438
7439         if (pf->vsi[vsi->idx] != vsi) {
7440                 dev_err(&pf->pdev->dev,
7441                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
7442                         pf->vsi[vsi->idx]->idx,
7443                         pf->vsi[vsi->idx],
7444                         pf->vsi[vsi->idx]->type,
7445                         vsi->idx, vsi, vsi->type);
7446                 goto unlock_vsi;
7447         }
7448
7449         /* updates the PF for this cleared vsi */
7450         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
7451         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
7452
7453         i40e_vsi_free_arrays(vsi, true);
7454         i40e_clear_rss_config_user(vsi);
7455
7456         pf->vsi[vsi->idx] = NULL;
7457         if (vsi->idx < pf->next_vsi)
7458                 pf->next_vsi = vsi->idx;
7459
7460 unlock_vsi:
7461         mutex_unlock(&pf->switch_mutex);
7462 free_vsi:
7463         kfree(vsi);
7464
7465         return 0;
7466 }
7467
7468 /**
7469  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
7470  * @vsi: the VSI being cleaned
7471  **/
7472 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
7473 {
7474         int i;
7475
7476         if (vsi->tx_rings && vsi->tx_rings[0]) {
7477                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7478                         kfree_rcu(vsi->tx_rings[i], rcu);
7479                         vsi->tx_rings[i] = NULL;
7480                         vsi->rx_rings[i] = NULL;
7481                 }
7482         }
7483 }
7484
7485 /**
7486  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
7487  * @vsi: the VSI being configured
7488  **/
7489 static int i40e_alloc_rings(struct i40e_vsi *vsi)
7490 {
7491         struct i40e_ring *tx_ring, *rx_ring;
7492         struct i40e_pf *pf = vsi->back;
7493         int i;
7494
7495         /* Set basic values in the rings to be used later during open() */
7496         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7497                 /* allocate space for both Tx and Rx in one shot */
7498                 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
7499                 if (!tx_ring)
7500                         goto err_out;
7501
7502                 tx_ring->queue_index = i;
7503                 tx_ring->reg_idx = vsi->base_queue + i;
7504                 tx_ring->ring_active = false;
7505                 tx_ring->vsi = vsi;
7506                 tx_ring->netdev = vsi->netdev;
7507                 tx_ring->dev = &pf->pdev->dev;
7508                 tx_ring->count = vsi->num_desc;
7509                 tx_ring->size = 0;
7510                 tx_ring->dcb_tc = 0;
7511                 if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE)
7512                         tx_ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7513                 tx_ring->tx_itr_setting = pf->tx_itr_default;
7514                 vsi->tx_rings[i] = tx_ring;
7515
7516                 rx_ring = &tx_ring[1];
7517                 rx_ring->queue_index = i;
7518                 rx_ring->reg_idx = vsi->base_queue + i;
7519                 rx_ring->ring_active = false;
7520                 rx_ring->vsi = vsi;
7521                 rx_ring->netdev = vsi->netdev;
7522                 rx_ring->dev = &pf->pdev->dev;
7523                 rx_ring->count = vsi->num_desc;
7524                 rx_ring->size = 0;
7525                 rx_ring->dcb_tc = 0;
7526                 if (pf->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED)
7527                         set_ring_16byte_desc_enabled(rx_ring);
7528                 else
7529                         clear_ring_16byte_desc_enabled(rx_ring);
7530                 rx_ring->rx_itr_setting = pf->rx_itr_default;
7531                 vsi->rx_rings[i] = rx_ring;
7532         }
7533
7534         return 0;
7535
7536 err_out:
7537         i40e_vsi_clear_rings(vsi);
7538         return -ENOMEM;
7539 }
7540
7541 /**
7542  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
7543  * @pf: board private structure
7544  * @vectors: the number of MSI-X vectors to request
7545  *
7546  * Returns the number of vectors reserved, or error
7547  **/
7548 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
7549 {
7550         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
7551                                         I40E_MIN_MSIX, vectors);
7552         if (vectors < 0) {
7553                 dev_info(&pf->pdev->dev,
7554                          "MSI-X vector reservation failed: %d\n", vectors);
7555                 vectors = 0;
7556         }
7557
7558         return vectors;
7559 }
7560
7561 /**
7562  * i40e_init_msix - Setup the MSIX capability
7563  * @pf: board private structure
7564  *
7565  * Work with the OS to set up the MSIX vectors needed.
7566  *
7567  * Returns the number of vectors reserved or negative on failure
7568  **/
7569 static int i40e_init_msix(struct i40e_pf *pf)
7570 {
7571         struct i40e_hw *hw = &pf->hw;
7572         int vectors_left;
7573         int v_budget, i;
7574         int v_actual;
7575         int iwarp_requested = 0;
7576
7577         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
7578                 return -ENODEV;
7579
7580         /* The number of vectors we'll request will be comprised of:
7581          *   - Add 1 for "other" cause for Admin Queue events, etc.
7582          *   - The number of LAN queue pairs
7583          *      - Queues being used for RSS.
7584          *              We don't need as many as max_rss_size vectors.
7585          *              use rss_size instead in the calculation since that
7586          *              is governed by number of cpus in the system.
7587          *      - assumes symmetric Tx/Rx pairing
7588          *   - The number of VMDq pairs
7589          *   - The CPU count within the NUMA node if iWARP is enabled
7590 #ifdef I40E_FCOE
7591          *   - The number of FCOE qps.
7592 #endif
7593          * Once we count this up, try the request.
7594          *
7595          * If we can't get what we want, we'll simplify to nearly nothing
7596          * and try again.  If that still fails, we punt.
7597          */
7598         vectors_left = hw->func_caps.num_msix_vectors;
7599         v_budget = 0;
7600
7601         /* reserve one vector for miscellaneous handler */
7602         if (vectors_left) {
7603                 v_budget++;
7604                 vectors_left--;
7605         }
7606
7607         /* reserve vectors for the main PF traffic queues */
7608         pf->num_lan_msix = min_t(int, num_online_cpus(), vectors_left);
7609         vectors_left -= pf->num_lan_msix;
7610         v_budget += pf->num_lan_msix;
7611
7612         /* reserve one vector for sideband flow director */
7613         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
7614                 if (vectors_left) {
7615                         v_budget++;
7616                         vectors_left--;
7617                 } else {
7618                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
7619                 }
7620         }
7621
7622 #ifdef I40E_FCOE
7623         /* can we reserve enough for FCoE? */
7624         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
7625                 if (!vectors_left)
7626                         pf->num_fcoe_msix = 0;
7627                 else if (vectors_left >= pf->num_fcoe_qps)
7628                         pf->num_fcoe_msix = pf->num_fcoe_qps;
7629                 else
7630                         pf->num_fcoe_msix = 1;
7631                 v_budget += pf->num_fcoe_msix;
7632                 vectors_left -= pf->num_fcoe_msix;
7633         }
7634
7635 #endif
7636         /* can we reserve enough for iWARP? */
7637         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
7638                 if (!vectors_left)
7639                         pf->num_iwarp_msix = 0;
7640                 else if (vectors_left < pf->num_iwarp_msix)
7641                         pf->num_iwarp_msix = 1;
7642                 v_budget += pf->num_iwarp_msix;
7643                 vectors_left -= pf->num_iwarp_msix;
7644         }
7645
7646         /* any vectors left over go for VMDq support */
7647         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
7648                 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps;
7649                 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted);
7650
7651                 /* if we're short on vectors for what's desired, we limit
7652                  * the queues per vmdq.  If this is still more than are
7653                  * available, the user will need to change the number of
7654                  * queues/vectors used by the PF later with the ethtool
7655                  * channels command
7656                  */
7657                 if (vmdq_vecs < vmdq_vecs_wanted)
7658                         pf->num_vmdq_qps = 1;
7659                 pf->num_vmdq_msix = pf->num_vmdq_qps;
7660
7661                 v_budget += vmdq_vecs;
7662                 vectors_left -= vmdq_vecs;
7663         }
7664
7665         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
7666                                    GFP_KERNEL);
7667         if (!pf->msix_entries)
7668                 return -ENOMEM;
7669
7670         for (i = 0; i < v_budget; i++)
7671                 pf->msix_entries[i].entry = i;
7672         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
7673
7674         if (v_actual != v_budget) {
7675                 /* If we have limited resources, we will start with no vectors
7676                  * for the special features and then allocate vectors to some
7677                  * of these features based on the policy and at the end disable
7678                  * the features that did not get any vectors.
7679                  */
7680                 iwarp_requested = pf->num_iwarp_msix;
7681                 pf->num_iwarp_msix = 0;
7682 #ifdef I40E_FCOE
7683                 pf->num_fcoe_qps = 0;
7684                 pf->num_fcoe_msix = 0;
7685 #endif
7686                 pf->num_vmdq_msix = 0;
7687         }
7688
7689         if (v_actual < I40E_MIN_MSIX) {
7690                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
7691                 kfree(pf->msix_entries);
7692                 pf->msix_entries = NULL;
7693                 return -ENODEV;
7694
7695         } else if (v_actual == I40E_MIN_MSIX) {
7696                 /* Adjust for minimal MSIX use */
7697                 pf->num_vmdq_vsis = 0;
7698                 pf->num_vmdq_qps = 0;
7699                 pf->num_lan_qps = 1;
7700                 pf->num_lan_msix = 1;
7701
7702         } else if (v_actual != v_budget) {
7703                 int vec;
7704
7705                 /* reserve the misc vector */
7706                 vec = v_actual - 1;
7707
7708                 /* Scale vector usage down */
7709                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
7710                 pf->num_vmdq_vsis = 1;
7711                 pf->num_vmdq_qps = 1;
7712                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
7713
7714                 /* partition out the remaining vectors */
7715                 switch (vec) {
7716                 case 2:
7717                         pf->num_lan_msix = 1;
7718                         break;
7719                 case 3:
7720                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
7721                                 pf->num_lan_msix = 1;
7722                                 pf->num_iwarp_msix = 1;
7723                         } else {
7724                                 pf->num_lan_msix = 2;
7725                         }
7726 #ifdef I40E_FCOE
7727                         /* give one vector to FCoE */
7728                         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
7729                                 pf->num_lan_msix = 1;
7730                                 pf->num_fcoe_msix = 1;
7731                         }
7732 #endif
7733                         break;
7734                 default:
7735                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
7736                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
7737                                                  iwarp_requested);
7738                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
7739                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
7740                         } else {
7741                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
7742                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
7743                         }
7744                         pf->num_lan_msix = min_t(int,
7745                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
7746                                                               pf->num_lan_msix);
7747 #ifdef I40E_FCOE
7748                         /* give one vector to FCoE */
7749                         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
7750                                 pf->num_fcoe_msix = 1;
7751                                 vec--;
7752                         }
7753 #endif
7754                         break;
7755                 }
7756         }
7757
7758         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
7759             (pf->num_vmdq_msix == 0)) {
7760                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
7761                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
7762         }
7763
7764         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
7765             (pf->num_iwarp_msix == 0)) {
7766                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
7767                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
7768         }
7769 #ifdef I40E_FCOE
7770
7771         if ((pf->flags & I40E_FLAG_FCOE_ENABLED) && (pf->num_fcoe_msix == 0)) {
7772                 dev_info(&pf->pdev->dev, "FCOE disabled, not enough MSI-X vectors\n");
7773                 pf->flags &= ~I40E_FLAG_FCOE_ENABLED;
7774         }
7775 #endif
7776         return v_actual;
7777 }
7778
7779 /**
7780  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
7781  * @vsi: the VSI being configured
7782  * @v_idx: index of the vector in the vsi struct
7783  *
7784  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
7785  **/
7786 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
7787 {
7788         struct i40e_q_vector *q_vector;
7789
7790         /* allocate q_vector */
7791         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
7792         if (!q_vector)
7793                 return -ENOMEM;
7794
7795         q_vector->vsi = vsi;
7796         q_vector->v_idx = v_idx;
7797         cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
7798         if (vsi->netdev)
7799                 netif_napi_add(vsi->netdev, &q_vector->napi,
7800                                i40e_napi_poll, NAPI_POLL_WEIGHT);
7801
7802         q_vector->rx.latency_range = I40E_LOW_LATENCY;
7803         q_vector->tx.latency_range = I40E_LOW_LATENCY;
7804
7805         /* tie q_vector and vsi together */
7806         vsi->q_vectors[v_idx] = q_vector;
7807
7808         return 0;
7809 }
7810
7811 /**
7812  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
7813  * @vsi: the VSI being configured
7814  *
7815  * We allocate one q_vector per queue interrupt.  If allocation fails we
7816  * return -ENOMEM.
7817  **/
7818 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
7819 {
7820         struct i40e_pf *pf = vsi->back;
7821         int v_idx, num_q_vectors;
7822         int err;
7823
7824         /* if not MSIX, give the one vector only to the LAN VSI */
7825         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7826                 num_q_vectors = vsi->num_q_vectors;
7827         else if (vsi == pf->vsi[pf->lan_vsi])
7828                 num_q_vectors = 1;
7829         else
7830                 return -EINVAL;
7831
7832         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
7833                 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
7834                 if (err)
7835                         goto err_out;
7836         }
7837
7838         return 0;
7839
7840 err_out:
7841         while (v_idx--)
7842                 i40e_free_q_vector(vsi, v_idx);
7843
7844         return err;
7845 }
7846
7847 /**
7848  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
7849  * @pf: board private structure to initialize
7850  **/
7851 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
7852 {
7853         int vectors = 0;
7854         ssize_t size;
7855
7856         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
7857                 vectors = i40e_init_msix(pf);
7858                 if (vectors < 0) {
7859                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
7860                                        I40E_FLAG_IWARP_ENABLED  |
7861 #ifdef I40E_FCOE
7862                                        I40E_FLAG_FCOE_ENABLED   |
7863 #endif
7864                                        I40E_FLAG_RSS_ENABLED    |
7865                                        I40E_FLAG_DCB_CAPABLE    |
7866                                        I40E_FLAG_SRIOV_ENABLED  |
7867                                        I40E_FLAG_FD_SB_ENABLED  |
7868                                        I40E_FLAG_FD_ATR_ENABLED |
7869                                        I40E_FLAG_VMDQ_ENABLED);
7870
7871                         /* rework the queue expectations without MSIX */
7872                         i40e_determine_queue_usage(pf);
7873                 }
7874         }
7875
7876         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
7877             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
7878                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
7879                 vectors = pci_enable_msi(pf->pdev);
7880                 if (vectors < 0) {
7881                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
7882                                  vectors);
7883                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
7884                 }
7885                 vectors = 1;  /* one MSI or Legacy vector */
7886         }
7887
7888         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
7889                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
7890
7891         /* set up vector assignment tracking */
7892         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
7893         pf->irq_pile = kzalloc(size, GFP_KERNEL);
7894         if (!pf->irq_pile) {
7895                 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n");
7896                 return -ENOMEM;
7897         }
7898         pf->irq_pile->num_entries = vectors;
7899         pf->irq_pile->search_hint = 0;
7900
7901         /* track first vector for misc interrupts, ignore return */
7902         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
7903
7904         return 0;
7905 }
7906
7907 /**
7908  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
7909  * @pf: board private structure
7910  *
7911  * This sets up the handler for MSIX 0, which is used to manage the
7912  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
7913  * when in MSI or Legacy interrupt mode.
7914  **/
7915 static int i40e_setup_misc_vector(struct i40e_pf *pf)
7916 {
7917         struct i40e_hw *hw = &pf->hw;
7918         int err = 0;
7919
7920         /* Only request the irq if this is the first time through, and
7921          * not when we're rebuilding after a Reset
7922          */
7923         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
7924                 err = request_irq(pf->msix_entries[0].vector,
7925                                   i40e_intr, 0, pf->int_name, pf);
7926                 if (err) {
7927                         dev_info(&pf->pdev->dev,
7928                                  "request_irq for %s failed: %d\n",
7929                                  pf->int_name, err);
7930                         return -EFAULT;
7931                 }
7932         }
7933
7934         i40e_enable_misc_int_causes(pf);
7935
7936         /* associate no queues to the misc vector */
7937         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
7938         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
7939
7940         i40e_flush(hw);
7941
7942         i40e_irq_dynamic_enable_icr0(pf, true);
7943
7944         return err;
7945 }
7946
7947 /**
7948  * i40e_config_rss_aq - Prepare for RSS using AQ commands
7949  * @vsi: vsi structure
7950  * @seed: RSS hash seed
7951  **/
7952 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
7953                               u8 *lut, u16 lut_size)
7954 {
7955         struct i40e_aqc_get_set_rss_key_data rss_key;
7956         struct i40e_pf *pf = vsi->back;
7957         struct i40e_hw *hw = &pf->hw;
7958         bool pf_lut = false;
7959         u8 *rss_lut;
7960         int ret, i;
7961
7962         memset(&rss_key, 0, sizeof(rss_key));
7963         memcpy(&rss_key, seed, sizeof(rss_key));
7964
7965         rss_lut = kzalloc(pf->rss_table_size, GFP_KERNEL);
7966         if (!rss_lut)
7967                 return -ENOMEM;
7968
7969         /* Populate the LUT with max no. of queues in round robin fashion */
7970         for (i = 0; i < vsi->rss_table_size; i++)
7971                 rss_lut[i] = i % vsi->rss_size;
7972
7973         ret = i40e_aq_set_rss_key(hw, vsi->id, &rss_key);
7974         if (ret) {
7975                 dev_info(&pf->pdev->dev,
7976                          "Cannot set RSS key, err %s aq_err %s\n",
7977                          i40e_stat_str(&pf->hw, ret),
7978                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7979                 goto config_rss_aq_out;
7980         }
7981
7982         if (vsi->type == I40E_VSI_MAIN)
7983                 pf_lut = true;
7984
7985         ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, rss_lut,
7986                                   vsi->rss_table_size);
7987         if (ret)
7988                 dev_info(&pf->pdev->dev,
7989                          "Cannot set RSS lut, err %s aq_err %s\n",
7990                          i40e_stat_str(&pf->hw, ret),
7991                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7992
7993 config_rss_aq_out:
7994         kfree(rss_lut);
7995         return ret;
7996 }
7997
7998 /**
7999  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
8000  * @vsi: VSI structure
8001  **/
8002 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
8003 {
8004         u8 seed[I40E_HKEY_ARRAY_SIZE];
8005         struct i40e_pf *pf = vsi->back;
8006         u8 *lut;
8007         int ret;
8008
8009         if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE))
8010                 return 0;
8011
8012         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8013         if (!lut)
8014                 return -ENOMEM;
8015
8016         i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8017         netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8018         vsi->rss_size = min_t(int, pf->alloc_rss_size, vsi->num_queue_pairs);
8019         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
8020         kfree(lut);
8021
8022         return ret;
8023 }
8024
8025 /**
8026  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
8027  * @vsi: Pointer to vsi structure
8028  * @seed: Buffter to store the hash keys
8029  * @lut: Buffer to store the lookup table entries
8030  * @lut_size: Size of buffer to store the lookup table entries
8031  *
8032  * Return 0 on success, negative on failure
8033  */
8034 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8035                            u8 *lut, u16 lut_size)
8036 {
8037         struct i40e_pf *pf = vsi->back;
8038         struct i40e_hw *hw = &pf->hw;
8039         int ret = 0;
8040
8041         if (seed) {
8042                 ret = i40e_aq_get_rss_key(hw, vsi->id,
8043                         (struct i40e_aqc_get_set_rss_key_data *)seed);
8044                 if (ret) {
8045                         dev_info(&pf->pdev->dev,
8046                                  "Cannot get RSS key, err %s aq_err %s\n",
8047                                  i40e_stat_str(&pf->hw, ret),
8048                                  i40e_aq_str(&pf->hw,
8049                                              pf->hw.aq.asq_last_status));
8050                         return ret;
8051                 }
8052         }
8053
8054         if (lut) {
8055                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8056
8057                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8058                 if (ret) {
8059                         dev_info(&pf->pdev->dev,
8060                                  "Cannot get RSS lut, err %s aq_err %s\n",
8061                                  i40e_stat_str(&pf->hw, ret),
8062                                  i40e_aq_str(&pf->hw,
8063                                              pf->hw.aq.asq_last_status));
8064                         return ret;
8065                 }
8066         }
8067
8068         return ret;
8069 }
8070
8071 /**
8072  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
8073  * @vsi: Pointer to vsi structure
8074  * @seed: RSS hash seed
8075  * @lut: Lookup table
8076  * @lut_size: Lookup table size
8077  *
8078  * Returns 0 on success, negative on failure
8079  **/
8080 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
8081                                const u8 *lut, u16 lut_size)
8082 {
8083         struct i40e_pf *pf = vsi->back;
8084         struct i40e_hw *hw = &pf->hw;
8085         u8 i;
8086
8087         /* Fill out hash function seed */
8088         if (seed) {
8089                 u32 *seed_dw = (u32 *)seed;
8090
8091                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8092                         i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
8093         }
8094
8095         if (lut) {
8096                 u32 *lut_dw = (u32 *)lut;
8097
8098                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
8099                         return -EINVAL;
8100
8101                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8102                         wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
8103         }
8104         i40e_flush(hw);
8105
8106         return 0;
8107 }
8108
8109 /**
8110  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
8111  * @vsi: Pointer to VSI structure
8112  * @seed: Buffer to store the keys
8113  * @lut: Buffer to store the lookup table entries
8114  * @lut_size: Size of buffer to store the lookup table entries
8115  *
8116  * Returns 0 on success, negative on failure
8117  */
8118 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
8119                             u8 *lut, u16 lut_size)
8120 {
8121         struct i40e_pf *pf = vsi->back;
8122         struct i40e_hw *hw = &pf->hw;
8123         u16 i;
8124
8125         if (seed) {
8126                 u32 *seed_dw = (u32 *)seed;
8127
8128                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8129                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
8130         }
8131         if (lut) {
8132                 u32 *lut_dw = (u32 *)lut;
8133
8134                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
8135                         return -EINVAL;
8136                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8137                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
8138         }
8139
8140         return 0;
8141 }
8142
8143 /**
8144  * i40e_config_rss - Configure RSS keys and lut
8145  * @vsi: Pointer to VSI structure
8146  * @seed: RSS hash seed
8147  * @lut: Lookup table
8148  * @lut_size: Lookup table size
8149  *
8150  * Returns 0 on success, negative on failure
8151  */
8152 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8153 {
8154         struct i40e_pf *pf = vsi->back;
8155
8156         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)
8157                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
8158         else
8159                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
8160 }
8161
8162 /**
8163  * i40e_get_rss - Get RSS keys and lut
8164  * @vsi: Pointer to VSI structure
8165  * @seed: Buffer to store the keys
8166  * @lut: Buffer to store the lookup table entries
8167  * lut_size: Size of buffer to store the lookup table entries
8168  *
8169  * Returns 0 on success, negative on failure
8170  */
8171 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8172 {
8173         struct i40e_pf *pf = vsi->back;
8174
8175         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)
8176                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
8177         else
8178                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
8179 }
8180
8181 /**
8182  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
8183  * @pf: Pointer to board private structure
8184  * @lut: Lookup table
8185  * @rss_table_size: Lookup table size
8186  * @rss_size: Range of queue number for hashing
8187  */
8188 static void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
8189                               u16 rss_table_size, u16 rss_size)
8190 {
8191         u16 i;
8192
8193         for (i = 0; i < rss_table_size; i++)
8194                 lut[i] = i % rss_size;
8195 }
8196
8197 /**
8198  * i40e_pf_config_rss - Prepare for RSS if used
8199  * @pf: board private structure
8200  **/
8201 static int i40e_pf_config_rss(struct i40e_pf *pf)
8202 {
8203         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8204         u8 seed[I40E_HKEY_ARRAY_SIZE];
8205         u8 *lut;
8206         struct i40e_hw *hw = &pf->hw;
8207         u32 reg_val;
8208         u64 hena;
8209         int ret;
8210
8211         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
8212         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
8213                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
8214         hena |= i40e_pf_get_default_rss_hena(pf);
8215
8216         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
8217         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
8218
8219         /* Determine the RSS table size based on the hardware capabilities */
8220         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
8221         reg_val = (pf->rss_table_size == 512) ?
8222                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
8223                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
8224         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
8225
8226         /* Determine the RSS size of the VSI */
8227         if (!vsi->rss_size)
8228                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
8229                                       vsi->num_queue_pairs);
8230
8231         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8232         if (!lut)
8233                 return -ENOMEM;
8234
8235         /* Use user configured lut if there is one, otherwise use default */
8236         if (vsi->rss_lut_user)
8237                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8238         else
8239                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8240
8241         /* Use user configured hash key if there is one, otherwise
8242          * use default.
8243          */
8244         if (vsi->rss_hkey_user)
8245                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8246         else
8247                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8248         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
8249         kfree(lut);
8250
8251         return ret;
8252 }
8253
8254 /**
8255  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
8256  * @pf: board private structure
8257  * @queue_count: the requested queue count for rss.
8258  *
8259  * returns 0 if rss is not enabled, if enabled returns the final rss queue
8260  * count which may be different from the requested queue count.
8261  **/
8262 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
8263 {
8264         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8265         int new_rss_size;
8266
8267         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
8268                 return 0;
8269
8270         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
8271
8272         if (queue_count != vsi->num_queue_pairs) {
8273                 vsi->req_queue_pairs = queue_count;
8274                 i40e_prep_for_reset(pf);
8275
8276                 pf->alloc_rss_size = new_rss_size;
8277
8278                 i40e_reset_and_rebuild(pf, true);
8279
8280                 /* Discard the user configured hash keys and lut, if less
8281                  * queues are enabled.
8282                  */
8283                 if (queue_count < vsi->rss_size) {
8284                         i40e_clear_rss_config_user(vsi);
8285                         dev_dbg(&pf->pdev->dev,
8286                                 "discard user configured hash keys and lut\n");
8287                 }
8288
8289                 /* Reset vsi->rss_size, as number of enabled queues changed */
8290                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
8291                                       vsi->num_queue_pairs);
8292
8293                 i40e_pf_config_rss(pf);
8294         }
8295         dev_info(&pf->pdev->dev, "RSS count/HW max RSS count:  %d/%d\n",
8296                  pf->alloc_rss_size, pf->rss_size_max);
8297         return pf->alloc_rss_size;
8298 }
8299
8300 /**
8301  * i40e_get_npar_bw_setting - Retrieve BW settings for this PF partition
8302  * @pf: board private structure
8303  **/
8304 i40e_status i40e_get_npar_bw_setting(struct i40e_pf *pf)
8305 {
8306         i40e_status status;
8307         bool min_valid, max_valid;
8308         u32 max_bw, min_bw;
8309
8310         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
8311                                            &min_valid, &max_valid);
8312
8313         if (!status) {
8314                 if (min_valid)
8315                         pf->npar_min_bw = min_bw;
8316                 if (max_valid)
8317                         pf->npar_max_bw = max_bw;
8318         }
8319
8320         return status;
8321 }
8322
8323 /**
8324  * i40e_set_npar_bw_setting - Set BW settings for this PF partition
8325  * @pf: board private structure
8326  **/
8327 i40e_status i40e_set_npar_bw_setting(struct i40e_pf *pf)
8328 {
8329         struct i40e_aqc_configure_partition_bw_data bw_data;
8330         i40e_status status;
8331
8332         /* Set the valid bit for this PF */
8333         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
8334         bw_data.max_bw[pf->hw.pf_id] = pf->npar_max_bw & I40E_ALT_BW_VALUE_MASK;
8335         bw_data.min_bw[pf->hw.pf_id] = pf->npar_min_bw & I40E_ALT_BW_VALUE_MASK;
8336
8337         /* Set the new bandwidths */
8338         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
8339
8340         return status;
8341 }
8342
8343 /**
8344  * i40e_commit_npar_bw_setting - Commit BW settings for this PF partition
8345  * @pf: board private structure
8346  **/
8347 i40e_status i40e_commit_npar_bw_setting(struct i40e_pf *pf)
8348 {
8349         /* Commit temporary BW setting to permanent NVM image */
8350         enum i40e_admin_queue_err last_aq_status;
8351         i40e_status ret;
8352         u16 nvm_word;
8353
8354         if (pf->hw.partition_id != 1) {
8355                 dev_info(&pf->pdev->dev,
8356                          "Commit BW only works on partition 1! This is partition %d",
8357                          pf->hw.partition_id);
8358                 ret = I40E_NOT_SUPPORTED;
8359                 goto bw_commit_out;
8360         }
8361
8362         /* Acquire NVM for read access */
8363         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
8364         last_aq_status = pf->hw.aq.asq_last_status;
8365         if (ret) {
8366                 dev_info(&pf->pdev->dev,
8367                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
8368                          i40e_stat_str(&pf->hw, ret),
8369                          i40e_aq_str(&pf->hw, last_aq_status));
8370                 goto bw_commit_out;
8371         }
8372
8373         /* Read word 0x10 of NVM - SW compatibility word 1 */
8374         ret = i40e_aq_read_nvm(&pf->hw,
8375                                I40E_SR_NVM_CONTROL_WORD,
8376                                0x10, sizeof(nvm_word), &nvm_word,
8377                                false, NULL);
8378         /* Save off last admin queue command status before releasing
8379          * the NVM
8380          */
8381         last_aq_status = pf->hw.aq.asq_last_status;
8382         i40e_release_nvm(&pf->hw);
8383         if (ret) {
8384                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
8385                          i40e_stat_str(&pf->hw, ret),
8386                          i40e_aq_str(&pf->hw, last_aq_status));
8387                 goto bw_commit_out;
8388         }
8389
8390         /* Wait a bit for NVM release to complete */
8391         msleep(50);
8392
8393         /* Acquire NVM for write access */
8394         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
8395         last_aq_status = pf->hw.aq.asq_last_status;
8396         if (ret) {
8397                 dev_info(&pf->pdev->dev,
8398                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
8399                          i40e_stat_str(&pf->hw, ret),
8400                          i40e_aq_str(&pf->hw, last_aq_status));
8401                 goto bw_commit_out;
8402         }
8403         /* Write it back out unchanged to initiate update NVM,
8404          * which will force a write of the shadow (alt) RAM to
8405          * the NVM - thus storing the bandwidth values permanently.
8406          */
8407         ret = i40e_aq_update_nvm(&pf->hw,
8408                                  I40E_SR_NVM_CONTROL_WORD,
8409                                  0x10, sizeof(nvm_word),
8410                                  &nvm_word, true, NULL);
8411         /* Save off last admin queue command status before releasing
8412          * the NVM
8413          */
8414         last_aq_status = pf->hw.aq.asq_last_status;
8415         i40e_release_nvm(&pf->hw);
8416         if (ret)
8417                 dev_info(&pf->pdev->dev,
8418                          "BW settings NOT SAVED, err %s aq_err %s\n",
8419                          i40e_stat_str(&pf->hw, ret),
8420                          i40e_aq_str(&pf->hw, last_aq_status));
8421 bw_commit_out:
8422
8423         return ret;
8424 }
8425
8426 /**
8427  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
8428  * @pf: board private structure to initialize
8429  *
8430  * i40e_sw_init initializes the Adapter private data structure.
8431  * Fields are initialized based on PCI device information and
8432  * OS network device settings (MTU size).
8433  **/
8434 static int i40e_sw_init(struct i40e_pf *pf)
8435 {
8436         int err = 0;
8437         int size;
8438
8439         pf->msg_enable = netif_msg_init(I40E_DEFAULT_MSG_ENABLE,
8440                                 (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK));
8441         if (debug != -1 && debug != I40E_DEFAULT_MSG_ENABLE) {
8442                 if (I40E_DEBUG_USER & debug)
8443                         pf->hw.debug_mask = debug;
8444                 pf->msg_enable = netif_msg_init((debug & ~I40E_DEBUG_USER),
8445                                                 I40E_DEFAULT_MSG_ENABLE);
8446         }
8447
8448         /* Set default capability flags */
8449         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
8450                     I40E_FLAG_MSI_ENABLED     |
8451                     I40E_FLAG_MSIX_ENABLED;
8452
8453         if (iommu_present(&pci_bus_type))
8454                 pf->flags |= I40E_FLAG_RX_PS_ENABLED;
8455         else
8456                 pf->flags |= I40E_FLAG_RX_1BUF_ENABLED;
8457
8458         /* Set default ITR */
8459         pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF;
8460         pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF;
8461
8462         /* Depending on PF configurations, it is possible that the RSS
8463          * maximum might end up larger than the available queues
8464          */
8465         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
8466         pf->alloc_rss_size = 1;
8467         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
8468         pf->rss_size_max = min_t(int, pf->rss_size_max,
8469                                  pf->hw.func_caps.num_tx_qp);
8470         if (pf->hw.func_caps.rss) {
8471                 pf->flags |= I40E_FLAG_RSS_ENABLED;
8472                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
8473                                            num_online_cpus());
8474         }
8475
8476         /* MFP mode enabled */
8477         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
8478                 pf->flags |= I40E_FLAG_MFP_ENABLED;
8479                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
8480                 if (i40e_get_npar_bw_setting(pf))
8481                         dev_warn(&pf->pdev->dev,
8482                                  "Could not get NPAR bw settings\n");
8483                 else
8484                         dev_info(&pf->pdev->dev,
8485                                  "Min BW = %8.8x, Max BW = %8.8x\n",
8486                                  pf->npar_min_bw, pf->npar_max_bw);
8487         }
8488
8489         /* FW/NVM is not yet fixed in this regard */
8490         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
8491             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
8492                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
8493                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
8494                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
8495                     pf->hw.num_partitions > 1)
8496                         dev_info(&pf->pdev->dev,
8497                                  "Flow Director Sideband mode Disabled in MFP mode\n");
8498                 else
8499                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8500                 pf->fdir_pf_filter_count =
8501                                  pf->hw.func_caps.fd_filters_guaranteed;
8502                 pf->hw.fdir_shared_filter_count =
8503                                  pf->hw.func_caps.fd_filters_best_effort;
8504         }
8505
8506         if (i40e_is_mac_710(&pf->hw) &&
8507             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
8508             (pf->hw.aq.fw_maj_ver < 4))) {
8509                 pf->flags |= I40E_FLAG_RESTART_AUTONEG;
8510                 /* No DCB support  for FW < v4.33 */
8511                 pf->flags |= I40E_FLAG_NO_DCB_SUPPORT;
8512         }
8513
8514         /* Disable FW LLDP if FW < v4.3 */
8515         if (i40e_is_mac_710(&pf->hw) &&
8516             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
8517             (pf->hw.aq.fw_maj_ver < 4)))
8518                 pf->flags |= I40E_FLAG_STOP_FW_LLDP;
8519
8520         /* Use the FW Set LLDP MIB API if FW > v4.40 */
8521         if (i40e_is_mac_710(&pf->hw) &&
8522             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
8523             (pf->hw.aq.fw_maj_ver >= 5)))
8524                 pf->flags |= I40E_FLAG_USE_SET_LLDP_MIB;
8525
8526         if (pf->hw.func_caps.vmdq) {
8527                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
8528                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
8529                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
8530         }
8531
8532         if (pf->hw.func_caps.iwarp) {
8533                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
8534                 /* IWARP needs one extra vector for CQP just like MISC.*/
8535                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
8536         }
8537
8538 #ifdef I40E_FCOE
8539         i40e_init_pf_fcoe(pf);
8540
8541 #endif /* I40E_FCOE */
8542 #ifdef CONFIG_PCI_IOV
8543         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
8544                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
8545                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
8546                 pf->num_req_vfs = min_t(int,
8547                                         pf->hw.func_caps.num_vfs,
8548                                         I40E_MAX_VF_COUNT);
8549         }
8550 #endif /* CONFIG_PCI_IOV */
8551         if (pf->hw.mac.type == I40E_MAC_X722) {
8552                 pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE |
8553                              I40E_FLAG_128_QP_RSS_CAPABLE |
8554                              I40E_FLAG_HW_ATR_EVICT_CAPABLE |
8555                              I40E_FLAG_OUTER_UDP_CSUM_CAPABLE |
8556                              I40E_FLAG_WB_ON_ITR_CAPABLE |
8557                              I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE |
8558                              I40E_FLAG_NO_PCI_LINK_CHECK |
8559                              I40E_FLAG_100M_SGMII_CAPABLE |
8560                              I40E_FLAG_USE_SET_LLDP_MIB |
8561                              I40E_FLAG_GENEVE_OFFLOAD_CAPABLE;
8562         } else if ((pf->hw.aq.api_maj_ver > 1) ||
8563                    ((pf->hw.aq.api_maj_ver == 1) &&
8564                     (pf->hw.aq.api_min_ver > 4))) {
8565                 /* Supported in FW API version higher than 1.4 */
8566                 pf->flags |= I40E_FLAG_GENEVE_OFFLOAD_CAPABLE;
8567                 pf->auto_disable_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE;
8568         } else {
8569                 pf->auto_disable_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE;
8570         }
8571
8572         pf->eeprom_version = 0xDEAD;
8573         pf->lan_veb = I40E_NO_VEB;
8574         pf->lan_vsi = I40E_NO_VSI;
8575
8576         /* By default FW has this off for performance reasons */
8577         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
8578
8579         /* set up queue assignment tracking */
8580         size = sizeof(struct i40e_lump_tracking)
8581                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
8582         pf->qp_pile = kzalloc(size, GFP_KERNEL);
8583         if (!pf->qp_pile) {
8584                 err = -ENOMEM;
8585                 goto sw_init_done;
8586         }
8587         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
8588         pf->qp_pile->search_hint = 0;
8589
8590         pf->tx_timeout_recovery_level = 1;
8591
8592         mutex_init(&pf->switch_mutex);
8593
8594         /* If NPAR is enabled nudge the Tx scheduler */
8595         if (pf->hw.func_caps.npar_enable && (!i40e_get_npar_bw_setting(pf)))
8596                 i40e_set_npar_bw_setting(pf);
8597
8598 sw_init_done:
8599         return err;
8600 }
8601
8602 /**
8603  * i40e_set_ntuple - set the ntuple feature flag and take action
8604  * @pf: board private structure to initialize
8605  * @features: the feature set that the stack is suggesting
8606  *
8607  * returns a bool to indicate if reset needs to happen
8608  **/
8609 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
8610 {
8611         bool need_reset = false;
8612
8613         /* Check if Flow Director n-tuple support was enabled or disabled.  If
8614          * the state changed, we need to reset.
8615          */
8616         if (features & NETIF_F_NTUPLE) {
8617                 /* Enable filters and mark for reset */
8618                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
8619                         need_reset = true;
8620                 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8621         } else {
8622                 /* turn off filters, mark for reset and clear SW filter list */
8623                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8624                         need_reset = true;
8625                         i40e_fdir_filter_exit(pf);
8626                 }
8627                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8628                 pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED;
8629                 /* reset fd counters */
8630                 pf->fd_add_err = pf->fd_atr_cnt = pf->fd_tcp_rule = 0;
8631                 pf->fdir_pf_active_filters = 0;
8632                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
8633                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
8634                         dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
8635                 /* if ATR was auto disabled it can be re-enabled. */
8636                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
8637                     (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
8638                         pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
8639         }
8640         return need_reset;
8641 }
8642
8643 /**
8644  * i40e_set_features - set the netdev feature flags
8645  * @netdev: ptr to the netdev being adjusted
8646  * @features: the feature set that the stack is suggesting
8647  **/
8648 static int i40e_set_features(struct net_device *netdev,
8649                              netdev_features_t features)
8650 {
8651         struct i40e_netdev_priv *np = netdev_priv(netdev);
8652         struct i40e_vsi *vsi = np->vsi;
8653         struct i40e_pf *pf = vsi->back;
8654         bool need_reset;
8655
8656         if (features & NETIF_F_HW_VLAN_CTAG_RX)
8657                 i40e_vlan_stripping_enable(vsi);
8658         else
8659                 i40e_vlan_stripping_disable(vsi);
8660
8661         need_reset = i40e_set_ntuple(pf, features);
8662
8663         if (need_reset)
8664                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED));
8665
8666         return 0;
8667 }
8668
8669 #if IS_ENABLED(CONFIG_VXLAN) || IS_ENABLED(CONFIG_GENEVE)
8670 /**
8671  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
8672  * @pf: board private structure
8673  * @port: The UDP port to look up
8674  *
8675  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
8676  **/
8677 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, __be16 port)
8678 {
8679         u8 i;
8680
8681         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
8682                 if (pf->udp_ports[i].index == port)
8683                         return i;
8684         }
8685
8686         return i;
8687 }
8688
8689 #endif
8690
8691 #if IS_ENABLED(CONFIG_VXLAN)
8692 /**
8693  * i40e_add_vxlan_port - Get notifications about VXLAN ports that come up
8694  * @netdev: This physical port's netdev
8695  * @sa_family: Socket Family that VXLAN is notifying us about
8696  * @port: New UDP port number that VXLAN started listening to
8697  **/
8698 static void i40e_add_vxlan_port(struct net_device *netdev,
8699                                 sa_family_t sa_family, __be16 port)
8700 {
8701         struct i40e_netdev_priv *np = netdev_priv(netdev);
8702         struct i40e_vsi *vsi = np->vsi;
8703         struct i40e_pf *pf = vsi->back;
8704         u8 next_idx;
8705         u8 idx;
8706
8707         idx = i40e_get_udp_port_idx(pf, port);
8708
8709         /* Check if port already exists */
8710         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
8711                 netdev_info(netdev, "vxlan port %d already offloaded\n",
8712                             ntohs(port));
8713                 return;
8714         }
8715
8716         /* Now check if there is space to add the new port */
8717         next_idx = i40e_get_udp_port_idx(pf, 0);
8718
8719         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
8720                 netdev_info(netdev, "maximum number of vxlan UDP ports reached, not adding port %d\n",
8721                             ntohs(port));
8722                 return;
8723         }
8724
8725         /* New port: add it and mark its index in the bitmap */
8726         pf->udp_ports[next_idx].index = port;
8727         pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
8728         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
8729         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
8730 }
8731
8732 /**
8733  * i40e_del_vxlan_port - Get notifications about VXLAN ports that go away
8734  * @netdev: This physical port's netdev
8735  * @sa_family: Socket Family that VXLAN is notifying us about
8736  * @port: UDP port number that VXLAN stopped listening to
8737  **/
8738 static void i40e_del_vxlan_port(struct net_device *netdev,
8739                                 sa_family_t sa_family, __be16 port)
8740 {
8741         struct i40e_netdev_priv *np = netdev_priv(netdev);
8742         struct i40e_vsi *vsi = np->vsi;
8743         struct i40e_pf *pf = vsi->back;
8744         u8 idx;
8745
8746         idx = i40e_get_udp_port_idx(pf, port);
8747
8748         /* Check if port already exists */
8749         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
8750                 /* if port exists, set it to 0 (mark for deletion)
8751                  * and make it pending
8752                  */
8753                 pf->udp_ports[idx].index = 0;
8754                 pf->pending_udp_bitmap |= BIT_ULL(idx);
8755                 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
8756         } else {
8757                 netdev_warn(netdev, "vxlan port %d was not found, not deleting\n",
8758                             ntohs(port));
8759         }
8760 }
8761 #endif
8762
8763 #if IS_ENABLED(CONFIG_GENEVE)
8764 /**
8765  * i40e_add_geneve_port - Get notifications about GENEVE ports that come up
8766  * @netdev: This physical port's netdev
8767  * @sa_family: Socket Family that GENEVE is notifying us about
8768  * @port: New UDP port number that GENEVE started listening to
8769  **/
8770 static void i40e_add_geneve_port(struct net_device *netdev,
8771                                  sa_family_t sa_family, __be16 port)
8772 {
8773         struct i40e_netdev_priv *np = netdev_priv(netdev);
8774         struct i40e_vsi *vsi = np->vsi;
8775         struct i40e_pf *pf = vsi->back;
8776         u8 next_idx;
8777         u8 idx;
8778
8779         if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE))
8780                 return;
8781
8782         idx = i40e_get_udp_port_idx(pf, port);
8783
8784         /* Check if port already exists */
8785         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
8786                 netdev_info(netdev, "udp port %d already offloaded\n",
8787                             ntohs(port));
8788                 return;
8789         }
8790
8791         /* Now check if there is space to add the new port */
8792         next_idx = i40e_get_udp_port_idx(pf, 0);
8793
8794         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
8795                 netdev_info(netdev, "maximum number of UDP ports reached, not adding port %d\n",
8796                             ntohs(port));
8797                 return;
8798         }
8799
8800         /* New port: add it and mark its index in the bitmap */
8801         pf->udp_ports[next_idx].index = port;
8802         pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
8803         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
8804         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
8805
8806         dev_info(&pf->pdev->dev, "adding geneve port %d\n", ntohs(port));
8807 }
8808
8809 /**
8810  * i40e_del_geneve_port - Get notifications about GENEVE ports that go away
8811  * @netdev: This physical port's netdev
8812  * @sa_family: Socket Family that GENEVE is notifying us about
8813  * @port: UDP port number that GENEVE stopped listening to
8814  **/
8815 static void i40e_del_geneve_port(struct net_device *netdev,
8816                                  sa_family_t sa_family, __be16 port)
8817 {
8818         struct i40e_netdev_priv *np = netdev_priv(netdev);
8819         struct i40e_vsi *vsi = np->vsi;
8820         struct i40e_pf *pf = vsi->back;
8821         u8 idx;
8822
8823         if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE))
8824                 return;
8825
8826         idx = i40e_get_udp_port_idx(pf, port);
8827
8828         /* Check if port already exists */
8829         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
8830                 /* if port exists, set it to 0 (mark for deletion)
8831                  * and make it pending
8832                  */
8833                 pf->udp_ports[idx].index = 0;
8834                 pf->pending_udp_bitmap |= BIT_ULL(idx);
8835                 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
8836
8837                 dev_info(&pf->pdev->dev, "deleting geneve port %d\n",
8838                          ntohs(port));
8839         } else {
8840                 netdev_warn(netdev, "geneve port %d was not found, not deleting\n",
8841                             ntohs(port));
8842         }
8843 }
8844 #endif
8845
8846 static int i40e_get_phys_port_id(struct net_device *netdev,
8847                                  struct netdev_phys_item_id *ppid)
8848 {
8849         struct i40e_netdev_priv *np = netdev_priv(netdev);
8850         struct i40e_pf *pf = np->vsi->back;
8851         struct i40e_hw *hw = &pf->hw;
8852
8853         if (!(pf->flags & I40E_FLAG_PORT_ID_VALID))
8854                 return -EOPNOTSUPP;
8855
8856         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
8857         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
8858
8859         return 0;
8860 }
8861
8862 /**
8863  * i40e_ndo_fdb_add - add an entry to the hardware database
8864  * @ndm: the input from the stack
8865  * @tb: pointer to array of nladdr (unused)
8866  * @dev: the net device pointer
8867  * @addr: the MAC address entry being added
8868  * @flags: instructions from stack about fdb operation
8869  */
8870 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
8871                             struct net_device *dev,
8872                             const unsigned char *addr, u16 vid,
8873                             u16 flags)
8874 {
8875         struct i40e_netdev_priv *np = netdev_priv(dev);
8876         struct i40e_pf *pf = np->vsi->back;
8877         int err = 0;
8878
8879         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
8880                 return -EOPNOTSUPP;
8881
8882         if (vid) {
8883                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
8884                 return -EINVAL;
8885         }
8886
8887         /* Hardware does not support aging addresses so if a
8888          * ndm_state is given only allow permanent addresses
8889          */
8890         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
8891                 netdev_info(dev, "FDB only supports static addresses\n");
8892                 return -EINVAL;
8893         }
8894
8895         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
8896                 err = dev_uc_add_excl(dev, addr);
8897         else if (is_multicast_ether_addr(addr))
8898                 err = dev_mc_add_excl(dev, addr);
8899         else
8900                 err = -EINVAL;
8901
8902         /* Only return duplicate errors if NLM_F_EXCL is set */
8903         if (err == -EEXIST && !(flags & NLM_F_EXCL))
8904                 err = 0;
8905
8906         return err;
8907 }
8908
8909 /**
8910  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
8911  * @dev: the netdev being configured
8912  * @nlh: RTNL message
8913  *
8914  * Inserts a new hardware bridge if not already created and
8915  * enables the bridging mode requested (VEB or VEPA). If the
8916  * hardware bridge has already been inserted and the request
8917  * is to change the mode then that requires a PF reset to
8918  * allow rebuild of the components with required hardware
8919  * bridge mode enabled.
8920  **/
8921 static int i40e_ndo_bridge_setlink(struct net_device *dev,
8922                                    struct nlmsghdr *nlh,
8923                                    u16 flags)
8924 {
8925         struct i40e_netdev_priv *np = netdev_priv(dev);
8926         struct i40e_vsi *vsi = np->vsi;
8927         struct i40e_pf *pf = vsi->back;
8928         struct i40e_veb *veb = NULL;
8929         struct nlattr *attr, *br_spec;
8930         int i, rem;
8931
8932         /* Only for PF VSI for now */
8933         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
8934                 return -EOPNOTSUPP;
8935
8936         /* Find the HW bridge for PF VSI */
8937         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
8938                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
8939                         veb = pf->veb[i];
8940         }
8941
8942         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
8943
8944         nla_for_each_nested(attr, br_spec, rem) {
8945                 __u16 mode;
8946
8947                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
8948                         continue;
8949
8950                 mode = nla_get_u16(attr);
8951                 if ((mode != BRIDGE_MODE_VEPA) &&
8952                     (mode != BRIDGE_MODE_VEB))
8953                         return -EINVAL;
8954
8955                 /* Insert a new HW bridge */
8956                 if (!veb) {
8957                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
8958                                              vsi->tc_config.enabled_tc);
8959                         if (veb) {
8960                                 veb->bridge_mode = mode;
8961                                 i40e_config_bridge_mode(veb);
8962                         } else {
8963                                 /* No Bridge HW offload available */
8964                                 return -ENOENT;
8965                         }
8966                         break;
8967                 } else if (mode != veb->bridge_mode) {
8968                         /* Existing HW bridge but different mode needs reset */
8969                         veb->bridge_mode = mode;
8970                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
8971                         if (mode == BRIDGE_MODE_VEB)
8972                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
8973                         else
8974                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
8975                         i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED));
8976                         break;
8977                 }
8978         }
8979
8980         return 0;
8981 }
8982
8983 /**
8984  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
8985  * @skb: skb buff
8986  * @pid: process id
8987  * @seq: RTNL message seq #
8988  * @dev: the netdev being configured
8989  * @filter_mask: unused
8990  * @nlflags: netlink flags passed in
8991  *
8992  * Return the mode in which the hardware bridge is operating in
8993  * i.e VEB or VEPA.
8994  **/
8995 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
8996                                    struct net_device *dev,
8997                                    u32 __always_unused filter_mask,
8998                                    int nlflags)
8999 {
9000         struct i40e_netdev_priv *np = netdev_priv(dev);
9001         struct i40e_vsi *vsi = np->vsi;
9002         struct i40e_pf *pf = vsi->back;
9003         struct i40e_veb *veb = NULL;
9004         int i;
9005
9006         /* Only for PF VSI for now */
9007         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9008                 return -EOPNOTSUPP;
9009
9010         /* Find the HW bridge for the PF VSI */
9011         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9012                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9013                         veb = pf->veb[i];
9014         }
9015
9016         if (!veb)
9017                 return 0;
9018
9019         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
9020                                        nlflags, 0, 0, filter_mask, NULL);
9021 }
9022
9023 /* Hardware supports L4 tunnel length of 128B (=2^7) which includes
9024  * inner mac plus all inner ethertypes.
9025  */
9026 #define I40E_MAX_TUNNEL_HDR_LEN 128
9027 /**
9028  * i40e_features_check - Validate encapsulated packet conforms to limits
9029  * @skb: skb buff
9030  * @dev: This physical port's netdev
9031  * @features: Offload features that the stack believes apply
9032  **/
9033 static netdev_features_t i40e_features_check(struct sk_buff *skb,
9034                                              struct net_device *dev,
9035                                              netdev_features_t features)
9036 {
9037         if (skb->encapsulation &&
9038             ((skb_inner_network_header(skb) - skb_transport_header(skb)) >
9039              I40E_MAX_TUNNEL_HDR_LEN))
9040                 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
9041
9042         return features;
9043 }
9044
9045 static const struct net_device_ops i40e_netdev_ops = {
9046         .ndo_open               = i40e_open,
9047         .ndo_stop               = i40e_close,
9048         .ndo_start_xmit         = i40e_lan_xmit_frame,
9049         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
9050         .ndo_set_rx_mode        = i40e_set_rx_mode,
9051         .ndo_validate_addr      = eth_validate_addr,
9052         .ndo_set_mac_address    = i40e_set_mac,
9053         .ndo_change_mtu         = i40e_change_mtu,
9054         .ndo_do_ioctl           = i40e_ioctl,
9055         .ndo_tx_timeout         = i40e_tx_timeout,
9056         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
9057         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
9058 #ifdef CONFIG_NET_POLL_CONTROLLER
9059         .ndo_poll_controller    = i40e_netpoll,
9060 #endif
9061         .ndo_setup_tc           = __i40e_setup_tc,
9062 #ifdef I40E_FCOE
9063         .ndo_fcoe_enable        = i40e_fcoe_enable,
9064         .ndo_fcoe_disable       = i40e_fcoe_disable,
9065 #endif
9066         .ndo_set_features       = i40e_set_features,
9067         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
9068         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
9069         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
9070         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
9071         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
9072         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
9073         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
9074 #if IS_ENABLED(CONFIG_VXLAN)
9075         .ndo_add_vxlan_port     = i40e_add_vxlan_port,
9076         .ndo_del_vxlan_port     = i40e_del_vxlan_port,
9077 #endif
9078 #if IS_ENABLED(CONFIG_GENEVE)
9079         .ndo_add_geneve_port    = i40e_add_geneve_port,
9080         .ndo_del_geneve_port    = i40e_del_geneve_port,
9081 #endif
9082         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
9083         .ndo_fdb_add            = i40e_ndo_fdb_add,
9084         .ndo_features_check     = i40e_features_check,
9085         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
9086         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
9087 };
9088
9089 /**
9090  * i40e_config_netdev - Setup the netdev flags
9091  * @vsi: the VSI being configured
9092  *
9093  * Returns 0 on success, negative value on failure
9094  **/
9095 static int i40e_config_netdev(struct i40e_vsi *vsi)
9096 {
9097         u8 brdcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
9098         struct i40e_pf *pf = vsi->back;
9099         struct i40e_hw *hw = &pf->hw;
9100         struct i40e_netdev_priv *np;
9101         struct net_device *netdev;
9102         u8 mac_addr[ETH_ALEN];
9103         int etherdev_size;
9104
9105         etherdev_size = sizeof(struct i40e_netdev_priv);
9106         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
9107         if (!netdev)
9108                 return -ENOMEM;
9109
9110         vsi->netdev = netdev;
9111         np = netdev_priv(netdev);
9112         np->vsi = vsi;
9113
9114         netdev->hw_enc_features |= NETIF_F_SG                   |
9115                                    NETIF_F_IP_CSUM              |
9116                                    NETIF_F_IPV6_CSUM            |
9117                                    NETIF_F_HIGHDMA              |
9118                                    NETIF_F_SOFT_FEATURES        |
9119                                    NETIF_F_TSO                  |
9120                                    NETIF_F_TSO_ECN              |
9121                                    NETIF_F_TSO6                 |
9122                                    NETIF_F_GSO_GRE              |
9123                                    NETIF_F_GSO_UDP_TUNNEL       |
9124                                    NETIF_F_GSO_UDP_TUNNEL_CSUM  |
9125                                    NETIF_F_SCTP_CRC             |
9126                                    NETIF_F_RXHASH               |
9127                                    NETIF_F_RXCSUM               |
9128                                    0;
9129
9130         if (!(pf->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE))
9131                 netdev->hw_enc_features ^= NETIF_F_GSO_UDP_TUNNEL_CSUM;
9132
9133         /* record features VLANs can make use of */
9134         netdev->vlan_features |= netdev->hw_enc_features;
9135
9136         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
9137                 netdev->hw_features |= NETIF_F_NTUPLE;
9138
9139         netdev->hw_features |= netdev->hw_enc_features  |
9140                                NETIF_F_HW_VLAN_CTAG_TX  |
9141                                NETIF_F_HW_VLAN_CTAG_RX;
9142
9143         netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
9144
9145         if (vsi->type == I40E_VSI_MAIN) {
9146                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
9147                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
9148                 /* The following steps are necessary to prevent reception
9149                  * of tagged packets - some older NVM configurations load a
9150                  * default a MAC-VLAN filter that accepts any tagged packet
9151                  * which must be replaced by a normal filter.
9152                  */
9153                 if (!i40e_rm_default_mac_filter(vsi, mac_addr)) {
9154                         spin_lock_bh(&vsi->mac_filter_list_lock);
9155                         i40e_add_filter(vsi, mac_addr,
9156                                         I40E_VLAN_ANY, false, true);
9157                         spin_unlock_bh(&vsi->mac_filter_list_lock);
9158                 }
9159         } else if ((pf->hw.aq.api_maj_ver > 1) ||
9160                    ((pf->hw.aq.api_maj_ver == 1) &&
9161                     (pf->hw.aq.api_min_ver > 4))) {
9162                 /* Supported in FW API version higher than 1.4 */
9163                 pf->flags |= I40E_FLAG_GENEVE_OFFLOAD_CAPABLE;
9164                 pf->auto_disable_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE;
9165         } else {
9166                 /* relate the VSI_VMDQ name to the VSI_MAIN name */
9167                 snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
9168                          pf->vsi[pf->lan_vsi]->netdev->name);
9169                 random_ether_addr(mac_addr);
9170
9171                 spin_lock_bh(&vsi->mac_filter_list_lock);
9172                 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY, false, false);
9173                 spin_unlock_bh(&vsi->mac_filter_list_lock);
9174         }
9175
9176         spin_lock_bh(&vsi->mac_filter_list_lock);
9177         i40e_add_filter(vsi, brdcast, I40E_VLAN_ANY, false, false);
9178         spin_unlock_bh(&vsi->mac_filter_list_lock);
9179
9180         ether_addr_copy(netdev->dev_addr, mac_addr);
9181         ether_addr_copy(netdev->perm_addr, mac_addr);
9182
9183         netdev->priv_flags |= IFF_UNICAST_FLT;
9184         netdev->priv_flags |= IFF_SUPP_NOFCS;
9185         /* Setup netdev TC information */
9186         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
9187
9188         netdev->netdev_ops = &i40e_netdev_ops;
9189         netdev->watchdog_timeo = 5 * HZ;
9190         i40e_set_ethtool_ops(netdev);
9191 #ifdef I40E_FCOE
9192         i40e_fcoe_config_netdev(netdev, vsi);
9193 #endif
9194
9195         return 0;
9196 }
9197
9198 /**
9199  * i40e_vsi_delete - Delete a VSI from the switch
9200  * @vsi: the VSI being removed
9201  *
9202  * Returns 0 on success, negative value on failure
9203  **/
9204 static void i40e_vsi_delete(struct i40e_vsi *vsi)
9205 {
9206         /* remove default VSI is not allowed */
9207         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
9208                 return;
9209
9210         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
9211 }
9212
9213 /**
9214  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
9215  * @vsi: the VSI being queried
9216  *
9217  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
9218  **/
9219 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
9220 {
9221         struct i40e_veb *veb;
9222         struct i40e_pf *pf = vsi->back;
9223
9224         /* Uplink is not a bridge so default to VEB */
9225         if (vsi->veb_idx == I40E_NO_VEB)
9226                 return 1;
9227
9228         veb = pf->veb[vsi->veb_idx];
9229         if (!veb) {
9230                 dev_info(&pf->pdev->dev,
9231                          "There is no veb associated with the bridge\n");
9232                 return -ENOENT;
9233         }
9234
9235         /* Uplink is a bridge in VEPA mode */
9236         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
9237                 return 0;
9238         } else {
9239                 /* Uplink is a bridge in VEB mode */
9240                 return 1;
9241         }
9242
9243         /* VEPA is now default bridge, so return 0 */
9244         return 0;
9245 }
9246
9247 /**
9248  * i40e_add_vsi - Add a VSI to the switch
9249  * @vsi: the VSI being configured
9250  *
9251  * This initializes a VSI context depending on the VSI type to be added and
9252  * passes it down to the add_vsi aq command.
9253  **/
9254 static int i40e_add_vsi(struct i40e_vsi *vsi)
9255 {
9256         int ret = -ENODEV;
9257         u8 laa_macaddr[ETH_ALEN];
9258         bool found_laa_mac_filter = false;
9259         struct i40e_pf *pf = vsi->back;
9260         struct i40e_hw *hw = &pf->hw;
9261         struct i40e_vsi_context ctxt;
9262         struct i40e_mac_filter *f, *ftmp;
9263
9264         u8 enabled_tc = 0x1; /* TC0 enabled */
9265         int f_count = 0;
9266
9267         memset(&ctxt, 0, sizeof(ctxt));
9268         switch (vsi->type) {
9269         case I40E_VSI_MAIN:
9270                 /* The PF's main VSI is already setup as part of the
9271                  * device initialization, so we'll not bother with
9272                  * the add_vsi call, but we will retrieve the current
9273                  * VSI context.
9274                  */
9275                 ctxt.seid = pf->main_vsi_seid;
9276                 ctxt.pf_num = pf->hw.pf_id;
9277                 ctxt.vf_num = 0;
9278                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9279                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9280                 if (ret) {
9281                         dev_info(&pf->pdev->dev,
9282                                  "couldn't get PF vsi config, err %s aq_err %s\n",
9283                                  i40e_stat_str(&pf->hw, ret),
9284                                  i40e_aq_str(&pf->hw,
9285                                              pf->hw.aq.asq_last_status));
9286                         return -ENOENT;
9287                 }
9288                 vsi->info = ctxt.info;
9289                 vsi->info.valid_sections = 0;
9290
9291                 vsi->seid = ctxt.seid;
9292                 vsi->id = ctxt.vsi_number;
9293
9294                 enabled_tc = i40e_pf_get_tc_map(pf);
9295
9296                 /* MFP mode setup queue map and update VSI */
9297                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
9298                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
9299                         memset(&ctxt, 0, sizeof(ctxt));
9300                         ctxt.seid = pf->main_vsi_seid;
9301                         ctxt.pf_num = pf->hw.pf_id;
9302                         ctxt.vf_num = 0;
9303                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
9304                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
9305                         if (ret) {
9306                                 dev_info(&pf->pdev->dev,
9307                                          "update vsi failed, err %s aq_err %s\n",
9308                                          i40e_stat_str(&pf->hw, ret),
9309                                          i40e_aq_str(&pf->hw,
9310                                                     pf->hw.aq.asq_last_status));
9311                                 ret = -ENOENT;
9312                                 goto err;
9313                         }
9314                         /* update the local VSI info queue map */
9315                         i40e_vsi_update_queue_map(vsi, &ctxt);
9316                         vsi->info.valid_sections = 0;
9317                 } else {
9318                         /* Default/Main VSI is only enabled for TC0
9319                          * reconfigure it to enable all TCs that are
9320                          * available on the port in SFP mode.
9321                          * For MFP case the iSCSI PF would use this
9322                          * flow to enable LAN+iSCSI TC.
9323                          */
9324                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
9325                         if (ret) {
9326                                 dev_info(&pf->pdev->dev,
9327                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
9328                                          enabled_tc,
9329                                          i40e_stat_str(&pf->hw, ret),
9330                                          i40e_aq_str(&pf->hw,
9331                                                     pf->hw.aq.asq_last_status));
9332                                 ret = -ENOENT;
9333                         }
9334                 }
9335                 break;
9336
9337         case I40E_VSI_FDIR:
9338                 ctxt.pf_num = hw->pf_id;
9339                 ctxt.vf_num = 0;
9340                 ctxt.uplink_seid = vsi->uplink_seid;
9341                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9342                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9343                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
9344                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
9345                         ctxt.info.valid_sections |=
9346                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9347                         ctxt.info.switch_id =
9348                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9349                 }
9350                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9351                 break;
9352
9353         case I40E_VSI_VMDQ2:
9354                 ctxt.pf_num = hw->pf_id;
9355                 ctxt.vf_num = 0;
9356                 ctxt.uplink_seid = vsi->uplink_seid;
9357                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9358                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
9359
9360                 /* This VSI is connected to VEB so the switch_id
9361                  * should be set to zero by default.
9362                  */
9363                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9364                         ctxt.info.valid_sections |=
9365                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9366                         ctxt.info.switch_id =
9367                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9368                 }
9369
9370                 /* Setup the VSI tx/rx queue map for TC0 only for now */
9371                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9372                 break;
9373
9374         case I40E_VSI_SRIOV:
9375                 ctxt.pf_num = hw->pf_id;
9376                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
9377                 ctxt.uplink_seid = vsi->uplink_seid;
9378                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9379                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
9380
9381                 /* This VSI is connected to VEB so the switch_id
9382                  * should be set to zero by default.
9383                  */
9384                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9385                         ctxt.info.valid_sections |=
9386                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9387                         ctxt.info.switch_id =
9388                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9389                 }
9390
9391                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
9392                         ctxt.info.valid_sections |=
9393                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
9394                         ctxt.info.queueing_opt_flags |=
9395                                                 I40E_AQ_VSI_QUE_OPT_TCP_ENA;
9396                 }
9397
9398                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
9399                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
9400                 if (pf->vf[vsi->vf_id].spoofchk) {
9401                         ctxt.info.valid_sections |=
9402                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
9403                         ctxt.info.sec_flags |=
9404                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
9405                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
9406                 }
9407                 /* Setup the VSI tx/rx queue map for TC0 only for now */
9408                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9409                 break;
9410
9411 #ifdef I40E_FCOE
9412         case I40E_VSI_FCOE:
9413                 ret = i40e_fcoe_vsi_init(vsi, &ctxt);
9414                 if (ret) {
9415                         dev_info(&pf->pdev->dev, "failed to initialize FCoE VSI\n");
9416                         return ret;
9417                 }
9418                 break;
9419
9420 #endif /* I40E_FCOE */
9421         case I40E_VSI_IWARP:
9422                 /* send down message to iWARP */
9423                 break;
9424
9425         default:
9426                 return -ENODEV;
9427         }
9428
9429         if (vsi->type != I40E_VSI_MAIN) {
9430                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
9431                 if (ret) {
9432                         dev_info(&vsi->back->pdev->dev,
9433                                  "add vsi failed, err %s aq_err %s\n",
9434                                  i40e_stat_str(&pf->hw, ret),
9435                                  i40e_aq_str(&pf->hw,
9436                                              pf->hw.aq.asq_last_status));
9437                         ret = -ENOENT;
9438                         goto err;
9439                 }
9440                 vsi->info = ctxt.info;
9441                 vsi->info.valid_sections = 0;
9442                 vsi->seid = ctxt.seid;
9443                 vsi->id = ctxt.vsi_number;
9444         }
9445
9446         spin_lock_bh(&vsi->mac_filter_list_lock);
9447         /* If macvlan filters already exist, force them to get loaded */
9448         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
9449                 f->changed = true;
9450                 f_count++;
9451
9452                 /* Expected to have only one MAC filter entry for LAA in list */
9453                 if (f->is_laa && vsi->type == I40E_VSI_MAIN) {
9454                         ether_addr_copy(laa_macaddr, f->macaddr);
9455                         found_laa_mac_filter = true;
9456                 }
9457         }
9458         spin_unlock_bh(&vsi->mac_filter_list_lock);
9459
9460         if (found_laa_mac_filter) {
9461                 struct i40e_aqc_remove_macvlan_element_data element;
9462
9463                 memset(&element, 0, sizeof(element));
9464                 ether_addr_copy(element.mac_addr, laa_macaddr);
9465                 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
9466                 ret = i40e_aq_remove_macvlan(hw, vsi->seid,
9467                                              &element, 1, NULL);
9468                 if (ret) {
9469                         /* some older FW has a different default */
9470                         element.flags |=
9471                                        I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
9472                         i40e_aq_remove_macvlan(hw, vsi->seid,
9473                                                &element, 1, NULL);
9474                 }
9475
9476                 i40e_aq_mac_address_write(hw,
9477                                           I40E_AQC_WRITE_TYPE_LAA_WOL,
9478                                           laa_macaddr, NULL);
9479         }
9480
9481         if (f_count) {
9482                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
9483                 pf->flags |= I40E_FLAG_FILTER_SYNC;
9484         }
9485
9486         /* Update VSI BW information */
9487         ret = i40e_vsi_get_bw_info(vsi);
9488         if (ret) {
9489                 dev_info(&pf->pdev->dev,
9490                          "couldn't get vsi bw info, err %s aq_err %s\n",
9491                          i40e_stat_str(&pf->hw, ret),
9492                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9493                 /* VSI is already added so not tearing that up */
9494                 ret = 0;
9495         }
9496
9497 err:
9498         return ret;
9499 }
9500
9501 /**
9502  * i40e_vsi_release - Delete a VSI and free its resources
9503  * @vsi: the VSI being removed
9504  *
9505  * Returns 0 on success or < 0 on error
9506  **/
9507 int i40e_vsi_release(struct i40e_vsi *vsi)
9508 {
9509         struct i40e_mac_filter *f, *ftmp;
9510         struct i40e_veb *veb = NULL;
9511         struct i40e_pf *pf;
9512         u16 uplink_seid;
9513         int i, n;
9514
9515         pf = vsi->back;
9516
9517         /* release of a VEB-owner or last VSI is not allowed */
9518         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
9519                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
9520                          vsi->seid, vsi->uplink_seid);
9521                 return -ENODEV;
9522         }
9523         if (vsi == pf->vsi[pf->lan_vsi] &&
9524             !test_bit(__I40E_DOWN, &pf->state)) {
9525                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
9526                 return -ENODEV;
9527         }
9528
9529         uplink_seid = vsi->uplink_seid;
9530         if (vsi->type != I40E_VSI_SRIOV) {
9531                 if (vsi->netdev_registered) {
9532                         vsi->netdev_registered = false;
9533                         if (vsi->netdev) {
9534                                 /* results in a call to i40e_close() */
9535                                 unregister_netdev(vsi->netdev);
9536                         }
9537                 } else {
9538                         i40e_vsi_close(vsi);
9539                 }
9540                 i40e_vsi_disable_irq(vsi);
9541         }
9542
9543         spin_lock_bh(&vsi->mac_filter_list_lock);
9544         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list)
9545                 i40e_del_filter(vsi, f->macaddr, f->vlan,
9546                                 f->is_vf, f->is_netdev);
9547         spin_unlock_bh(&vsi->mac_filter_list_lock);
9548
9549         i40e_sync_vsi_filters(vsi);
9550
9551         i40e_vsi_delete(vsi);
9552         i40e_vsi_free_q_vectors(vsi);
9553         if (vsi->netdev) {
9554                 free_netdev(vsi->netdev);
9555                 vsi->netdev = NULL;
9556         }
9557         i40e_vsi_clear_rings(vsi);
9558         i40e_vsi_clear(vsi);
9559
9560         /* If this was the last thing on the VEB, except for the
9561          * controlling VSI, remove the VEB, which puts the controlling
9562          * VSI onto the next level down in the switch.
9563          *
9564          * Well, okay, there's one more exception here: don't remove
9565          * the orphan VEBs yet.  We'll wait for an explicit remove request
9566          * from up the network stack.
9567          */
9568         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
9569                 if (pf->vsi[i] &&
9570                     pf->vsi[i]->uplink_seid == uplink_seid &&
9571                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
9572                         n++;      /* count the VSIs */
9573                 }
9574         }
9575         for (i = 0; i < I40E_MAX_VEB; i++) {
9576                 if (!pf->veb[i])
9577                         continue;
9578                 if (pf->veb[i]->uplink_seid == uplink_seid)
9579                         n++;     /* count the VEBs */
9580                 if (pf->veb[i]->seid == uplink_seid)
9581                         veb = pf->veb[i];
9582         }
9583         if (n == 0 && veb && veb->uplink_seid != 0)
9584                 i40e_veb_release(veb);
9585
9586         return 0;
9587 }
9588
9589 /**
9590  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
9591  * @vsi: ptr to the VSI
9592  *
9593  * This should only be called after i40e_vsi_mem_alloc() which allocates the
9594  * corresponding SW VSI structure and initializes num_queue_pairs for the
9595  * newly allocated VSI.
9596  *
9597  * Returns 0 on success or negative on failure
9598  **/
9599 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
9600 {
9601         int ret = -ENOENT;
9602         struct i40e_pf *pf = vsi->back;
9603
9604         if (vsi->q_vectors[0]) {
9605                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
9606                          vsi->seid);
9607                 return -EEXIST;
9608         }
9609
9610         if (vsi->base_vector) {
9611                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
9612                          vsi->seid, vsi->base_vector);
9613                 return -EEXIST;
9614         }
9615
9616         ret = i40e_vsi_alloc_q_vectors(vsi);
9617         if (ret) {
9618                 dev_info(&pf->pdev->dev,
9619                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
9620                          vsi->num_q_vectors, vsi->seid, ret);
9621                 vsi->num_q_vectors = 0;
9622                 goto vector_setup_out;
9623         }
9624
9625         /* In Legacy mode, we do not have to get any other vector since we
9626          * piggyback on the misc/ICR0 for queue interrupts.
9627         */
9628         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
9629                 return ret;
9630         if (vsi->num_q_vectors)
9631                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
9632                                                  vsi->num_q_vectors, vsi->idx);
9633         if (vsi->base_vector < 0) {
9634                 dev_info(&pf->pdev->dev,
9635                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
9636                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
9637                 i40e_vsi_free_q_vectors(vsi);
9638                 ret = -ENOENT;
9639                 goto vector_setup_out;
9640         }
9641
9642 vector_setup_out:
9643         return ret;
9644 }
9645
9646 /**
9647  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
9648  * @vsi: pointer to the vsi.
9649  *
9650  * This re-allocates a vsi's queue resources.
9651  *
9652  * Returns pointer to the successfully allocated and configured VSI sw struct
9653  * on success, otherwise returns NULL on failure.
9654  **/
9655 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
9656 {
9657         struct i40e_pf *pf;
9658         u8 enabled_tc;
9659         int ret;
9660
9661         if (!vsi)
9662                 return NULL;
9663
9664         pf = vsi->back;
9665
9666         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
9667         i40e_vsi_clear_rings(vsi);
9668
9669         i40e_vsi_free_arrays(vsi, false);
9670         i40e_set_num_rings_in_vsi(vsi);
9671         ret = i40e_vsi_alloc_arrays(vsi, false);
9672         if (ret)
9673                 goto err_vsi;
9674
9675         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
9676         if (ret < 0) {
9677                 dev_info(&pf->pdev->dev,
9678                          "failed to get tracking for %d queues for VSI %d err %d\n",
9679                          vsi->alloc_queue_pairs, vsi->seid, ret);
9680                 goto err_vsi;
9681         }
9682         vsi->base_queue = ret;
9683
9684         /* Update the FW view of the VSI. Force a reset of TC and queue
9685          * layout configurations.
9686          */
9687         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
9688         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
9689         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
9690         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
9691
9692         /* assign it some queues */
9693         ret = i40e_alloc_rings(vsi);
9694         if (ret)
9695                 goto err_rings;
9696
9697         /* map all of the rings to the q_vectors */
9698         i40e_vsi_map_rings_to_vectors(vsi);
9699         return vsi;
9700
9701 err_rings:
9702         i40e_vsi_free_q_vectors(vsi);
9703         if (vsi->netdev_registered) {
9704                 vsi->netdev_registered = false;
9705                 unregister_netdev(vsi->netdev);
9706                 free_netdev(vsi->netdev);
9707                 vsi->netdev = NULL;
9708         }
9709         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
9710 err_vsi:
9711         i40e_vsi_clear(vsi);
9712         return NULL;
9713 }
9714
9715 /**
9716  * i40e_macaddr_init - explicitly write the mac address filters.
9717  *
9718  * @vsi: pointer to the vsi.
9719  * @macaddr: the MAC address
9720  *
9721  * This is needed when the macaddr has been obtained by other
9722  * means than the default, e.g., from Open Firmware or IDPROM.
9723  * Returns 0 on success, negative on failure
9724  **/
9725 static int i40e_macaddr_init(struct i40e_vsi *vsi, u8 *macaddr)
9726 {
9727         int ret;
9728         struct i40e_aqc_add_macvlan_element_data element;
9729
9730         ret = i40e_aq_mac_address_write(&vsi->back->hw,
9731                                         I40E_AQC_WRITE_TYPE_LAA_WOL,
9732                                         macaddr, NULL);
9733         if (ret) {
9734                 dev_info(&vsi->back->pdev->dev,
9735                          "Addr change for VSI failed: %d\n", ret);
9736                 return -EADDRNOTAVAIL;
9737         }
9738
9739         memset(&element, 0, sizeof(element));
9740         ether_addr_copy(element.mac_addr, macaddr);
9741         element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH);
9742         ret = i40e_aq_add_macvlan(&vsi->back->hw, vsi->seid, &element, 1, NULL);
9743         if (ret) {
9744                 dev_info(&vsi->back->pdev->dev,
9745                          "add filter failed err %s aq_err %s\n",
9746                          i40e_stat_str(&vsi->back->hw, ret),
9747                          i40e_aq_str(&vsi->back->hw,
9748                                      vsi->back->hw.aq.asq_last_status));
9749         }
9750         return ret;
9751 }
9752
9753 /**
9754  * i40e_vsi_setup - Set up a VSI by a given type
9755  * @pf: board private structure
9756  * @type: VSI type
9757  * @uplink_seid: the switch element to link to
9758  * @param1: usage depends upon VSI type. For VF types, indicates VF id
9759  *
9760  * This allocates the sw VSI structure and its queue resources, then add a VSI
9761  * to the identified VEB.
9762  *
9763  * Returns pointer to the successfully allocated and configure VSI sw struct on
9764  * success, otherwise returns NULL on failure.
9765  **/
9766 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
9767                                 u16 uplink_seid, u32 param1)
9768 {
9769         struct i40e_vsi *vsi = NULL;
9770         struct i40e_veb *veb = NULL;
9771         int ret, i;
9772         int v_idx;
9773
9774         /* The requested uplink_seid must be either
9775          *     - the PF's port seid
9776          *              no VEB is needed because this is the PF
9777          *              or this is a Flow Director special case VSI
9778          *     - seid of an existing VEB
9779          *     - seid of a VSI that owns an existing VEB
9780          *     - seid of a VSI that doesn't own a VEB
9781          *              a new VEB is created and the VSI becomes the owner
9782          *     - seid of the PF VSI, which is what creates the first VEB
9783          *              this is a special case of the previous
9784          *
9785          * Find which uplink_seid we were given and create a new VEB if needed
9786          */
9787         for (i = 0; i < I40E_MAX_VEB; i++) {
9788                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
9789                         veb = pf->veb[i];
9790                         break;
9791                 }
9792         }
9793
9794         if (!veb && uplink_seid != pf->mac_seid) {
9795
9796                 for (i = 0; i < pf->num_alloc_vsi; i++) {
9797                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
9798                                 vsi = pf->vsi[i];
9799                                 break;
9800                         }
9801                 }
9802                 if (!vsi) {
9803                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
9804                                  uplink_seid);
9805                         return NULL;
9806                 }
9807
9808                 if (vsi->uplink_seid == pf->mac_seid)
9809                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
9810                                              vsi->tc_config.enabled_tc);
9811                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
9812                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
9813                                              vsi->tc_config.enabled_tc);
9814                 if (veb) {
9815                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
9816                                 dev_info(&vsi->back->pdev->dev,
9817                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
9818                                 return NULL;
9819                         }
9820                         /* We come up by default in VEPA mode if SRIOV is not
9821                          * already enabled, in which case we can't force VEPA
9822                          * mode.
9823                          */
9824                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
9825                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
9826                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
9827                         }
9828                         i40e_config_bridge_mode(veb);
9829                 }
9830                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9831                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9832                                 veb = pf->veb[i];
9833                 }
9834                 if (!veb) {
9835                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
9836                         return NULL;
9837                 }
9838
9839                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
9840                 uplink_seid = veb->seid;
9841         }
9842
9843         /* get vsi sw struct */
9844         v_idx = i40e_vsi_mem_alloc(pf, type);
9845         if (v_idx < 0)
9846                 goto err_alloc;
9847         vsi = pf->vsi[v_idx];
9848         if (!vsi)
9849                 goto err_alloc;
9850         vsi->type = type;
9851         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
9852
9853         if (type == I40E_VSI_MAIN)
9854                 pf->lan_vsi = v_idx;
9855         else if (type == I40E_VSI_SRIOV)
9856                 vsi->vf_id = param1;
9857         /* assign it some queues */
9858         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs,
9859                                 vsi->idx);
9860         if (ret < 0) {
9861                 dev_info(&pf->pdev->dev,
9862                          "failed to get tracking for %d queues for VSI %d err=%d\n",
9863                          vsi->alloc_queue_pairs, vsi->seid, ret);
9864                 goto err_vsi;
9865         }
9866         vsi->base_queue = ret;
9867
9868         /* get a VSI from the hardware */
9869         vsi->uplink_seid = uplink_seid;
9870         ret = i40e_add_vsi(vsi);
9871         if (ret)
9872                 goto err_vsi;
9873
9874         switch (vsi->type) {
9875         /* setup the netdev if needed */
9876         case I40E_VSI_MAIN:
9877                 /* Apply relevant filters if a platform-specific mac
9878                  * address was selected.
9879                  */
9880                 if (!!(pf->flags & I40E_FLAG_PF_MAC)) {
9881                         ret = i40e_macaddr_init(vsi, pf->hw.mac.addr);
9882                         if (ret) {
9883                                 dev_warn(&pf->pdev->dev,
9884                                          "could not set up macaddr; err %d\n",
9885                                          ret);
9886                         }
9887                 }
9888         case I40E_VSI_VMDQ2:
9889         case I40E_VSI_FCOE:
9890                 ret = i40e_config_netdev(vsi);
9891                 if (ret)
9892                         goto err_netdev;
9893                 ret = register_netdev(vsi->netdev);
9894                 if (ret)
9895                         goto err_netdev;
9896                 vsi->netdev_registered = true;
9897                 netif_carrier_off(vsi->netdev);
9898 #ifdef CONFIG_I40E_DCB
9899                 /* Setup DCB netlink interface */
9900                 i40e_dcbnl_setup(vsi);
9901 #endif /* CONFIG_I40E_DCB */
9902                 /* fall through */
9903
9904         case I40E_VSI_FDIR:
9905                 /* set up vectors and rings if needed */
9906                 ret = i40e_vsi_setup_vectors(vsi);
9907                 if (ret)
9908                         goto err_msix;
9909
9910                 ret = i40e_alloc_rings(vsi);
9911                 if (ret)
9912                         goto err_rings;
9913
9914                 /* map all of the rings to the q_vectors */
9915                 i40e_vsi_map_rings_to_vectors(vsi);
9916
9917                 i40e_vsi_reset_stats(vsi);
9918                 break;
9919
9920         default:
9921                 /* no netdev or rings for the other VSI types */
9922                 break;
9923         }
9924
9925         if ((pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) &&
9926             (vsi->type == I40E_VSI_VMDQ2)) {
9927                 ret = i40e_vsi_config_rss(vsi);
9928         }
9929         return vsi;
9930
9931 err_rings:
9932         i40e_vsi_free_q_vectors(vsi);
9933 err_msix:
9934         if (vsi->netdev_registered) {
9935                 vsi->netdev_registered = false;
9936                 unregister_netdev(vsi->netdev);
9937                 free_netdev(vsi->netdev);
9938                 vsi->netdev = NULL;
9939         }
9940 err_netdev:
9941         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
9942 err_vsi:
9943         i40e_vsi_clear(vsi);
9944 err_alloc:
9945         return NULL;
9946 }
9947
9948 /**
9949  * i40e_veb_get_bw_info - Query VEB BW information
9950  * @veb: the veb to query
9951  *
9952  * Query the Tx scheduler BW configuration data for given VEB
9953  **/
9954 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
9955 {
9956         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
9957         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
9958         struct i40e_pf *pf = veb->pf;
9959         struct i40e_hw *hw = &pf->hw;
9960         u32 tc_bw_max;
9961         int ret = 0;
9962         int i;
9963
9964         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
9965                                                   &bw_data, NULL);
9966         if (ret) {
9967                 dev_info(&pf->pdev->dev,
9968                          "query veb bw config failed, err %s aq_err %s\n",
9969                          i40e_stat_str(&pf->hw, ret),
9970                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
9971                 goto out;
9972         }
9973
9974         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
9975                                                    &ets_data, NULL);
9976         if (ret) {
9977                 dev_info(&pf->pdev->dev,
9978                          "query veb bw ets config failed, err %s aq_err %s\n",
9979                          i40e_stat_str(&pf->hw, ret),
9980                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
9981                 goto out;
9982         }
9983
9984         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
9985         veb->bw_max_quanta = ets_data.tc_bw_max;
9986         veb->is_abs_credits = bw_data.absolute_credits_enable;
9987         veb->enabled_tc = ets_data.tc_valid_bits;
9988         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
9989                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
9990         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
9991                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
9992                 veb->bw_tc_limit_credits[i] =
9993                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
9994                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
9995         }
9996
9997 out:
9998         return ret;
9999 }
10000
10001 /**
10002  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
10003  * @pf: board private structure
10004  *
10005  * On error: returns error code (negative)
10006  * On success: returns vsi index in PF (positive)
10007  **/
10008 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
10009 {
10010         int ret = -ENOENT;
10011         struct i40e_veb *veb;
10012         int i;
10013
10014         /* Need to protect the allocation of switch elements at the PF level */
10015         mutex_lock(&pf->switch_mutex);
10016
10017         /* VEB list may be fragmented if VEB creation/destruction has
10018          * been happening.  We can afford to do a quick scan to look
10019          * for any free slots in the list.
10020          *
10021          * find next empty veb slot, looping back around if necessary
10022          */
10023         i = 0;
10024         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
10025                 i++;
10026         if (i >= I40E_MAX_VEB) {
10027                 ret = -ENOMEM;
10028                 goto err_alloc_veb;  /* out of VEB slots! */
10029         }
10030
10031         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
10032         if (!veb) {
10033                 ret = -ENOMEM;
10034                 goto err_alloc_veb;
10035         }
10036         veb->pf = pf;
10037         veb->idx = i;
10038         veb->enabled_tc = 1;
10039
10040         pf->veb[i] = veb;
10041         ret = i;
10042 err_alloc_veb:
10043         mutex_unlock(&pf->switch_mutex);
10044         return ret;
10045 }
10046
10047 /**
10048  * i40e_switch_branch_release - Delete a branch of the switch tree
10049  * @branch: where to start deleting
10050  *
10051  * This uses recursion to find the tips of the branch to be
10052  * removed, deleting until we get back to and can delete this VEB.
10053  **/
10054 static void i40e_switch_branch_release(struct i40e_veb *branch)
10055 {
10056         struct i40e_pf *pf = branch->pf;
10057         u16 branch_seid = branch->seid;
10058         u16 veb_idx = branch->idx;
10059         int i;
10060
10061         /* release any VEBs on this VEB - RECURSION */
10062         for (i = 0; i < I40E_MAX_VEB; i++) {
10063                 if (!pf->veb[i])
10064                         continue;
10065                 if (pf->veb[i]->uplink_seid == branch->seid)
10066                         i40e_switch_branch_release(pf->veb[i]);
10067         }
10068
10069         /* Release the VSIs on this VEB, but not the owner VSI.
10070          *
10071          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
10072          *       the VEB itself, so don't use (*branch) after this loop.
10073          */
10074         for (i = 0; i < pf->num_alloc_vsi; i++) {
10075                 if (!pf->vsi[i])
10076                         continue;
10077                 if (pf->vsi[i]->uplink_seid == branch_seid &&
10078                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10079                         i40e_vsi_release(pf->vsi[i]);
10080                 }
10081         }
10082
10083         /* There's one corner case where the VEB might not have been
10084          * removed, so double check it here and remove it if needed.
10085          * This case happens if the veb was created from the debugfs
10086          * commands and no VSIs were added to it.
10087          */
10088         if (pf->veb[veb_idx])
10089                 i40e_veb_release(pf->veb[veb_idx]);
10090 }
10091
10092 /**
10093  * i40e_veb_clear - remove veb struct
10094  * @veb: the veb to remove
10095  **/
10096 static void i40e_veb_clear(struct i40e_veb *veb)
10097 {
10098         if (!veb)
10099                 return;
10100
10101         if (veb->pf) {
10102                 struct i40e_pf *pf = veb->pf;
10103
10104                 mutex_lock(&pf->switch_mutex);
10105                 if (pf->veb[veb->idx] == veb)
10106                         pf->veb[veb->idx] = NULL;
10107                 mutex_unlock(&pf->switch_mutex);
10108         }
10109
10110         kfree(veb);
10111 }
10112
10113 /**
10114  * i40e_veb_release - Delete a VEB and free its resources
10115  * @veb: the VEB being removed
10116  **/
10117 void i40e_veb_release(struct i40e_veb *veb)
10118 {
10119         struct i40e_vsi *vsi = NULL;
10120         struct i40e_pf *pf;
10121         int i, n = 0;
10122
10123         pf = veb->pf;
10124
10125         /* find the remaining VSI and check for extras */
10126         for (i = 0; i < pf->num_alloc_vsi; i++) {
10127                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
10128                         n++;
10129                         vsi = pf->vsi[i];
10130                 }
10131         }
10132         if (n != 1) {
10133                 dev_info(&pf->pdev->dev,
10134                          "can't remove VEB %d with %d VSIs left\n",
10135                          veb->seid, n);
10136                 return;
10137         }
10138
10139         /* move the remaining VSI to uplink veb */
10140         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
10141         if (veb->uplink_seid) {
10142                 vsi->uplink_seid = veb->uplink_seid;
10143                 if (veb->uplink_seid == pf->mac_seid)
10144                         vsi->veb_idx = I40E_NO_VEB;
10145                 else
10146                         vsi->veb_idx = veb->veb_idx;
10147         } else {
10148                 /* floating VEB */
10149                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10150                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
10151         }
10152
10153         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10154         i40e_veb_clear(veb);
10155 }
10156
10157 /**
10158  * i40e_add_veb - create the VEB in the switch
10159  * @veb: the VEB to be instantiated
10160  * @vsi: the controlling VSI
10161  **/
10162 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
10163 {
10164         struct i40e_pf *pf = veb->pf;
10165         bool is_default = veb->pf->cur_promisc;
10166         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
10167         int ret;
10168
10169         /* get a VEB from the hardware */
10170         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
10171                               veb->enabled_tc, is_default,
10172                               &veb->seid, enable_stats, NULL);
10173         if (ret) {
10174                 dev_info(&pf->pdev->dev,
10175                          "couldn't add VEB, err %s aq_err %s\n",
10176                          i40e_stat_str(&pf->hw, ret),
10177                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10178                 return -EPERM;
10179         }
10180
10181         /* get statistics counter */
10182         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
10183                                          &veb->stats_idx, NULL, NULL, NULL);
10184         if (ret) {
10185                 dev_info(&pf->pdev->dev,
10186                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
10187                          i40e_stat_str(&pf->hw, ret),
10188                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10189                 return -EPERM;
10190         }
10191         ret = i40e_veb_get_bw_info(veb);
10192         if (ret) {
10193                 dev_info(&pf->pdev->dev,
10194                          "couldn't get VEB bw info, err %s aq_err %s\n",
10195                          i40e_stat_str(&pf->hw, ret),
10196                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10197                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10198                 return -ENOENT;
10199         }
10200
10201         vsi->uplink_seid = veb->seid;
10202         vsi->veb_idx = veb->idx;
10203         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10204
10205         return 0;
10206 }
10207
10208 /**
10209  * i40e_veb_setup - Set up a VEB
10210  * @pf: board private structure
10211  * @flags: VEB setup flags
10212  * @uplink_seid: the switch element to link to
10213  * @vsi_seid: the initial VSI seid
10214  * @enabled_tc: Enabled TC bit-map
10215  *
10216  * This allocates the sw VEB structure and links it into the switch
10217  * It is possible and legal for this to be a duplicate of an already
10218  * existing VEB.  It is also possible for both uplink and vsi seids
10219  * to be zero, in order to create a floating VEB.
10220  *
10221  * Returns pointer to the successfully allocated VEB sw struct on
10222  * success, otherwise returns NULL on failure.
10223  **/
10224 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
10225                                 u16 uplink_seid, u16 vsi_seid,
10226                                 u8 enabled_tc)
10227 {
10228         struct i40e_veb *veb, *uplink_veb = NULL;
10229         int vsi_idx, veb_idx;
10230         int ret;
10231
10232         /* if one seid is 0, the other must be 0 to create a floating relay */
10233         if ((uplink_seid == 0 || vsi_seid == 0) &&
10234             (uplink_seid + vsi_seid != 0)) {
10235                 dev_info(&pf->pdev->dev,
10236                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
10237                          uplink_seid, vsi_seid);
10238                 return NULL;
10239         }
10240
10241         /* make sure there is such a vsi and uplink */
10242         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
10243                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
10244                         break;
10245         if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
10246                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
10247                          vsi_seid);
10248                 return NULL;
10249         }
10250
10251         if (uplink_seid && uplink_seid != pf->mac_seid) {
10252                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10253                         if (pf->veb[veb_idx] &&
10254                             pf->veb[veb_idx]->seid == uplink_seid) {
10255                                 uplink_veb = pf->veb[veb_idx];
10256                                 break;
10257                         }
10258                 }
10259                 if (!uplink_veb) {
10260                         dev_info(&pf->pdev->dev,
10261                                  "uplink seid %d not found\n", uplink_seid);
10262                         return NULL;
10263                 }
10264         }
10265
10266         /* get veb sw struct */
10267         veb_idx = i40e_veb_mem_alloc(pf);
10268         if (veb_idx < 0)
10269                 goto err_alloc;
10270         veb = pf->veb[veb_idx];
10271         veb->flags = flags;
10272         veb->uplink_seid = uplink_seid;
10273         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
10274         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
10275
10276         /* create the VEB in the switch */
10277         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
10278         if (ret)
10279                 goto err_veb;
10280         if (vsi_idx == pf->lan_vsi)
10281                 pf->lan_veb = veb->idx;
10282
10283         return veb;
10284
10285 err_veb:
10286         i40e_veb_clear(veb);
10287 err_alloc:
10288         return NULL;
10289 }
10290
10291 /**
10292  * i40e_setup_pf_switch_element - set PF vars based on switch type
10293  * @pf: board private structure
10294  * @ele: element we are building info from
10295  * @num_reported: total number of elements
10296  * @printconfig: should we print the contents
10297  *
10298  * helper function to assist in extracting a few useful SEID values.
10299  **/
10300 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
10301                                 struct i40e_aqc_switch_config_element_resp *ele,
10302                                 u16 num_reported, bool printconfig)
10303 {
10304         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
10305         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
10306         u8 element_type = ele->element_type;
10307         u16 seid = le16_to_cpu(ele->seid);
10308
10309         if (printconfig)
10310                 dev_info(&pf->pdev->dev,
10311                          "type=%d seid=%d uplink=%d downlink=%d\n",
10312                          element_type, seid, uplink_seid, downlink_seid);
10313
10314         switch (element_type) {
10315         case I40E_SWITCH_ELEMENT_TYPE_MAC:
10316                 pf->mac_seid = seid;
10317                 break;
10318         case I40E_SWITCH_ELEMENT_TYPE_VEB:
10319                 /* Main VEB? */
10320                 if (uplink_seid != pf->mac_seid)
10321                         break;
10322                 if (pf->lan_veb == I40E_NO_VEB) {
10323                         int v;
10324
10325                         /* find existing or else empty VEB */
10326                         for (v = 0; v < I40E_MAX_VEB; v++) {
10327                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
10328                                         pf->lan_veb = v;
10329                                         break;
10330                                 }
10331                         }
10332                         if (pf->lan_veb == I40E_NO_VEB) {
10333                                 v = i40e_veb_mem_alloc(pf);
10334                                 if (v < 0)
10335                                         break;
10336                                 pf->lan_veb = v;
10337                         }
10338                 }
10339
10340                 pf->veb[pf->lan_veb]->seid = seid;
10341                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
10342                 pf->veb[pf->lan_veb]->pf = pf;
10343                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
10344                 break;
10345         case I40E_SWITCH_ELEMENT_TYPE_VSI:
10346                 if (num_reported != 1)
10347                         break;
10348                 /* This is immediately after a reset so we can assume this is
10349                  * the PF's VSI
10350                  */
10351                 pf->mac_seid = uplink_seid;
10352                 pf->pf_seid = downlink_seid;
10353                 pf->main_vsi_seid = seid;
10354                 if (printconfig)
10355                         dev_info(&pf->pdev->dev,
10356                                  "pf_seid=%d main_vsi_seid=%d\n",
10357                                  pf->pf_seid, pf->main_vsi_seid);
10358                 break;
10359         case I40E_SWITCH_ELEMENT_TYPE_PF:
10360         case I40E_SWITCH_ELEMENT_TYPE_VF:
10361         case I40E_SWITCH_ELEMENT_TYPE_EMP:
10362         case I40E_SWITCH_ELEMENT_TYPE_BMC:
10363         case I40E_SWITCH_ELEMENT_TYPE_PE:
10364         case I40E_SWITCH_ELEMENT_TYPE_PA:
10365                 /* ignore these for now */
10366                 break;
10367         default:
10368                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
10369                          element_type, seid);
10370                 break;
10371         }
10372 }
10373
10374 /**
10375  * i40e_fetch_switch_configuration - Get switch config from firmware
10376  * @pf: board private structure
10377  * @printconfig: should we print the contents
10378  *
10379  * Get the current switch configuration from the device and
10380  * extract a few useful SEID values.
10381  **/
10382 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
10383 {
10384         struct i40e_aqc_get_switch_config_resp *sw_config;
10385         u16 next_seid = 0;
10386         int ret = 0;
10387         u8 *aq_buf;
10388         int i;
10389
10390         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
10391         if (!aq_buf)
10392                 return -ENOMEM;
10393
10394         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
10395         do {
10396                 u16 num_reported, num_total;
10397
10398                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
10399                                                 I40E_AQ_LARGE_BUF,
10400                                                 &next_seid, NULL);
10401                 if (ret) {
10402                         dev_info(&pf->pdev->dev,
10403                                  "get switch config failed err %s aq_err %s\n",
10404                                  i40e_stat_str(&pf->hw, ret),
10405                                  i40e_aq_str(&pf->hw,
10406                                              pf->hw.aq.asq_last_status));
10407                         kfree(aq_buf);
10408                         return -ENOENT;
10409                 }
10410
10411                 num_reported = le16_to_cpu(sw_config->header.num_reported);
10412                 num_total = le16_to_cpu(sw_config->header.num_total);
10413
10414                 if (printconfig)
10415                         dev_info(&pf->pdev->dev,
10416                                  "header: %d reported %d total\n",
10417                                  num_reported, num_total);
10418
10419                 for (i = 0; i < num_reported; i++) {
10420                         struct i40e_aqc_switch_config_element_resp *ele =
10421                                 &sw_config->element[i];
10422
10423                         i40e_setup_pf_switch_element(pf, ele, num_reported,
10424                                                      printconfig);
10425                 }
10426         } while (next_seid != 0);
10427
10428         kfree(aq_buf);
10429         return ret;
10430 }
10431
10432 /**
10433  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
10434  * @pf: board private structure
10435  * @reinit: if the Main VSI needs to re-initialized.
10436  *
10437  * Returns 0 on success, negative value on failure
10438  **/
10439 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
10440 {
10441         int ret;
10442
10443         /* find out what's out there already */
10444         ret = i40e_fetch_switch_configuration(pf, false);
10445         if (ret) {
10446                 dev_info(&pf->pdev->dev,
10447                          "couldn't fetch switch config, err %s aq_err %s\n",
10448                          i40e_stat_str(&pf->hw, ret),
10449                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10450                 return ret;
10451         }
10452         i40e_pf_reset_stats(pf);
10453
10454         /* first time setup */
10455         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
10456                 struct i40e_vsi *vsi = NULL;
10457                 u16 uplink_seid;
10458
10459                 /* Set up the PF VSI associated with the PF's main VSI
10460                  * that is already in the HW switch
10461                  */
10462                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
10463                         uplink_seid = pf->veb[pf->lan_veb]->seid;
10464                 else
10465                         uplink_seid = pf->mac_seid;
10466                 if (pf->lan_vsi == I40E_NO_VSI)
10467                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
10468                 else if (reinit)
10469                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
10470                 if (!vsi) {
10471                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
10472                         i40e_fdir_teardown(pf);
10473                         return -EAGAIN;
10474                 }
10475         } else {
10476                 /* force a reset of TC and queue layout configurations */
10477                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
10478
10479                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
10480                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
10481                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
10482         }
10483         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
10484
10485         i40e_fdir_sb_setup(pf);
10486
10487         /* Setup static PF queue filter control settings */
10488         ret = i40e_setup_pf_filter_control(pf);
10489         if (ret) {
10490                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
10491                          ret);
10492                 /* Failure here should not stop continuing other steps */
10493         }
10494
10495         /* enable RSS in the HW, even for only one queue, as the stack can use
10496          * the hash
10497          */
10498         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
10499                 i40e_pf_config_rss(pf);
10500
10501         /* fill in link information and enable LSE reporting */
10502         i40e_update_link_info(&pf->hw);
10503         i40e_link_event(pf);
10504
10505         /* Initialize user-specific link properties */
10506         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
10507                                   I40E_AQ_AN_COMPLETED) ? true : false);
10508
10509         i40e_ptp_init(pf);
10510
10511         return ret;
10512 }
10513
10514 /**
10515  * i40e_determine_queue_usage - Work out queue distribution
10516  * @pf: board private structure
10517  **/
10518 static void i40e_determine_queue_usage(struct i40e_pf *pf)
10519 {
10520         int queues_left;
10521
10522         pf->num_lan_qps = 0;
10523 #ifdef I40E_FCOE
10524         pf->num_fcoe_qps = 0;
10525 #endif
10526
10527         /* Find the max queues to be put into basic use.  We'll always be
10528          * using TC0, whether or not DCB is running, and TC0 will get the
10529          * big RSS set.
10530          */
10531         queues_left = pf->hw.func_caps.num_tx_qp;
10532
10533         if ((queues_left == 1) ||
10534             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
10535                 /* one qp for PF, no queues for anything else */
10536                 queues_left = 0;
10537                 pf->alloc_rss_size = pf->num_lan_qps = 1;
10538
10539                 /* make sure all the fancies are disabled */
10540                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
10541                                I40E_FLAG_IWARP_ENABLED  |
10542 #ifdef I40E_FCOE
10543                                I40E_FLAG_FCOE_ENABLED   |
10544 #endif
10545                                I40E_FLAG_FD_SB_ENABLED  |
10546                                I40E_FLAG_FD_ATR_ENABLED |
10547                                I40E_FLAG_DCB_CAPABLE    |
10548                                I40E_FLAG_SRIOV_ENABLED  |
10549                                I40E_FLAG_VMDQ_ENABLED);
10550         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
10551                                   I40E_FLAG_FD_SB_ENABLED |
10552                                   I40E_FLAG_FD_ATR_ENABLED |
10553                                   I40E_FLAG_DCB_CAPABLE))) {
10554                 /* one qp for PF */
10555                 pf->alloc_rss_size = pf->num_lan_qps = 1;
10556                 queues_left -= pf->num_lan_qps;
10557
10558                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
10559                                I40E_FLAG_IWARP_ENABLED  |
10560 #ifdef I40E_FCOE
10561                                I40E_FLAG_FCOE_ENABLED   |
10562 #endif
10563                                I40E_FLAG_FD_SB_ENABLED  |
10564                                I40E_FLAG_FD_ATR_ENABLED |
10565                                I40E_FLAG_DCB_ENABLED    |
10566                                I40E_FLAG_VMDQ_ENABLED);
10567         } else {
10568                 /* Not enough queues for all TCs */
10569                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
10570                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
10571                         pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10572                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
10573                 }
10574                 pf->num_lan_qps = max_t(int, pf->rss_size_max,
10575                                         num_online_cpus());
10576                 pf->num_lan_qps = min_t(int, pf->num_lan_qps,
10577                                         pf->hw.func_caps.num_tx_qp);
10578
10579                 queues_left -= pf->num_lan_qps;
10580         }
10581
10582 #ifdef I40E_FCOE
10583         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
10584                 if (I40E_DEFAULT_FCOE <= queues_left) {
10585                         pf->num_fcoe_qps = I40E_DEFAULT_FCOE;
10586                 } else if (I40E_MINIMUM_FCOE <= queues_left) {
10587                         pf->num_fcoe_qps = I40E_MINIMUM_FCOE;
10588                 } else {
10589                         pf->num_fcoe_qps = 0;
10590                         pf->flags &= ~I40E_FLAG_FCOE_ENABLED;
10591                         dev_info(&pf->pdev->dev, "not enough queues for FCoE. FCoE feature will be disabled\n");
10592                 }
10593
10594                 queues_left -= pf->num_fcoe_qps;
10595         }
10596
10597 #endif
10598         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10599                 if (queues_left > 1) {
10600                         queues_left -= 1; /* save 1 queue for FD */
10601                 } else {
10602                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
10603                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
10604                 }
10605         }
10606
10607         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
10608             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
10609                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
10610                                         (queues_left / pf->num_vf_qps));
10611                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
10612         }
10613
10614         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
10615             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
10616                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
10617                                           (queues_left / pf->num_vmdq_qps));
10618                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
10619         }
10620
10621         pf->queues_left = queues_left;
10622         dev_dbg(&pf->pdev->dev,
10623                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
10624                 pf->hw.func_caps.num_tx_qp,
10625                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
10626                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
10627                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
10628                 queues_left);
10629 #ifdef I40E_FCOE
10630         dev_dbg(&pf->pdev->dev, "fcoe queues = %d\n", pf->num_fcoe_qps);
10631 #endif
10632 }
10633
10634 /**
10635  * i40e_setup_pf_filter_control - Setup PF static filter control
10636  * @pf: PF to be setup
10637  *
10638  * i40e_setup_pf_filter_control sets up a PF's initial filter control
10639  * settings. If PE/FCoE are enabled then it will also set the per PF
10640  * based filter sizes required for them. It also enables Flow director,
10641  * ethertype and macvlan type filter settings for the pf.
10642  *
10643  * Returns 0 on success, negative on failure
10644  **/
10645 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
10646 {
10647         struct i40e_filter_control_settings *settings = &pf->filter_settings;
10648
10649         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
10650
10651         /* Flow Director is enabled */
10652         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
10653                 settings->enable_fdir = true;
10654
10655         /* Ethtype and MACVLAN filters enabled for PF */
10656         settings->enable_ethtype = true;
10657         settings->enable_macvlan = true;
10658
10659         if (i40e_set_filter_control(&pf->hw, settings))
10660                 return -ENOENT;
10661
10662         return 0;
10663 }
10664
10665 #define INFO_STRING_LEN 255
10666 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
10667 static void i40e_print_features(struct i40e_pf *pf)
10668 {
10669         struct i40e_hw *hw = &pf->hw;
10670         char *buf;
10671         int i;
10672
10673         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
10674         if (!buf)
10675                 return;
10676
10677         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
10678 #ifdef CONFIG_PCI_IOV
10679         i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
10680 #endif
10681         i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d RX: %s",
10682                       pf->hw.func_caps.num_vsis,
10683                       pf->vsi[pf->lan_vsi]->num_queue_pairs,
10684                       pf->flags & I40E_FLAG_RX_PS_ENABLED ? "PS" : "1BUF");
10685
10686         if (pf->flags & I40E_FLAG_RSS_ENABLED)
10687                 i += snprintf(&buf[i], REMAIN(i), " RSS");
10688         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
10689                 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
10690         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10691                 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
10692                 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
10693         }
10694         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
10695                 i += snprintf(&buf[i], REMAIN(i), " DCB");
10696 #if IS_ENABLED(CONFIG_VXLAN)
10697         i += snprintf(&buf[i], REMAIN(i), " VxLAN");
10698 #endif
10699 #if IS_ENABLED(CONFIG_GENEVE)
10700         i += snprintf(&buf[i], REMAIN(i), " Geneve");
10701 #endif
10702         if (pf->flags & I40E_FLAG_PTP)
10703                 i += snprintf(&buf[i], REMAIN(i), " PTP");
10704 #ifdef I40E_FCOE
10705         if (pf->flags & I40E_FLAG_FCOE_ENABLED)
10706                 i += snprintf(&buf[i], REMAIN(i), " FCOE");
10707 #endif
10708         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
10709                 i += snprintf(&buf[i], REMAIN(i), " VEB");
10710         else
10711                 i += snprintf(&buf[i], REMAIN(i), " VEPA");
10712
10713         dev_info(&pf->pdev->dev, "%s\n", buf);
10714         kfree(buf);
10715         WARN_ON(i > INFO_STRING_LEN);
10716 }
10717
10718 /**
10719  * i40e_get_platform_mac_addr - get platform-specific MAC address
10720  *
10721  * @pdev: PCI device information struct
10722  * @pf: board private structure
10723  *
10724  * Look up the MAC address in Open Firmware  on systems that support it,
10725  * and use IDPROM on SPARC if no OF address is found. On return, the
10726  * I40E_FLAG_PF_MAC will be wset in pf->flags if a platform-specific value
10727  * has been selected.
10728  **/
10729 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
10730 {
10731         pf->flags &= ~I40E_FLAG_PF_MAC;
10732         if (!eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
10733                 pf->flags |= I40E_FLAG_PF_MAC;
10734 }
10735
10736 /**
10737  * i40e_probe - Device initialization routine
10738  * @pdev: PCI device information struct
10739  * @ent: entry in i40e_pci_tbl
10740  *
10741  * i40e_probe initializes a PF identified by a pci_dev structure.
10742  * The OS initialization, configuring of the PF private structure,
10743  * and a hardware reset occur.
10744  *
10745  * Returns 0 on success, negative on failure
10746  **/
10747 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
10748 {
10749         struct i40e_aq_get_phy_abilities_resp abilities;
10750         struct i40e_pf *pf;
10751         struct i40e_hw *hw;
10752         static u16 pfs_found;
10753         u16 wol_nvm_bits;
10754         u16 link_status;
10755         int err;
10756         u32 val;
10757         u32 i;
10758         u8 set_fc_aq_fail;
10759
10760         err = pci_enable_device_mem(pdev);
10761         if (err)
10762                 return err;
10763
10764         /* set up for high or low dma */
10765         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
10766         if (err) {
10767                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
10768                 if (err) {
10769                         dev_err(&pdev->dev,
10770                                 "DMA configuration failed: 0x%x\n", err);
10771                         goto err_dma;
10772                 }
10773         }
10774
10775         /* set up pci connections */
10776         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
10777                                            IORESOURCE_MEM), i40e_driver_name);
10778         if (err) {
10779                 dev_info(&pdev->dev,
10780                          "pci_request_selected_regions failed %d\n", err);
10781                 goto err_pci_reg;
10782         }
10783
10784         pci_enable_pcie_error_reporting(pdev);
10785         pci_set_master(pdev);
10786
10787         /* Now that we have a PCI connection, we need to do the
10788          * low level device setup.  This is primarily setting up
10789          * the Admin Queue structures and then querying for the
10790          * device's current profile information.
10791          */
10792         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
10793         if (!pf) {
10794                 err = -ENOMEM;
10795                 goto err_pf_alloc;
10796         }
10797         pf->next_vsi = 0;
10798         pf->pdev = pdev;
10799         set_bit(__I40E_DOWN, &pf->state);
10800
10801         hw = &pf->hw;
10802         hw->back = pf;
10803
10804         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
10805                                 I40E_MAX_CSR_SPACE);
10806
10807         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
10808         if (!hw->hw_addr) {
10809                 err = -EIO;
10810                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
10811                          (unsigned int)pci_resource_start(pdev, 0),
10812                          pf->ioremap_len, err);
10813                 goto err_ioremap;
10814         }
10815         hw->vendor_id = pdev->vendor;
10816         hw->device_id = pdev->device;
10817         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
10818         hw->subsystem_vendor_id = pdev->subsystem_vendor;
10819         hw->subsystem_device_id = pdev->subsystem_device;
10820         hw->bus.device = PCI_SLOT(pdev->devfn);
10821         hw->bus.func = PCI_FUNC(pdev->devfn);
10822         pf->instance = pfs_found;
10823
10824         /* set up the locks for the AQ, do this only once in probe
10825          * and destroy them only once in remove
10826          */
10827         mutex_init(&hw->aq.asq_mutex);
10828         mutex_init(&hw->aq.arq_mutex);
10829
10830         if (debug != -1) {
10831                 pf->msg_enable = pf->hw.debug_mask;
10832                 pf->msg_enable = debug;
10833         }
10834
10835         /* do a special CORER for clearing PXE mode once at init */
10836         if (hw->revision_id == 0 &&
10837             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
10838                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
10839                 i40e_flush(hw);
10840                 msleep(200);
10841                 pf->corer_count++;
10842
10843                 i40e_clear_pxe_mode(hw);
10844         }
10845
10846         /* Reset here to make sure all is clean and to define PF 'n' */
10847         i40e_clear_hw(hw);
10848         err = i40e_pf_reset(hw);
10849         if (err) {
10850                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
10851                 goto err_pf_reset;
10852         }
10853         pf->pfr_count++;
10854
10855         hw->aq.num_arq_entries = I40E_AQ_LEN;
10856         hw->aq.num_asq_entries = I40E_AQ_LEN;
10857         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
10858         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
10859         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
10860
10861         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
10862                  "%s-%s:misc",
10863                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
10864
10865         err = i40e_init_shared_code(hw);
10866         if (err) {
10867                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
10868                          err);
10869                 goto err_pf_reset;
10870         }
10871
10872         /* set up a default setting for link flow control */
10873         pf->hw.fc.requested_mode = I40E_FC_NONE;
10874
10875         err = i40e_init_adminq(hw);
10876         if (err) {
10877                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
10878                         dev_info(&pdev->dev,
10879                                  "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
10880                 else
10881                         dev_info(&pdev->dev,
10882                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
10883
10884                 goto err_pf_reset;
10885         }
10886
10887         /* provide nvm, fw, api versions */
10888         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n",
10889                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
10890                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
10891                  i40e_nvm_version_str(hw));
10892
10893         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
10894             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
10895                 dev_info(&pdev->dev,
10896                          "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
10897         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
10898                  hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
10899                 dev_info(&pdev->dev,
10900                          "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
10901
10902         i40e_verify_eeprom(pf);
10903
10904         /* Rev 0 hardware was never productized */
10905         if (hw->revision_id < 1)
10906                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
10907
10908         i40e_clear_pxe_mode(hw);
10909         err = i40e_get_capabilities(pf);
10910         if (err)
10911                 goto err_adminq_setup;
10912
10913         err = i40e_sw_init(pf);
10914         if (err) {
10915                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
10916                 goto err_sw_init;
10917         }
10918
10919         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10920                                 hw->func_caps.num_rx_qp,
10921                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
10922         if (err) {
10923                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
10924                 goto err_init_lan_hmc;
10925         }
10926
10927         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10928         if (err) {
10929                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
10930                 err = -ENOENT;
10931                 goto err_configure_lan_hmc;
10932         }
10933
10934         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
10935          * Ignore error return codes because if it was already disabled via
10936          * hardware settings this will fail
10937          */
10938         if (pf->flags & I40E_FLAG_STOP_FW_LLDP) {
10939                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
10940                 i40e_aq_stop_lldp(hw, true, NULL);
10941         }
10942
10943         i40e_get_mac_addr(hw, hw->mac.addr);
10944         /* allow a platform config to override the HW addr */
10945         i40e_get_platform_mac_addr(pdev, pf);
10946         if (!is_valid_ether_addr(hw->mac.addr)) {
10947                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
10948                 err = -EIO;
10949                 goto err_mac_addr;
10950         }
10951         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
10952         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
10953         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
10954         if (is_valid_ether_addr(hw->mac.port_addr))
10955                 pf->flags |= I40E_FLAG_PORT_ID_VALID;
10956 #ifdef I40E_FCOE
10957         err = i40e_get_san_mac_addr(hw, hw->mac.san_addr);
10958         if (err)
10959                 dev_info(&pdev->dev,
10960                          "(non-fatal) SAN MAC retrieval failed: %d\n", err);
10961         if (!is_valid_ether_addr(hw->mac.san_addr)) {
10962                 dev_warn(&pdev->dev, "invalid SAN MAC address %pM, falling back to LAN MAC\n",
10963                          hw->mac.san_addr);
10964                 ether_addr_copy(hw->mac.san_addr, hw->mac.addr);
10965         }
10966         dev_info(&pf->pdev->dev, "SAN MAC: %pM\n", hw->mac.san_addr);
10967 #endif /* I40E_FCOE */
10968
10969         pci_set_drvdata(pdev, pf);
10970         pci_save_state(pdev);
10971 #ifdef CONFIG_I40E_DCB
10972         err = i40e_init_pf_dcb(pf);
10973         if (err) {
10974                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
10975                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10976                 /* Continue without DCB enabled */
10977         }
10978 #endif /* CONFIG_I40E_DCB */
10979
10980         /* set up periodic task facility */
10981         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
10982         pf->service_timer_period = HZ;
10983
10984         INIT_WORK(&pf->service_task, i40e_service_task);
10985         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
10986         pf->flags |= I40E_FLAG_NEED_LINK_UPDATE;
10987
10988         /* NVM bit on means WoL disabled for the port */
10989         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
10990         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
10991                 pf->wol_en = false;
10992         else
10993                 pf->wol_en = true;
10994         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
10995
10996         /* set up the main switch operations */
10997         i40e_determine_queue_usage(pf);
10998         err = i40e_init_interrupt_scheme(pf);
10999         if (err)
11000                 goto err_switch_setup;
11001
11002         /* The number of VSIs reported by the FW is the minimum guaranteed
11003          * to us; HW supports far more and we share the remaining pool with
11004          * the other PFs. We allocate space for more than the guarantee with
11005          * the understanding that we might not get them all later.
11006          */
11007         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
11008                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
11009         else
11010                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
11011
11012         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
11013         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
11014                           GFP_KERNEL);
11015         if (!pf->vsi) {
11016                 err = -ENOMEM;
11017                 goto err_switch_setup;
11018         }
11019
11020 #ifdef CONFIG_PCI_IOV
11021         /* prep for VF support */
11022         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11023             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11024             !test_bit(__I40E_BAD_EEPROM, &pf->state)) {
11025                 if (pci_num_vf(pdev))
11026                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
11027         }
11028 #endif
11029         err = i40e_setup_pf_switch(pf, false);
11030         if (err) {
11031                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
11032                 goto err_vsis;
11033         }
11034
11035         /* Make sure flow control is set according to current settings */
11036         err = i40e_set_fc(hw, &set_fc_aq_fail, true);
11037         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET)
11038                 dev_dbg(&pf->pdev->dev,
11039                         "Set fc with err %s aq_err %s on get_phy_cap\n",
11040                         i40e_stat_str(hw, err),
11041                         i40e_aq_str(hw, hw->aq.asq_last_status));
11042         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET)
11043                 dev_dbg(&pf->pdev->dev,
11044                         "Set fc with err %s aq_err %s on set_phy_config\n",
11045                         i40e_stat_str(hw, err),
11046                         i40e_aq_str(hw, hw->aq.asq_last_status));
11047         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE)
11048                 dev_dbg(&pf->pdev->dev,
11049                         "Set fc with err %s aq_err %s on get_link_info\n",
11050                         i40e_stat_str(hw, err),
11051                         i40e_aq_str(hw, hw->aq.asq_last_status));
11052
11053         /* if FDIR VSI was set up, start it now */
11054         for (i = 0; i < pf->num_alloc_vsi; i++) {
11055                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
11056                         i40e_vsi_open(pf->vsi[i]);
11057                         break;
11058                 }
11059         }
11060
11061         /* The driver only wants link up/down and module qualification
11062          * reports from firmware.  Note the negative logic.
11063          */
11064         err = i40e_aq_set_phy_int_mask(&pf->hw,
11065                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
11066                                          I40E_AQ_EVENT_MEDIA_NA |
11067                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
11068         if (err)
11069                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
11070                          i40e_stat_str(&pf->hw, err),
11071                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11072
11073         /* Reconfigure hardware for allowing smaller MSS in the case
11074          * of TSO, so that we avoid the MDD being fired and causing
11075          * a reset in the case of small MSS+TSO.
11076          */
11077         val = rd32(hw, I40E_REG_MSS);
11078         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11079                 val &= ~I40E_REG_MSS_MIN_MASK;
11080                 val |= I40E_64BYTE_MSS;
11081                 wr32(hw, I40E_REG_MSS, val);
11082         }
11083
11084         if (pf->flags & I40E_FLAG_RESTART_AUTONEG) {
11085                 msleep(75);
11086                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11087                 if (err)
11088                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
11089                                  i40e_stat_str(&pf->hw, err),
11090                                  i40e_aq_str(&pf->hw,
11091                                              pf->hw.aq.asq_last_status));
11092         }
11093         /* The main driver is (mostly) up and happy. We need to set this state
11094          * before setting up the misc vector or we get a race and the vector
11095          * ends up disabled forever.
11096          */
11097         clear_bit(__I40E_DOWN, &pf->state);
11098
11099         /* In case of MSIX we are going to setup the misc vector right here
11100          * to handle admin queue events etc. In case of legacy and MSI
11101          * the misc functionality and queue processing is combined in
11102          * the same vector and that gets setup at open.
11103          */
11104         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11105                 err = i40e_setup_misc_vector(pf);
11106                 if (err) {
11107                         dev_info(&pdev->dev,
11108                                  "setup of misc vector failed: %d\n", err);
11109                         goto err_vsis;
11110                 }
11111         }
11112
11113 #ifdef CONFIG_PCI_IOV
11114         /* prep for VF support */
11115         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11116             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11117             !test_bit(__I40E_BAD_EEPROM, &pf->state)) {
11118                 /* disable link interrupts for VFs */
11119                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
11120                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
11121                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
11122                 i40e_flush(hw);
11123
11124                 if (pci_num_vf(pdev)) {
11125                         dev_info(&pdev->dev,
11126                                  "Active VFs found, allocating resources.\n");
11127                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
11128                         if (err)
11129                                 dev_info(&pdev->dev,
11130                                          "Error %d allocating resources for existing VFs\n",
11131                                          err);
11132                 }
11133         }
11134 #endif /* CONFIG_PCI_IOV */
11135
11136         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11137                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
11138                                                       pf->num_iwarp_msix,
11139                                                       I40E_IWARP_IRQ_PILE_ID);
11140                 if (pf->iwarp_base_vector < 0) {
11141                         dev_info(&pdev->dev,
11142                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
11143                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
11144                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11145                 }
11146         }
11147
11148         i40e_dbg_pf_init(pf);
11149
11150         /* tell the firmware that we're starting */
11151         i40e_send_version(pf);
11152
11153         /* since everything's happy, start the service_task timer */
11154         mod_timer(&pf->service_timer,
11155                   round_jiffies(jiffies + pf->service_timer_period));
11156
11157         /* add this PF to client device list and launch a client service task */
11158         err = i40e_lan_add_device(pf);
11159         if (err)
11160                 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
11161                          err);
11162
11163 #ifdef I40E_FCOE
11164         /* create FCoE interface */
11165         i40e_fcoe_vsi_setup(pf);
11166
11167 #endif
11168 #define PCI_SPEED_SIZE 8
11169 #define PCI_WIDTH_SIZE 8
11170         /* Devices on the IOSF bus do not have this information
11171          * and will report PCI Gen 1 x 1 by default so don't bother
11172          * checking them.
11173          */
11174         if (!(pf->flags & I40E_FLAG_NO_PCI_LINK_CHECK)) {
11175                 char speed[PCI_SPEED_SIZE] = "Unknown";
11176                 char width[PCI_WIDTH_SIZE] = "Unknown";
11177
11178                 /* Get the negotiated link width and speed from PCI config
11179                  * space
11180                  */
11181                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
11182                                           &link_status);
11183
11184                 i40e_set_pci_config_data(hw, link_status);
11185
11186                 switch (hw->bus.speed) {
11187                 case i40e_bus_speed_8000:
11188                         strncpy(speed, "8.0", PCI_SPEED_SIZE); break;
11189                 case i40e_bus_speed_5000:
11190                         strncpy(speed, "5.0", PCI_SPEED_SIZE); break;
11191                 case i40e_bus_speed_2500:
11192                         strncpy(speed, "2.5", PCI_SPEED_SIZE); break;
11193                 default:
11194                         break;
11195                 }
11196                 switch (hw->bus.width) {
11197                 case i40e_bus_width_pcie_x8:
11198                         strncpy(width, "8", PCI_WIDTH_SIZE); break;
11199                 case i40e_bus_width_pcie_x4:
11200                         strncpy(width, "4", PCI_WIDTH_SIZE); break;
11201                 case i40e_bus_width_pcie_x2:
11202                         strncpy(width, "2", PCI_WIDTH_SIZE); break;
11203                 case i40e_bus_width_pcie_x1:
11204                         strncpy(width, "1", PCI_WIDTH_SIZE); break;
11205                 default:
11206                         break;
11207                 }
11208
11209                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
11210                          speed, width);
11211
11212                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
11213                     hw->bus.speed < i40e_bus_speed_8000) {
11214                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
11215                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
11216                 }
11217         }
11218
11219         /* get the requested speeds from the fw */
11220         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
11221         if (err)
11222                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
11223                         i40e_stat_str(&pf->hw, err),
11224                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11225         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
11226
11227         /* get the supported phy types from the fw */
11228         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
11229         if (err)
11230                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
11231                         i40e_stat_str(&pf->hw, err),
11232                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11233         pf->hw.phy.phy_types = le32_to_cpu(abilities.phy_type);
11234
11235         /* Add a filter to drop all Flow control frames from any VSI from being
11236          * transmitted. By doing so we stop a malicious VF from sending out
11237          * PAUSE or PFC frames and potentially controlling traffic for other
11238          * PF/VF VSIs.
11239          * The FW can still send Flow control frames if enabled.
11240          */
11241         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11242                                                        pf->main_vsi_seid);
11243
11244         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
11245             (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
11246                 pf->flags |= I40E_FLAG_HAVE_10GBASET_PHY;
11247
11248         /* print a string summarizing features */
11249         i40e_print_features(pf);
11250
11251         return 0;
11252
11253         /* Unwind what we've done if something failed in the setup */
11254 err_vsis:
11255         set_bit(__I40E_DOWN, &pf->state);
11256         i40e_clear_interrupt_scheme(pf);
11257         kfree(pf->vsi);
11258 err_switch_setup:
11259         i40e_reset_interrupt_capability(pf);
11260         del_timer_sync(&pf->service_timer);
11261 err_mac_addr:
11262 err_configure_lan_hmc:
11263         (void)i40e_shutdown_lan_hmc(hw);
11264 err_init_lan_hmc:
11265         kfree(pf->qp_pile);
11266 err_sw_init:
11267 err_adminq_setup:
11268 err_pf_reset:
11269         iounmap(hw->hw_addr);
11270 err_ioremap:
11271         kfree(pf);
11272 err_pf_alloc:
11273         pci_disable_pcie_error_reporting(pdev);
11274         pci_release_selected_regions(pdev,
11275                                      pci_select_bars(pdev, IORESOURCE_MEM));
11276 err_pci_reg:
11277 err_dma:
11278         pci_disable_device(pdev);
11279         return err;
11280 }
11281
11282 /**
11283  * i40e_remove - Device removal routine
11284  * @pdev: PCI device information struct
11285  *
11286  * i40e_remove is called by the PCI subsystem to alert the driver
11287  * that is should release a PCI device.  This could be caused by a
11288  * Hot-Plug event, or because the driver is going to be removed from
11289  * memory.
11290  **/
11291 static void i40e_remove(struct pci_dev *pdev)
11292 {
11293         struct i40e_pf *pf = pci_get_drvdata(pdev);
11294         struct i40e_hw *hw = &pf->hw;
11295         i40e_status ret_code;
11296         int i;
11297
11298         i40e_dbg_pf_exit(pf);
11299
11300         i40e_ptp_stop(pf);
11301
11302         /* Disable RSS in hw */
11303         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
11304         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
11305
11306         /* no more scheduling of any task */
11307         set_bit(__I40E_SUSPENDED, &pf->state);
11308         set_bit(__I40E_DOWN, &pf->state);
11309         if (pf->service_timer.data)
11310                 del_timer_sync(&pf->service_timer);
11311         if (pf->service_task.func)
11312                 cancel_work_sync(&pf->service_task);
11313
11314         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
11315                 i40e_free_vfs(pf);
11316                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
11317         }
11318
11319         i40e_fdir_teardown(pf);
11320
11321         /* If there is a switch structure or any orphans, remove them.
11322          * This will leave only the PF's VSI remaining.
11323          */
11324         for (i = 0; i < I40E_MAX_VEB; i++) {
11325                 if (!pf->veb[i])
11326                         continue;
11327
11328                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
11329                     pf->veb[i]->uplink_seid == 0)
11330                         i40e_switch_branch_release(pf->veb[i]);
11331         }
11332
11333         /* Now we can shutdown the PF's VSI, just before we kill
11334          * adminq and hmc.
11335          */
11336         if (pf->vsi[pf->lan_vsi])
11337                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
11338
11339         /* remove attached clients */
11340         ret_code = i40e_lan_del_device(pf);
11341         if (ret_code) {
11342                 dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
11343                          ret_code);
11344         }
11345
11346         /* shutdown and destroy the HMC */
11347         if (hw->hmc.hmc_obj) {
11348                 ret_code = i40e_shutdown_lan_hmc(hw);
11349                 if (ret_code)
11350                         dev_warn(&pdev->dev,
11351                                  "Failed to destroy the HMC resources: %d\n",
11352                                  ret_code);
11353         }
11354
11355         /* shutdown the adminq */
11356         ret_code = i40e_shutdown_adminq(hw);
11357         if (ret_code)
11358                 dev_warn(&pdev->dev,
11359                          "Failed to destroy the Admin Queue resources: %d\n",
11360                          ret_code);
11361
11362         /* destroy the locks only once, here */
11363         mutex_destroy(&hw->aq.arq_mutex);
11364         mutex_destroy(&hw->aq.asq_mutex);
11365
11366         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
11367         i40e_clear_interrupt_scheme(pf);
11368         for (i = 0; i < pf->num_alloc_vsi; i++) {
11369                 if (pf->vsi[i]) {
11370                         i40e_vsi_clear_rings(pf->vsi[i]);
11371                         i40e_vsi_clear(pf->vsi[i]);
11372                         pf->vsi[i] = NULL;
11373                 }
11374         }
11375
11376         for (i = 0; i < I40E_MAX_VEB; i++) {
11377                 kfree(pf->veb[i]);
11378                 pf->veb[i] = NULL;
11379         }
11380
11381         kfree(pf->qp_pile);
11382         kfree(pf->vsi);
11383
11384         iounmap(hw->hw_addr);
11385         kfree(pf);
11386         pci_release_selected_regions(pdev,
11387                                      pci_select_bars(pdev, IORESOURCE_MEM));
11388
11389         pci_disable_pcie_error_reporting(pdev);
11390         pci_disable_device(pdev);
11391 }
11392
11393 /**
11394  * i40e_pci_error_detected - warning that something funky happened in PCI land
11395  * @pdev: PCI device information struct
11396  *
11397  * Called to warn that something happened and the error handling steps
11398  * are in progress.  Allows the driver to quiesce things, be ready for
11399  * remediation.
11400  **/
11401 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
11402                                                 enum pci_channel_state error)
11403 {
11404         struct i40e_pf *pf = pci_get_drvdata(pdev);
11405
11406         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
11407
11408         /* shutdown all operations */
11409         if (!test_bit(__I40E_SUSPENDED, &pf->state)) {
11410                 rtnl_lock();
11411                 i40e_prep_for_reset(pf);
11412                 rtnl_unlock();
11413         }
11414
11415         /* Request a slot reset */
11416         return PCI_ERS_RESULT_NEED_RESET;
11417 }
11418
11419 /**
11420  * i40e_pci_error_slot_reset - a PCI slot reset just happened
11421  * @pdev: PCI device information struct
11422  *
11423  * Called to find if the driver can work with the device now that
11424  * the pci slot has been reset.  If a basic connection seems good
11425  * (registers are readable and have sane content) then return a
11426  * happy little PCI_ERS_RESULT_xxx.
11427  **/
11428 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
11429 {
11430         struct i40e_pf *pf = pci_get_drvdata(pdev);
11431         pci_ers_result_t result;
11432         int err;
11433         u32 reg;
11434
11435         dev_dbg(&pdev->dev, "%s\n", __func__);
11436         if (pci_enable_device_mem(pdev)) {
11437                 dev_info(&pdev->dev,
11438                          "Cannot re-enable PCI device after reset.\n");
11439                 result = PCI_ERS_RESULT_DISCONNECT;
11440         } else {
11441                 pci_set_master(pdev);
11442                 pci_restore_state(pdev);
11443                 pci_save_state(pdev);
11444                 pci_wake_from_d3(pdev, false);
11445
11446                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
11447                 if (reg == 0)
11448                         result = PCI_ERS_RESULT_RECOVERED;
11449                 else
11450                         result = PCI_ERS_RESULT_DISCONNECT;
11451         }
11452
11453         err = pci_cleanup_aer_uncorrect_error_status(pdev);
11454         if (err) {
11455                 dev_info(&pdev->dev,
11456                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
11457                          err);
11458                 /* non-fatal, continue */
11459         }
11460
11461         return result;
11462 }
11463
11464 /**
11465  * i40e_pci_error_resume - restart operations after PCI error recovery
11466  * @pdev: PCI device information struct
11467  *
11468  * Called to allow the driver to bring things back up after PCI error
11469  * and/or reset recovery has finished.
11470  **/
11471 static void i40e_pci_error_resume(struct pci_dev *pdev)
11472 {
11473         struct i40e_pf *pf = pci_get_drvdata(pdev);
11474
11475         dev_dbg(&pdev->dev, "%s\n", __func__);
11476         if (test_bit(__I40E_SUSPENDED, &pf->state))
11477                 return;
11478
11479         rtnl_lock();
11480         i40e_handle_reset_warning(pf);
11481         rtnl_unlock();
11482 }
11483
11484 /**
11485  * i40e_shutdown - PCI callback for shutting down
11486  * @pdev: PCI device information struct
11487  **/
11488 static void i40e_shutdown(struct pci_dev *pdev)
11489 {
11490         struct i40e_pf *pf = pci_get_drvdata(pdev);
11491         struct i40e_hw *hw = &pf->hw;
11492
11493         set_bit(__I40E_SUSPENDED, &pf->state);
11494         set_bit(__I40E_DOWN, &pf->state);
11495         rtnl_lock();
11496         i40e_prep_for_reset(pf);
11497         rtnl_unlock();
11498
11499         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
11500         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
11501
11502         del_timer_sync(&pf->service_timer);
11503         cancel_work_sync(&pf->service_task);
11504         i40e_fdir_teardown(pf);
11505
11506         rtnl_lock();
11507         i40e_prep_for_reset(pf);
11508         rtnl_unlock();
11509
11510         wr32(hw, I40E_PFPM_APM,
11511              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
11512         wr32(hw, I40E_PFPM_WUFC,
11513              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
11514
11515         i40e_clear_interrupt_scheme(pf);
11516
11517         if (system_state == SYSTEM_POWER_OFF) {
11518                 pci_wake_from_d3(pdev, pf->wol_en);
11519                 pci_set_power_state(pdev, PCI_D3hot);
11520         }
11521 }
11522
11523 #ifdef CONFIG_PM
11524 /**
11525  * i40e_suspend - PCI callback for moving to D3
11526  * @pdev: PCI device information struct
11527  **/
11528 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
11529 {
11530         struct i40e_pf *pf = pci_get_drvdata(pdev);
11531         struct i40e_hw *hw = &pf->hw;
11532
11533         set_bit(__I40E_SUSPENDED, &pf->state);
11534         set_bit(__I40E_DOWN, &pf->state);
11535
11536         rtnl_lock();
11537         i40e_prep_for_reset(pf);
11538         rtnl_unlock();
11539
11540         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
11541         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
11542
11543         pci_wake_from_d3(pdev, pf->wol_en);
11544         pci_set_power_state(pdev, PCI_D3hot);
11545
11546         return 0;
11547 }
11548
11549 /**
11550  * i40e_resume - PCI callback for waking up from D3
11551  * @pdev: PCI device information struct
11552  **/
11553 static int i40e_resume(struct pci_dev *pdev)
11554 {
11555         struct i40e_pf *pf = pci_get_drvdata(pdev);
11556         u32 err;
11557
11558         pci_set_power_state(pdev, PCI_D0);
11559         pci_restore_state(pdev);
11560         /* pci_restore_state() clears dev->state_saves, so
11561          * call pci_save_state() again to restore it.
11562          */
11563         pci_save_state(pdev);
11564
11565         err = pci_enable_device_mem(pdev);
11566         if (err) {
11567                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
11568                 return err;
11569         }
11570         pci_set_master(pdev);
11571
11572         /* no wakeup events while running */
11573         pci_wake_from_d3(pdev, false);
11574
11575         /* handling the reset will rebuild the device state */
11576         if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) {
11577                 clear_bit(__I40E_DOWN, &pf->state);
11578                 rtnl_lock();
11579                 i40e_reset_and_rebuild(pf, false);
11580                 rtnl_unlock();
11581         }
11582
11583         return 0;
11584 }
11585
11586 #endif
11587 static const struct pci_error_handlers i40e_err_handler = {
11588         .error_detected = i40e_pci_error_detected,
11589         .slot_reset = i40e_pci_error_slot_reset,
11590         .resume = i40e_pci_error_resume,
11591 };
11592
11593 static struct pci_driver i40e_driver = {
11594         .name     = i40e_driver_name,
11595         .id_table = i40e_pci_tbl,
11596         .probe    = i40e_probe,
11597         .remove   = i40e_remove,
11598 #ifdef CONFIG_PM
11599         .suspend  = i40e_suspend,
11600         .resume   = i40e_resume,
11601 #endif
11602         .shutdown = i40e_shutdown,
11603         .err_handler = &i40e_err_handler,
11604         .sriov_configure = i40e_pci_sriov_configure,
11605 };
11606
11607 /**
11608  * i40e_init_module - Driver registration routine
11609  *
11610  * i40e_init_module is the first routine called when the driver is
11611  * loaded. All it does is register with the PCI subsystem.
11612  **/
11613 static int __init i40e_init_module(void)
11614 {
11615         pr_info("%s: %s - version %s\n", i40e_driver_name,
11616                 i40e_driver_string, i40e_driver_version_str);
11617         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
11618
11619         /* we will see if single thread per module is enough for now,
11620          * it can't be any worse than using the system workqueue which
11621          * was already single threaded
11622          */
11623         i40e_wq = create_singlethread_workqueue(i40e_driver_name);
11624         if (!i40e_wq) {
11625                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
11626                 return -ENOMEM;
11627         }
11628
11629         i40e_dbg_init();
11630         return pci_register_driver(&i40e_driver);
11631 }
11632 module_init(i40e_init_module);
11633
11634 /**
11635  * i40e_exit_module - Driver exit cleanup routine
11636  *
11637  * i40e_exit_module is called just before the driver is removed
11638  * from memory.
11639  **/
11640 static void __exit i40e_exit_module(void)
11641 {
11642         pci_unregister_driver(&i40e_driver);
11643         destroy_workqueue(i40e_wq);
11644         i40e_dbg_exit();
11645 }
11646 module_exit(i40e_exit_module);