drivers/net/ethernet/intel/iavf/iavf_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
   3
   4 #include "iavf.h"
   5 #include "iavf_prototype.h"
   6 #include "iavf_client.h"
   7 /* All iavf tracepoints are defined by the include below, which must
   8  * be included exactly once across the whole kernel with
   9  * CREATE_TRACE_POINTS defined
  10  */
  11 #define CREATE_TRACE_POINTS
  12 #include "iavf_trace.h"
  13
  14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
  15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
  16 static int iavf_close(struct net_device *netdev);
  17 static int iavf_init_get_resources(struct iavf_adapter *adapter);
  18 static int iavf_check_reset_complete(struct iavf_hw *hw);
  19
  20 char iavf_driver_name[] = "iavf";
  21 static const char iavf_driver_string[] =
  22         "Intel(R) Ethernet Adaptive Virtual Function Network Driver";
  23
  24 static const char iavf_copyright[] =
  25         "Copyright (c) 2013 - 2018 Intel Corporation.";
  26
  27 /* iavf_pci_tbl - PCI Device ID Table
  28  *
  29  * Wildcard entries (PCI_ANY_ID) should come last
  30  * Last entry must be all 0s
  31  *
  32  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  33  *   Class, Class Mask, private data (not used) }
  34  */
  35 static const struct pci_device_id iavf_pci_tbl[] = {
  36         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},
  37         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},
  38         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},
  39         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},
  40         /* required last entry */
  41         {0, }
  42 };
  43
  44 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
  45
  46 MODULE_ALIAS("i40evf");
  47 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  48 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver");
  49 MODULE_LICENSE("GPL v2");
  50
  51 static const struct net_device_ops iavf_netdev_ops;
  52 struct workqueue_struct *iavf_wq;
  53
  54 /**
  55  * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
  56  * @hw:   pointer to the HW structure
  57  * @mem:  ptr to mem struct to fill out
  58  * @size: size of memory requested
  59  * @alignment: what to align the allocation to
  60  **/
  61 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw,
  62                                          struct iavf_dma_mem *mem,
  63                                          u64 size, u32 alignment)
  64 {
  65         struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
  66
  67         if (!mem)
  68                 return IAVF_ERR_PARAM;
  69
  70         mem->size = ALIGN(size, alignment);
  71         mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
  72                                      (dma_addr_t *)&mem->pa, GFP_KERNEL);
  73         if (mem->va)
  74                 return 0;
  75         else
  76                 return IAVF_ERR_NO_MEMORY;
  77 }
  78
  79 /**
  80  * iavf_free_dma_mem_d - OS specific memory free for shared code
  81  * @hw:   pointer to the HW structure
  82  * @mem:  ptr to mem struct to free
  83  **/
  84 enum iavf_status iavf_free_dma_mem_d(struct iavf_hw *hw,
  85                                      struct iavf_dma_mem *mem)
  86 {
  87         struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
  88
  89         if (!mem || !mem->va)
  90                 return IAVF_ERR_PARAM;
  91         dma_free_coherent(&adapter->pdev->dev, mem->size,
  92                           mem->va, (dma_addr_t)mem->pa);
  93         return 0;
  94 }
  95
  96 /**
  97  * iavf_allocate_virt_mem_d - OS specific memory alloc for shared code
  98  * @hw:   pointer to the HW structure
  99  * @mem:  ptr to mem struct to fill out
 100  * @size: size of memory requested
 101  **/
 102 enum iavf_status iavf_allocate_virt_mem_d(struct iavf_hw *hw,
 103                                           struct iavf_virt_mem *mem, u32 size)
 104 {
 105         if (!mem)
 106                 return IAVF_ERR_PARAM;
 107
 108         mem->size = size;
 109         mem->va = kzalloc(size, GFP_KERNEL);
 110
 111         if (mem->va)
 112                 return 0;
 113         else
 114                 return IAVF_ERR_NO_MEMORY;
 115 }
 116
 117 /**
 118  * iavf_free_virt_mem_d - OS specific memory free for shared code
 119  * @hw:   pointer to the HW structure
 120  * @mem:  ptr to mem struct to free
 121  **/
 122 enum iavf_status iavf_free_virt_mem_d(struct iavf_hw *hw,
 123                                       struct iavf_virt_mem *mem)
 124 {
 125         if (!mem)
 126                 return IAVF_ERR_PARAM;
 127
 128         /* it's ok to kfree a NULL pointer */
 129         kfree(mem->va);
 130
 131         return 0;
 132 }
 133
 134 /**
 135  * iavf_lock_timeout - try to lock mutex but give up after timeout
 136  * @lock: mutex that should be locked
 137  * @msecs: timeout in msecs
 138  *
 139  * Returns 0 on success, negative on failure
 140  **/
 141 static int iavf_lock_timeout(struct mutex *lock, unsigned int msecs)
 142 {
 143         unsigned int wait, delay = 10;
 144
 145         for (wait = 0; wait < msecs; wait += delay) {
 146                 if (mutex_trylock(lock))
 147                         return 0;
 148
 149                 msleep(delay);
 150         }
 151
 152         return -1;
 153 }
 154
 155 /**
 156  * iavf_schedule_reset - Set the flags and schedule a reset event
 157  * @adapter: board private structure
 158  **/
 159 void iavf_schedule_reset(struct iavf_adapter *adapter)
 160 {
 161         if (!(adapter->flags &
 162               (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {
 163                 adapter->flags |= IAVF_FLAG_RESET_NEEDED;
 164                 queue_work(iavf_wq, &adapter->reset_task);
 165         }
 166 }
 167
 168 /**
 169  * iavf_tx_timeout - Respond to a Tx Hang
 170  * @netdev: network interface device structure
 171  * @txqueue: queue number that is timing out
 172  **/
 173 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 174 {
 175         struct iavf_adapter *adapter = netdev_priv(netdev);
 176
 177         adapter->tx_timeout_count++;
 178         iavf_schedule_reset(adapter);
 179 }
 180
 181 /**
 182  * iavf_misc_irq_disable - Mask off interrupt generation on the NIC
 183  * @adapter: board private structure
 184  **/
 185 static void iavf_misc_irq_disable(struct iavf_adapter *adapter)
 186 {
 187         struct iavf_hw *hw = &adapter->hw;
 188
 189         if (!adapter->msix_entries)
 190                 return;
 191
 192         wr32(hw, IAVF_VFINT_DYN_CTL01, 0);
 193
 194         iavf_flush(hw);
 195
 196         synchronize_irq(adapter->msix_entries[0].vector);
 197 }
 198
 199 /**
 200  * iavf_misc_irq_enable - Enable default interrupt generation settings
 201  * @adapter: board private structure
 202  **/
 203 static void iavf_misc_irq_enable(struct iavf_adapter *adapter)
 204 {
 205         struct iavf_hw *hw = &adapter->hw;
 206
 207         wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK |
 208                                        IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
 209         wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
 210
 211         iavf_flush(hw);
 212 }
 213
 214 /**
 215  * iavf_irq_disable - Mask off interrupt generation on the NIC
 216  * @adapter: board private structure
 217  **/
 218 static void iavf_irq_disable(struct iavf_adapter *adapter)
 219 {
 220         int i;
 221         struct iavf_hw *hw = &adapter->hw;
 222
 223         if (!adapter->msix_entries)
 224                 return;
 225
 226         for (i = 1; i < adapter->num_msix_vectors; i++) {
 227                 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0);
 228                 synchronize_irq(adapter->msix_entries[i].vector);
 229         }
 230         iavf_flush(hw);
 231 }
 232
 233 /**
 234  * iavf_irq_enable_queues - Enable interrupt for specified queues
 235  * @adapter: board private structure
 236  * @mask: bitmap of queues to enable
 237  **/
 238 void iavf_irq_enable_queues(struct iavf_adapter *adapter, u32 mask)
 239 {
 240         struct iavf_hw *hw = &adapter->hw;
 241         int i;
 242
 243         for (i = 1; i < adapter->num_msix_vectors; i++) {
 244                 if (mask & BIT(i - 1)) {
 245                         wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1),
 246                              IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
 247                              IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK);
 248                 }
 249         }
 250 }
 251
 252 /**
 253  * iavf_irq_enable - Enable default interrupt generation settings
 254  * @adapter: board private structure
 255  * @flush: boolean value whether to run rd32()
 256  **/
 257 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)
 258 {
 259         struct iavf_hw *hw = &adapter->hw;
 260
 261         iavf_misc_irq_enable(adapter);
 262         iavf_irq_enable_queues(adapter, ~0);
 263
 264         if (flush)
 265                 iavf_flush(hw);
 266 }
 267
 268 /**
 269  * iavf_msix_aq - Interrupt handler for vector 0
 270  * @irq: interrupt number
 271  * @data: pointer to netdev
 272  **/
 273 static irqreturn_t iavf_msix_aq(int irq, void *data)
 274 {
 275         struct net_device *netdev = data;
 276         struct iavf_adapter *adapter = netdev_priv(netdev);
 277         struct iavf_hw *hw = &adapter->hw;
 278
 279         /* handle non-queue interrupts, these reads clear the registers */
 280         rd32(hw, IAVF_VFINT_ICR01);
 281         rd32(hw, IAVF_VFINT_ICR0_ENA1);
 282
 283         /* schedule work on the private workqueue */
 284         queue_work(iavf_wq, &adapter->adminq_task);
 285
 286         return IRQ_HANDLED;
 287 }
 288
 289 /**
 290  * iavf_msix_clean_rings - MSIX mode Interrupt Handler
 291  * @irq: interrupt number
 292  * @data: pointer to a q_vector
 293  **/
 294 static irqreturn_t iavf_msix_clean_rings(int irq, void *data)
 295 {
 296         struct iavf_q_vector *q_vector = data;
 297
 298         if (!q_vector->tx.ring && !q_vector->rx.ring)
 299                 return IRQ_HANDLED;
 300
 301         napi_schedule_irqoff(&q_vector->napi);
 302
 303         return IRQ_HANDLED;
 304 }
 305
 306 /**
 307  * iavf_map_vector_to_rxq - associate irqs with rx queues
 308  * @adapter: board private structure
 309  * @v_idx: interrupt number
 310  * @r_idx: queue number
 311  **/
 312 static void
 313 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)
 314 {
 315         struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
 316         struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx];
 317         struct iavf_hw *hw = &adapter->hw;
 318
 319         rx_ring->q_vector = q_vector;
 320         rx_ring->next = q_vector->rx.ring;
 321         rx_ring->vsi = &adapter->vsi;
 322         q_vector->rx.ring = rx_ring;
 323         q_vector->rx.count++;
 324         q_vector->rx.next_update = jiffies + 1;
 325         q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
 326         q_vector->ring_mask |= BIT(r_idx);
 327         wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx),
 328              q_vector->rx.current_itr >> 1);
 329         q_vector->rx.current_itr = q_vector->rx.target_itr;
 330 }
 331
 332 /**
 333  * iavf_map_vector_to_txq - associate irqs with tx queues
 334  * @adapter: board private structure
 335  * @v_idx: interrupt number
 336  * @t_idx: queue number
 337  **/
 338 static void
 339 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)
 340 {
 341         struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
 342         struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx];
 343         struct iavf_hw *hw = &adapter->hw;
 344
 345         tx_ring->q_vector = q_vector;
 346         tx_ring->next = q_vector->tx.ring;
 347         tx_ring->vsi = &adapter->vsi;
 348         q_vector->tx.ring = tx_ring;
 349         q_vector->tx.count++;
 350         q_vector->tx.next_update = jiffies + 1;
 351         q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
 352         q_vector->num_ringpairs++;
 353         wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx),
 354              q_vector->tx.target_itr >> 1);
 355         q_vector->tx.current_itr = q_vector->tx.target_itr;
 356 }
 357
 358 /**
 359  * iavf_map_rings_to_vectors - Maps descriptor rings to vectors
 360  * @adapter: board private structure to initialize
 361  *
 362  * This function maps descriptor rings to the queue-specific vectors
 363  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 364  * one vector per ring/queue, but on a constrained vector budget, we
 365  * group the rings as "efficiently" as possible.  You would add new
 366  * mapping configurations in here.
 367  **/
 368 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)
 369 {
 370         int rings_remaining = adapter->num_active_queues;
 371         int ridx = 0, vidx = 0;
 372         int q_vectors;
 373
 374         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 375
 376         for (; ridx < rings_remaining; ridx++) {
 377                 iavf_map_vector_to_rxq(adapter, vidx, ridx);
 378                 iavf_map_vector_to_txq(adapter, vidx, ridx);
 379
 380                 /* In the case where we have more queues than vectors, continue
 381                  * round-robin on vectors until all queues are mapped.
 382                  */
 383                 if (++vidx >= q_vectors)
 384                         vidx = 0;
 385         }
 386
 387         adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
 388 }
 389
 390 /**
 391  * iavf_irq_affinity_notify - Callback for affinity changes
 392  * @notify: context as to what irq was changed
 393  * @mask: the new affinity mask
 394  *
 395  * This is a callback function used by the irq_set_affinity_notifier function
 396  * so that we may register to receive changes to the irq affinity masks.
 397  **/
 398 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,
 399                                      const cpumask_t *mask)
 400 {
 401         struct iavf_q_vector *q_vector =
 402                 container_of(notify, struct iavf_q_vector, affinity_notify);
 403
 404         cpumask_copy(&q_vector->affinity_mask, mask);
 405 }
 406
 407 /**
 408  * iavf_irq_affinity_release - Callback for affinity notifier release
 409  * @ref: internal core kernel usage
 410  *
 411  * This is a callback function used by the irq_set_affinity_notifier function
 412  * to inform the current notification subscriber that they will no longer
 413  * receive notifications.
 414  **/
 415 static void iavf_irq_affinity_release(struct kref *ref) {}
 416
 417 /**
 418  * iavf_request_traffic_irqs - Initialize MSI-X interrupts
 419  * @adapter: board private structure
 420  * @basename: device basename
 421  *
 422  * Allocates MSI-X vectors for tx and rx handling, and requests
 423  * interrupts from the kernel.
 424  **/
 425 static int
 426 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)
 427 {
 428         unsigned int vector, q_vectors;
 429         unsigned int rx_int_idx = 0, tx_int_idx = 0;
 430         int irq_num, err;
 431         int cpu;
 432
 433         iavf_irq_disable(adapter);
 434         /* Decrement for Other and TCP Timer vectors */
 435         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 436
 437         for (vector = 0; vector < q_vectors; vector++) {
 438                 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector];
 439
 440                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 441
 442                 if (q_vector->tx.ring && q_vector->rx.ring) {
 443                         snprintf(q_vector->name, sizeof(q_vector->name),
 444                                  "iavf-%s-TxRx-%d", basename, rx_int_idx++);
 445                         tx_int_idx++;
 446                 } else if (q_vector->rx.ring) {
 447                         snprintf(q_vector->name, sizeof(q_vector->name),
 448                                  "iavf-%s-rx-%d", basename, rx_int_idx++);
 449                 } else if (q_vector->tx.ring) {
 450                         snprintf(q_vector->name, sizeof(q_vector->name),
 451                                  "iavf-%s-tx-%d", basename, tx_int_idx++);
 452                 } else {
 453                         /* skip this unused q_vector */
 454                         continue;
 455                 }
 456                 err = request_irq(irq_num,
 457                                   iavf_msix_clean_rings,
 458                                   0,
 459                                   q_vector->name,
 460                                   q_vector);
 461                 if (err) {
 462                         dev_info(&adapter->pdev->dev,
 463                                  "Request_irq failed, error: %d\n", err);
 464                         goto free_queue_irqs;
 465                 }
 466                 /* register for affinity change notifications */
 467                 q_vector->affinity_notify.notify = iavf_irq_affinity_notify;
 468                 q_vector->affinity_notify.release =
 469                                                    iavf_irq_affinity_release;
 470                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
 471                 /* Spread the IRQ affinity hints across online CPUs. Note that
 472                  * get_cpu_mask returns a mask with a permanent lifetime so
 473                  * it's safe to use as a hint for irq_set_affinity_hint.
 474                  */
 475                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
 476                 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
 477         }
 478
 479         return 0;
 480
 481 free_queue_irqs:
 482         while (vector) {
 483                 vector--;
 484                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 485                 irq_set_affinity_notifier(irq_num, NULL);
 486                 irq_set_affinity_hint(irq_num, NULL);
 487                 free_irq(irq_num, &adapter->q_vectors[vector]);
 488         }
 489         return err;
 490 }
 491
 492 /**
 493  * iavf_request_misc_irq - Initialize MSI-X interrupts
 494  * @adapter: board private structure
 495  *
 496  * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
 497  * vector is only for the admin queue, and stays active even when the netdev
 498  * is closed.
 499  **/
 500 static int iavf_request_misc_irq(struct iavf_adapter *adapter)
 501 {
 502         struct net_device *netdev = adapter->netdev;
 503         int err;
 504
 505         snprintf(adapter->misc_vector_name,
 506                  sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx",
 507                  dev_name(&adapter->pdev->dev));
 508         err = request_irq(adapter->msix_entries[0].vector,
 509                           &iavf_msix_aq, 0,
 510                           adapter->misc_vector_name, netdev);
 511         if (err) {
 512                 dev_err(&adapter->pdev->dev,
 513                         "request_irq for %s failed: %d\n",
 514                         adapter->misc_vector_name, err);
 515                 free_irq(adapter->msix_entries[0].vector, netdev);
 516         }
 517         return err;
 518 }
 519
 520 /**
 521  * iavf_free_traffic_irqs - Free MSI-X interrupts
 522  * @adapter: board private structure
 523  *
 524  * Frees all MSI-X vectors other than 0.
 525  **/
 526 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)
 527 {
 528         int vector, irq_num, q_vectors;
 529
 530         if (!adapter->msix_entries)
 531                 return;
 532
 533         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 534
 535         for (vector = 0; vector < q_vectors; vector++) {
 536                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 537                 irq_set_affinity_notifier(irq_num, NULL);
 538                 irq_set_affinity_hint(irq_num, NULL);
 539                 free_irq(irq_num, &adapter->q_vectors[vector]);
 540         }
 541 }
 542
 543 /**
 544  * iavf_free_misc_irq - Free MSI-X miscellaneous vector
 545  * @adapter: board private structure
 546  *
 547  * Frees MSI-X vector 0.
 548  **/
 549 static void iavf_free_misc_irq(struct iavf_adapter *adapter)
 550 {
 551         struct net_device *netdev = adapter->netdev;
 552
 553         if (!adapter->msix_entries)
 554                 return;
 555
 556         free_irq(adapter->msix_entries[0].vector, netdev);
 557 }
 558
 559 /**
 560  * iavf_configure_tx - Configure Transmit Unit after Reset
 561  * @adapter: board private structure
 562  *
 563  * Configure the Tx unit of the MAC after a reset.
 564  **/
 565 static void iavf_configure_tx(struct iavf_adapter *adapter)
 566 {
 567         struct iavf_hw *hw = &adapter->hw;
 568         int i;
 569
 570         for (i = 0; i < adapter->num_active_queues; i++)
 571                 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i);
 572 }
 573
 574 /**
 575  * iavf_configure_rx - Configure Receive Unit after Reset
 576  * @adapter: board private structure
 577  *
 578  * Configure the Rx unit of the MAC after a reset.
 579  **/
 580 static void iavf_configure_rx(struct iavf_adapter *adapter)
 581 {
 582         unsigned int rx_buf_len = IAVF_RXBUFFER_2048;
 583         struct iavf_hw *hw = &adapter->hw;
 584         int i;
 585
 586         /* Legacy Rx will always default to a 2048 buffer size. */
 587 #if (PAGE_SIZE < 8192)
 588         if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) {
 589                 struct net_device *netdev = adapter->netdev;
 590
 591                 /* For jumbo frames on systems with 4K pages we have to use
 592                  * an order 1 page, so we might as well increase the size
 593                  * of our Rx buffer to make better use of the available space
 594                  */
 595                 rx_buf_len = IAVF_RXBUFFER_3072;
 596
 597                 /* We use a 1536 buffer size for configurations with
 598                  * standard Ethernet mtu.  On x86 this gives us enough room
 599                  * for shared info and 192 bytes of padding.
 600                  */
 601                 if (!IAVF_2K_TOO_SMALL_WITH_PADDING &&
 602                     (netdev->mtu <= ETH_DATA_LEN))
 603                         rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN;
 604         }
 605 #endif
 606
 607         for (i = 0; i < adapter->num_active_queues; i++) {
 608                 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i);
 609                 adapter->rx_rings[i].rx_buf_len = rx_buf_len;
 610
 611                 if (adapter->flags & IAVF_FLAG_LEGACY_RX)
 612                         clear_ring_build_skb_enabled(&adapter->rx_rings[i]);
 613                 else
 614                         set_ring_build_skb_enabled(&adapter->rx_rings[i]);
 615         }
 616 }
 617
 618 /**
 619  * iavf_find_vlan - Search filter list for specific vlan filter
 620  * @adapter: board private structure
 621  * @vlan: vlan tag
 622  *
 623  * Returns ptr to the filter object or NULL. Must be called while holding the
 624  * mac_vlan_list_lock.
 625  **/
 626 static struct
 627 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, u16 vlan)
 628 {
 629         struct iavf_vlan_filter *f;
 630
 631         list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 632                 if (vlan == f->vlan)
 633                         return f;
 634         }
 635         return NULL;
 636 }
 637
 638 /**
 639  * iavf_add_vlan - Add a vlan filter to the list
 640  * @adapter: board private structure
 641  * @vlan: VLAN tag
 642  *
 643  * Returns ptr to the filter object or NULL when no memory available.
 644  **/
 645 static struct
 646 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, u16 vlan)
 647 {
 648         struct iavf_vlan_filter *f = NULL;
 649
 650         spin_lock_bh(&adapter->mac_vlan_list_lock);
 651
 652         f = iavf_find_vlan(adapter, vlan);
 653         if (!f) {
 654                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 655                 if (!f)
 656                         goto clearout;
 657
 658                 f->vlan = vlan;
 659
 660                 list_add_tail(&f->list, &adapter->vlan_filter_list);
 661                 f->add = true;
 662                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
 663         }
 664
 665 clearout:
 666         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 667         return f;
 668 }
 669
 670 /**
 671  * iavf_del_vlan - Remove a vlan filter from the list
 672  * @adapter: board private structure
 673  * @vlan: VLAN tag
 674  **/
 675 static void iavf_del_vlan(struct iavf_adapter *adapter, u16 vlan)
 676 {
 677         struct iavf_vlan_filter *f;
 678
 679         spin_lock_bh(&adapter->mac_vlan_list_lock);
 680
 681         f = iavf_find_vlan(adapter, vlan);
 682         if (f) {
 683                 f->remove = true;
 684                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
 685         }
 686
 687         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 688 }
 689
 690 /**
 691  * iavf_vlan_rx_add_vid - Add a VLAN filter to a device
 692  * @netdev: network device struct
 693  * @proto: unused protocol data
 694  * @vid: VLAN tag
 695  **/
 696 static int iavf_vlan_rx_add_vid(struct net_device *netdev,
 697                                 __always_unused __be16 proto, u16 vid)
 698 {
 699         struct iavf_adapter *adapter = netdev_priv(netdev);
 700
 701         if (!VLAN_ALLOWED(adapter))
 702                 return -EIO;
 703         if (iavf_add_vlan(adapter, vid) == NULL)
 704                 return -ENOMEM;
 705         return 0;
 706 }
 707
 708 /**
 709  * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device
 710  * @netdev: network device struct
 711  * @proto: unused protocol data
 712  * @vid: VLAN tag
 713  **/
 714 static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
 715                                  __always_unused __be16 proto, u16 vid)
 716 {
 717         struct iavf_adapter *adapter = netdev_priv(netdev);
 718
 719         if (VLAN_ALLOWED(adapter)) {
 720                 iavf_del_vlan(adapter, vid);
 721                 return 0;
 722         }
 723         return -EIO;
 724 }
 725
 726 /**
 727  * iavf_find_filter - Search filter list for specific mac filter
 728  * @adapter: board private structure
 729  * @macaddr: the MAC address
 730  *
 731  * Returns ptr to the filter object or NULL. Must be called while holding the
 732  * mac_vlan_list_lock.
 733  **/
 734 static struct
 735 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,
 736                                   const u8 *macaddr)
 737 {
 738         struct iavf_mac_filter *f;
 739
 740         if (!macaddr)
 741                 return NULL;
 742
 743         list_for_each_entry(f, &adapter->mac_filter_list, list) {
 744                 if (ether_addr_equal(macaddr, f->macaddr))
 745                         return f;
 746         }
 747         return NULL;
 748 }
 749
 750 /**
 751  * iavf_add_filter - Add a mac filter to the filter list
 752  * @adapter: board private structure
 753  * @macaddr: the MAC address
 754  *
 755  * Returns ptr to the filter object or NULL when no memory available.
 756  **/
 757 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
 758                                         const u8 *macaddr)
 759 {
 760         struct iavf_mac_filter *f;
 761
 762         if (!macaddr)
 763                 return NULL;
 764
 765         f = iavf_find_filter(adapter, macaddr);
 766         if (!f) {
 767                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 768                 if (!f)
 769                         return f;
 770
 771                 ether_addr_copy(f->macaddr, macaddr);
 772
 773                 list_add_tail(&f->list, &adapter->mac_filter_list);
 774                 f->add = true;
 775                 f->is_new_mac = true;
 776                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
 777         } else {
 778                 f->remove = false;
 779         }
 780
 781         return f;
 782 }
 783
 784 /**
 785  * iavf_set_mac - NDO callback to set port mac address
 786  * @netdev: network interface device structure
 787  * @p: pointer to an address structure
 788  *
 789  * Returns 0 on success, negative on failure
 790  **/
 791 static int iavf_set_mac(struct net_device *netdev, void *p)
 792 {
 793         struct iavf_adapter *adapter = netdev_priv(netdev);
 794         struct iavf_hw *hw = &adapter->hw;
 795         struct iavf_mac_filter *f;
 796         struct sockaddr *addr = p;
 797
 798         if (!is_valid_ether_addr(addr->sa_data))
 799                 return -EADDRNOTAVAIL;
 800
 801         if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
 802                 return 0;
 803
 804         spin_lock_bh(&adapter->mac_vlan_list_lock);
 805
 806         f = iavf_find_filter(adapter, hw->mac.addr);
 807         if (f) {
 808                 f->remove = true;
 809                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
 810         }
 811
 812         f = iavf_add_filter(adapter, addr->sa_data);
 813
 814         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 815
 816         if (f) {
 817                 ether_addr_copy(hw->mac.addr, addr->sa_data);
 818         }
 819
 820         return (f == NULL) ? -ENOMEM : 0;
 821 }
 822
 823 /**
 824  * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address
 825  * @netdev: the netdevice
 826  * @addr: address to add
 827  *
 828  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
 829  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
 830  */
 831 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)
 832 {
 833         struct iavf_adapter *adapter = netdev_priv(netdev);
 834
 835         if (iavf_add_filter(adapter, addr))
 836                 return 0;
 837         else
 838                 return -ENOMEM;
 839 }
 840
 841 /**
 842  * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
 843  * @netdev: the netdevice
 844  * @addr: address to add
 845  *
 846  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
 847  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
 848  */
 849 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)
 850 {
 851         struct iavf_adapter *adapter = netdev_priv(netdev);
 852         struct iavf_mac_filter *f;
 853
 854         /* Under some circumstances, we might receive a request to delete
 855          * our own device address from our uc list. Because we store the
 856          * device address in the VSI's MAC/VLAN filter list, we need to ignore
 857          * such requests and not delete our device address from this list.
 858          */
 859         if (ether_addr_equal(addr, netdev->dev_addr))
 860                 return 0;
 861
 862         f = iavf_find_filter(adapter, addr);
 863         if (f) {
 864                 f->remove = true;
 865                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
 866         }
 867         return 0;
 868 }
 869
 870 /**
 871  * iavf_set_rx_mode - NDO callback to set the netdev filters
 872  * @netdev: network interface device structure
 873  **/
 874 static void iavf_set_rx_mode(struct net_device *netdev)
 875 {
 876         struct iavf_adapter *adapter = netdev_priv(netdev);
 877
 878         spin_lock_bh(&adapter->mac_vlan_list_lock);
 879         __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
 880         __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
 881         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 882
 883         if (netdev->flags & IFF_PROMISC &&
 884             !(adapter->flags & IAVF_FLAG_PROMISC_ON))
 885                 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC;
 886         else if (!(netdev->flags & IFF_PROMISC) &&
 887                  adapter->flags & IAVF_FLAG_PROMISC_ON)
 888                 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC;
 889
 890         if (netdev->flags & IFF_ALLMULTI &&
 891             !(adapter->flags & IAVF_FLAG_ALLMULTI_ON))
 892                 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI;
 893         else if (!(netdev->flags & IFF_ALLMULTI) &&
 894                  adapter->flags & IAVF_FLAG_ALLMULTI_ON)
 895                 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI;
 896 }
 897
 898 /**
 899  * iavf_napi_enable_all - enable NAPI on all queue vectors
 900  * @adapter: board private structure
 901  **/
 902 static void iavf_napi_enable_all(struct iavf_adapter *adapter)
 903 {
 904         int q_idx;
 905         struct iavf_q_vector *q_vector;
 906         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 907
 908         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
 909                 struct napi_struct *napi;
 910
 911                 q_vector = &adapter->q_vectors[q_idx];
 912                 napi = &q_vector->napi;
 913                 napi_enable(napi);
 914         }
 915 }
 916
 917 /**
 918  * iavf_napi_disable_all - disable NAPI on all queue vectors
 919  * @adapter: board private structure
 920  **/
 921 static void iavf_napi_disable_all(struct iavf_adapter *adapter)
 922 {
 923         int q_idx;
 924         struct iavf_q_vector *q_vector;
 925         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 926
 927         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
 928                 q_vector = &adapter->q_vectors[q_idx];
 929                 napi_disable(&q_vector->napi);
 930         }
 931 }
 932
 933 /**
 934  * iavf_configure - set up transmit and receive data structures
 935  * @adapter: board private structure
 936  **/
 937 static void iavf_configure(struct iavf_adapter *adapter)
 938 {
 939         struct net_device *netdev = adapter->netdev;
 940         int i;
 941
 942         iavf_set_rx_mode(netdev);
 943
 944         iavf_configure_tx(adapter);
 945         iavf_configure_rx(adapter);
 946         adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;
 947
 948         for (i = 0; i < adapter->num_active_queues; i++) {
 949                 struct iavf_ring *ring = &adapter->rx_rings[i];
 950
 951                 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring));
 952         }
 953 }
 954
 955 /**
 956  * iavf_up_complete - Finish the last steps of bringing up a connection
 957  * @adapter: board private structure
 958  *
 959  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
 960  **/
 961 static void iavf_up_complete(struct iavf_adapter *adapter)
 962 {
 963         adapter->state = __IAVF_RUNNING;
 964         clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
 965
 966         iavf_napi_enable_all(adapter);
 967
 968         adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES;
 969         if (CLIENT_ENABLED(adapter))
 970                 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN;
 971         mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
 972 }
 973
 974 /**
 975  * iavf_down - Shutdown the connection processing
 976  * @adapter: board private structure
 977  *
 978  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
 979  **/
 980 void iavf_down(struct iavf_adapter *adapter)
 981 {
 982         struct net_device *netdev = adapter->netdev;
 983         struct iavf_vlan_filter *vlf;
 984         struct iavf_cloud_filter *cf;
 985         struct iavf_fdir_fltr *fdir;
 986         struct iavf_mac_filter *f;
 987         struct iavf_adv_rss *rss;
 988
 989         if (adapter->state <= __IAVF_DOWN_PENDING)
 990                 return;
 991
 992         netif_carrier_off(netdev);
 993         netif_tx_disable(netdev);
 994         adapter->link_up = false;
 995         iavf_napi_disable_all(adapter);
 996         iavf_irq_disable(adapter);
 997
 998         spin_lock_bh(&adapter->mac_vlan_list_lock);
 999
1000         /* clear the sync flag on all filters */
1001         __dev_uc_unsync(adapter->netdev, NULL);
1002         __dev_mc_unsync(adapter->netdev, NULL);
1003
1004         /* remove all MAC filters */
1005         list_for_each_entry(f, &adapter->mac_filter_list, list) {
1006                 f->remove = true;
1007         }
1008
1009         /* remove all VLAN filters */
1010         list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
1011                 vlf->remove = true;
1012         }
1013
1014         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1015
1016         /* remove all cloud filters */
1017         spin_lock_bh(&adapter->cloud_filter_list_lock);
1018         list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
1019                 cf->del = true;
1020         }
1021         spin_unlock_bh(&adapter->cloud_filter_list_lock);
1022
1023         /* remove all Flow Director filters */
1024         spin_lock_bh(&adapter->fdir_fltr_lock);
1025         list_for_each_entry(fdir, &adapter->fdir_list_head, list) {
1026                 fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
1027         }
1028         spin_unlock_bh(&adapter->fdir_fltr_lock);
1029
1030         /* remove all advance RSS configuration */
1031         spin_lock_bh(&adapter->adv_rss_lock);
1032         list_for_each_entry(rss, &adapter->adv_rss_list_head, list)
1033                 rss->state = IAVF_ADV_RSS_DEL_REQUEST;
1034         spin_unlock_bh(&adapter->adv_rss_lock);
1035
1036         if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) &&
1037             adapter->state != __IAVF_RESETTING) {
1038                 /* cancel any current operation */
1039                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1040                 /* Schedule operations to close down the HW. Don't wait
1041                  * here for this to complete. The watchdog is still running
1042                  * and it will take care of this.
1043                  */
1044                 adapter->aq_required = IAVF_FLAG_AQ_DEL_MAC_FILTER;
1045                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
1046                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
1047                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
1048                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
1049                 adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES;
1050         }
1051
1052         mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
1053 }
1054
1055 /**
1056  * iavf_acquire_msix_vectors - Setup the MSIX capability
1057  * @adapter: board private structure
1058  * @vectors: number of vectors to request
1059  *
1060  * Work with the OS to set up the MSIX vectors needed.
1061  *
1062  * Returns 0 on success, negative on failure
1063  **/
1064 static int
1065 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)
1066 {
1067         int err, vector_threshold;
1068
1069         /* We'll want at least 3 (vector_threshold):
1070          * 0) Other (Admin Queue and link, mostly)
1071          * 1) TxQ[0] Cleanup
1072          * 2) RxQ[0] Cleanup
1073          */
1074         vector_threshold = MIN_MSIX_COUNT;
1075
1076         /* The more we get, the more we will assign to Tx/Rx Cleanup
1077          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1078          * Right now, we simply care about how many we'll get; we'll
1079          * set them up later while requesting irq's.
1080          */
1081         err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1082                                     vector_threshold, vectors);
1083         if (err < 0) {
1084                 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
1085                 kfree(adapter->msix_entries);
1086                 adapter->msix_entries = NULL;
1087                 return err;
1088         }
1089
1090         /* Adjust for only the vectors we'll use, which is minimum
1091          * of max_msix_q_vectors + NONQ_VECS, or the number of
1092          * vectors we were allocated.
1093          */
1094         adapter->num_msix_vectors = err;
1095         return 0;
1096 }
1097
1098 /**
1099  * iavf_free_queues - Free memory for all rings
1100  * @adapter: board private structure to initialize
1101  *
1102  * Free all of the memory associated with queue pairs.
1103  **/
1104 static void iavf_free_queues(struct iavf_adapter *adapter)
1105 {
1106         if (!adapter->vsi_res)
1107                 return;
1108         adapter->num_active_queues = 0;
1109         kfree(adapter->tx_rings);
1110         adapter->tx_rings = NULL;
1111         kfree(adapter->rx_rings);
1112         adapter->rx_rings = NULL;
1113 }
1114
1115 /**
1116  * iavf_alloc_queues - Allocate memory for all rings
1117  * @adapter: board private structure to initialize
1118  *
1119  * We allocate one ring per queue at run-time since we don't know the
1120  * number of queues at compile-time.  The polling_netdev array is
1121  * intended for Multiqueue, but should work fine with a single queue.
1122  **/
1123 static int iavf_alloc_queues(struct iavf_adapter *adapter)
1124 {
1125         int i, num_active_queues;
1126
1127         /* If we're in reset reallocating queues we don't actually know yet for
1128          * certain the PF gave us the number of queues we asked for but we'll
1129          * assume it did.  Once basic reset is finished we'll confirm once we
1130          * start negotiating config with PF.
1131          */
1132         if (adapter->num_req_queues)
1133                 num_active_queues = adapter->num_req_queues;
1134         else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1135                  adapter->num_tc)
1136                 num_active_queues = adapter->ch_config.total_qps;
1137         else
1138                 num_active_queues = min_t(int,
1139                                           adapter->vsi_res->num_queue_pairs,
1140                                           (int)(num_online_cpus()));
1141
1142
1143         adapter->tx_rings = kcalloc(num_active_queues,
1144                                     sizeof(struct iavf_ring), GFP_KERNEL);
1145         if (!adapter->tx_rings)
1146                 goto err_out;
1147         adapter->rx_rings = kcalloc(num_active_queues,
1148                                     sizeof(struct iavf_ring), GFP_KERNEL);
1149         if (!adapter->rx_rings)
1150                 goto err_out;
1151
1152         for (i = 0; i < num_active_queues; i++) {
1153                 struct iavf_ring *tx_ring;
1154                 struct iavf_ring *rx_ring;
1155
1156                 tx_ring = &adapter->tx_rings[i];
1157
1158                 tx_ring->queue_index = i;
1159                 tx_ring->netdev = adapter->netdev;
1160                 tx_ring->dev = &adapter->pdev->dev;
1161                 tx_ring->count = adapter->tx_desc_count;
1162                 tx_ring->itr_setting = IAVF_ITR_TX_DEF;
1163                 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)
1164                         tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR;
1165
1166                 rx_ring = &adapter->rx_rings[i];
1167                 rx_ring->queue_index = i;
1168                 rx_ring->netdev = adapter->netdev;
1169                 rx_ring->dev = &adapter->pdev->dev;
1170                 rx_ring->count = adapter->rx_desc_count;
1171                 rx_ring->itr_setting = IAVF_ITR_RX_DEF;
1172         }
1173
1174         adapter->num_active_queues = num_active_queues;
1175
1176         return 0;
1177
1178 err_out:
1179         iavf_free_queues(adapter);
1180         return -ENOMEM;
1181 }
1182
1183 /**
1184  * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported
1185  * @adapter: board private structure to initialize
1186  *
1187  * Attempt to configure the interrupts using the best available
1188  * capabilities of the hardware and the kernel.
1189  **/
1190 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)
1191 {
1192         int vector, v_budget;
1193         int pairs = 0;
1194         int err = 0;
1195
1196         if (!adapter->vsi_res) {
1197                 err = -EIO;
1198                 goto out;
1199         }
1200         pairs = adapter->num_active_queues;
1201
1202         /* It's easy to be greedy for MSI-X vectors, but it really doesn't do
1203          * us much good if we have more vectors than CPUs. However, we already
1204          * limit the total number of queues by the number of CPUs so we do not
1205          * need any further limiting here.
1206          */
1207         v_budget = min_t(int, pairs + NONQ_VECS,
1208                          (int)adapter->vf_res->max_vectors);
1209
1210         adapter->msix_entries = kcalloc(v_budget,
1211                                         sizeof(struct msix_entry), GFP_KERNEL);
1212         if (!adapter->msix_entries) {
1213                 err = -ENOMEM;
1214                 goto out;
1215         }
1216
1217         for (vector = 0; vector < v_budget; vector++)
1218                 adapter->msix_entries[vector].entry = vector;
1219
1220         err = iavf_acquire_msix_vectors(adapter, v_budget);
1221
1222 out:
1223         netif_set_real_num_rx_queues(adapter->netdev, pairs);
1224         netif_set_real_num_tx_queues(adapter->netdev, pairs);
1225         return err;
1226 }
1227
1228 /**
1229  * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands
1230  * @adapter: board private structure
1231  *
1232  * Return 0 on success, negative on failure
1233  **/
1234 static int iavf_config_rss_aq(struct iavf_adapter *adapter)
1235 {
1236         struct iavf_aqc_get_set_rss_key_data *rss_key =
1237                 (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key;
1238         struct iavf_hw *hw = &adapter->hw;
1239         int ret = 0;
1240
1241         if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
1242                 /* bail because we already have a command pending */
1243                 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
1244                         adapter->current_op);
1245                 return -EBUSY;
1246         }
1247
1248         ret = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);
1249         if (ret) {
1250                 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
1251                         iavf_stat_str(hw, ret),
1252                         iavf_aq_str(hw, hw->aq.asq_last_status));
1253                 return ret;
1254
1255         }
1256
1257         ret = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false,
1258                                   adapter->rss_lut, adapter->rss_lut_size);
1259         if (ret) {
1260                 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
1261                         iavf_stat_str(hw, ret),
1262                         iavf_aq_str(hw, hw->aq.asq_last_status));
1263         }
1264
1265         return ret;
1266
1267 }
1268
1269 /**
1270  * iavf_config_rss_reg - Configure RSS keys and lut by writing registers
1271  * @adapter: board private structure
1272  *
1273  * Returns 0 on success, negative on failure
1274  **/
1275 static int iavf_config_rss_reg(struct iavf_adapter *adapter)
1276 {
1277         struct iavf_hw *hw = &adapter->hw;
1278         u32 *dw;
1279         u16 i;
1280
1281         dw = (u32 *)adapter->rss_key;
1282         for (i = 0; i <= adapter->rss_key_size / 4; i++)
1283                 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]);
1284
1285         dw = (u32 *)adapter->rss_lut;
1286         for (i = 0; i <= adapter->rss_lut_size / 4; i++)
1287                 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]);
1288
1289         iavf_flush(hw);
1290
1291         return 0;
1292 }
1293
1294 /**
1295  * iavf_config_rss - Configure RSS keys and lut
1296  * @adapter: board private structure
1297  *
1298  * Returns 0 on success, negative on failure
1299  **/
1300 int iavf_config_rss(struct iavf_adapter *adapter)
1301 {
1302
1303         if (RSS_PF(adapter)) {
1304                 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |
1305                                         IAVF_FLAG_AQ_SET_RSS_KEY;
1306                 return 0;
1307         } else if (RSS_AQ(adapter)) {
1308                 return iavf_config_rss_aq(adapter);
1309         } else {
1310                 return iavf_config_rss_reg(adapter);
1311         }
1312 }
1313
1314 /**
1315  * iavf_fill_rss_lut - Fill the lut with default values
1316  * @adapter: board private structure
1317  **/
1318 static void iavf_fill_rss_lut(struct iavf_adapter *adapter)
1319 {
1320         u16 i;
1321
1322         for (i = 0; i < adapter->rss_lut_size; i++)
1323                 adapter->rss_lut[i] = i % adapter->num_active_queues;
1324 }
1325
1326 /**
1327  * iavf_init_rss - Prepare for RSS
1328  * @adapter: board private structure
1329  *
1330  * Return 0 on success, negative on failure
1331  **/
1332 static int iavf_init_rss(struct iavf_adapter *adapter)
1333 {
1334         struct iavf_hw *hw = &adapter->hw;
1335         int ret;
1336
1337         if (!RSS_PF(adapter)) {
1338                 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1339                 if (adapter->vf_res->vf_cap_flags &
1340                     VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
1341                         adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED;
1342                 else
1343                         adapter->hena = IAVF_DEFAULT_RSS_HENA;
1344
1345                 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena);
1346                 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32));
1347         }
1348
1349         iavf_fill_rss_lut(adapter);
1350         netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);
1351         ret = iavf_config_rss(adapter);
1352
1353         return ret;
1354 }
1355
1356 /**
1357  * iavf_alloc_q_vectors - Allocate memory for interrupt vectors
1358  * @adapter: board private structure to initialize
1359  *
1360  * We allocate one q_vector per queue interrupt.  If allocation fails we
1361  * return -ENOMEM.
1362  **/
1363 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
1364 {
1365         int q_idx = 0, num_q_vectors;
1366         struct iavf_q_vector *q_vector;
1367
1368         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1369         adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),
1370                                      GFP_KERNEL);
1371         if (!adapter->q_vectors)
1372                 return -ENOMEM;
1373
1374         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1375                 q_vector = &adapter->q_vectors[q_idx];
1376                 q_vector->adapter = adapter;
1377                 q_vector->vsi = &adapter->vsi;
1378                 q_vector->v_idx = q_idx;
1379                 q_vector->reg_idx = q_idx;
1380                 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
1381                 netif_napi_add(adapter->netdev, &q_vector->napi,
1382                                iavf_napi_poll, NAPI_POLL_WEIGHT);
1383         }
1384
1385         return 0;
1386 }
1387
1388 /**
1389  * iavf_free_q_vectors - Free memory allocated for interrupt vectors
1390  * @adapter: board private structure to initialize
1391  *
1392  * This function frees the memory allocated to the q_vectors.  In addition if
1393  * NAPI is enabled it will delete any references to the NAPI struct prior
1394  * to freeing the q_vector.
1395  **/
1396 static void iavf_free_q_vectors(struct iavf_adapter *adapter)
1397 {
1398         int q_idx, num_q_vectors;
1399         int napi_vectors;
1400
1401         if (!adapter->q_vectors)
1402                 return;
1403
1404         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1405         napi_vectors = adapter->num_active_queues;
1406
1407         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1408                 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx];
1409
1410                 if (q_idx < napi_vectors)
1411                         netif_napi_del(&q_vector->napi);
1412         }
1413         kfree(adapter->q_vectors);
1414         adapter->q_vectors = NULL;
1415 }
1416
1417 /**
1418  * iavf_reset_interrupt_capability - Reset MSIX setup
1419  * @adapter: board private structure
1420  *
1421  **/
1422 void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)
1423 {
1424         if (!adapter->msix_entries)
1425                 return;
1426
1427         pci_disable_msix(adapter->pdev);
1428         kfree(adapter->msix_entries);
1429         adapter->msix_entries = NULL;
1430 }
1431
1432 /**
1433  * iavf_init_interrupt_scheme - Determine if MSIX is supported and init
1434  * @adapter: board private structure to initialize
1435  *
1436  **/
1437 int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)
1438 {
1439         int err;
1440
1441         err = iavf_alloc_queues(adapter);
1442         if (err) {
1443                 dev_err(&adapter->pdev->dev,
1444                         "Unable to allocate memory for queues\n");
1445                 goto err_alloc_queues;
1446         }
1447
1448         rtnl_lock();
1449         err = iavf_set_interrupt_capability(adapter);
1450         rtnl_unlock();
1451         if (err) {
1452                 dev_err(&adapter->pdev->dev,
1453                         "Unable to setup interrupt capabilities\n");
1454                 goto err_set_interrupt;
1455         }
1456
1457         err = iavf_alloc_q_vectors(adapter);
1458         if (err) {
1459                 dev_err(&adapter->pdev->dev,
1460                         "Unable to allocate memory for queue vectors\n");
1461                 goto err_alloc_q_vectors;
1462         }
1463
1464         /* If we've made it so far while ADq flag being ON, then we haven't
1465          * bailed out anywhere in middle. And ADq isn't just enabled but actual
1466          * resources have been allocated in the reset path.
1467          * Now we can truly claim that ADq is enabled.
1468          */
1469         if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1470             adapter->num_tc)
1471                 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
1472                          adapter->num_tc);
1473
1474         dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1475                  (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
1476                  adapter->num_active_queues);
1477
1478         return 0;
1479 err_alloc_q_vectors:
1480         iavf_reset_interrupt_capability(adapter);
1481 err_set_interrupt:
1482         iavf_free_queues(adapter);
1483 err_alloc_queues:
1484         return err;
1485 }
1486
1487 /**
1488  * iavf_free_rss - Free memory used by RSS structs
1489  * @adapter: board private structure
1490  **/
1491 static void iavf_free_rss(struct iavf_adapter *adapter)
1492 {
1493         kfree(adapter->rss_key);
1494         adapter->rss_key = NULL;
1495
1496         kfree(adapter->rss_lut);
1497         adapter->rss_lut = NULL;
1498 }
1499
1500 /**
1501  * iavf_reinit_interrupt_scheme - Reallocate queues and vectors
1502  * @adapter: board private structure
1503  *
1504  * Returns 0 on success, negative on failure
1505  **/
1506 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter)
1507 {
1508         struct net_device *netdev = adapter->netdev;
1509         int err;
1510
1511         if (netif_running(netdev))
1512                 iavf_free_traffic_irqs(adapter);
1513         iavf_free_misc_irq(adapter);
1514         iavf_reset_interrupt_capability(adapter);
1515         iavf_free_q_vectors(adapter);
1516         iavf_free_queues(adapter);
1517
1518         err =  iavf_init_interrupt_scheme(adapter);
1519         if (err)
1520                 goto err;
1521
1522         netif_tx_stop_all_queues(netdev);
1523
1524         err = iavf_request_misc_irq(adapter);
1525         if (err)
1526                 goto err;
1527
1528         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1529
1530         iavf_map_rings_to_vectors(adapter);
1531 err:
1532         return err;
1533 }
1534
1535 /**
1536  * iavf_process_aq_command - process aq_required flags
1537  * and sends aq command
1538  * @adapter: pointer to iavf adapter structure
1539  *
1540  * Returns 0 on success
1541  * Returns error code if no command was sent
1542  * or error code if the command failed.
1543  **/
1544 static int iavf_process_aq_command(struct iavf_adapter *adapter)
1545 {
1546         if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG)
1547                 return iavf_send_vf_config_msg(adapter);
1548         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {
1549                 iavf_disable_queues(adapter);
1550                 return 0;
1551         }
1552
1553         if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {
1554                 iavf_map_queues(adapter);
1555                 return 0;
1556         }
1557
1558         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {
1559                 iavf_add_ether_addrs(adapter);
1560                 return 0;
1561         }
1562
1563         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {
1564                 iavf_add_vlans(adapter);
1565                 return 0;
1566         }
1567
1568         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {
1569                 iavf_del_ether_addrs(adapter);
1570                 return 0;
1571         }
1572
1573         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {
1574                 iavf_del_vlans(adapter);
1575                 return 0;
1576         }
1577
1578         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
1579                 iavf_enable_vlan_stripping(adapter);
1580                 return 0;
1581         }
1582
1583         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
1584                 iavf_disable_vlan_stripping(adapter);
1585                 return 0;
1586         }
1587
1588         if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {
1589                 iavf_configure_queues(adapter);
1590                 return 0;
1591         }
1592
1593         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {
1594                 iavf_enable_queues(adapter);
1595                 return 0;
1596         }
1597
1598         if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {
1599                 /* This message goes straight to the firmware, not the
1600                  * PF, so we don't have to set current_op as we will
1601                  * not get a response through the ARQ.
1602                  */
1603                 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;
1604                 return 0;
1605         }
1606         if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {
1607                 iavf_get_hena(adapter);
1608                 return 0;
1609         }
1610         if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {
1611                 iavf_set_hena(adapter);
1612                 return 0;
1613         }
1614         if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {
1615                 iavf_set_rss_key(adapter);
1616                 return 0;
1617         }
1618         if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {
1619                 iavf_set_rss_lut(adapter);
1620                 return 0;
1621         }
1622
1623         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) {
1624                 iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC |
1625                                        FLAG_VF_MULTICAST_PROMISC);
1626                 return 0;
1627         }
1628
1629         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) {
1630                 iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC);
1631                 return 0;
1632         }
1633
1634         if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) &&
1635             (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) {
1636                 iavf_set_promiscuous(adapter, 0);
1637                 return 0;
1638         }
1639
1640         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {
1641                 iavf_enable_channels(adapter);
1642                 return 0;
1643         }
1644
1645         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {
1646                 iavf_disable_channels(adapter);
1647                 return 0;
1648         }
1649         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
1650                 iavf_add_cloud_filter(adapter);
1651                 return 0;
1652         }
1653
1654         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
1655                 iavf_del_cloud_filter(adapter);
1656                 return 0;
1657         }
1658         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
1659                 iavf_del_cloud_filter(adapter);
1660                 return 0;
1661         }
1662         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
1663                 iavf_add_cloud_filter(adapter);
1664                 return 0;
1665         }
1666         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) {
1667                 iavf_add_fdir_filter(adapter);
1668                 return IAVF_SUCCESS;
1669         }
1670         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) {
1671                 iavf_del_fdir_filter(adapter);
1672                 return IAVF_SUCCESS;
1673         }
1674         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) {
1675                 iavf_add_adv_rss_cfg(adapter);
1676                 return 0;
1677         }
1678         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) {
1679                 iavf_del_adv_rss_cfg(adapter);
1680                 return 0;
1681         }
1682         return -EAGAIN;
1683 }
1684
1685 /**
1686  * iavf_startup - first step of driver startup
1687  * @adapter: board private structure
1688  *
1689  * Function process __IAVF_STARTUP driver state.
1690  * When success the state is changed to __IAVF_INIT_VERSION_CHECK
1691  * when fails it returns -EAGAIN
1692  **/
1693 static int iavf_startup(struct iavf_adapter *adapter)
1694 {
1695         struct pci_dev *pdev = adapter->pdev;
1696         struct iavf_hw *hw = &adapter->hw;
1697         int err;
1698
1699         WARN_ON(adapter->state != __IAVF_STARTUP);
1700
1701         /* driver loaded, probe complete */
1702         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
1703         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
1704         err = iavf_set_mac_type(hw);
1705         if (err) {
1706                 dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", err);
1707                 goto err;
1708         }
1709
1710         err = iavf_check_reset_complete(hw);
1711         if (err) {
1712                 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
1713                          err);
1714                 goto err;
1715         }
1716         hw->aq.num_arq_entries = IAVF_AQ_LEN;
1717         hw->aq.num_asq_entries = IAVF_AQ_LEN;
1718         hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
1719         hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
1720
1721         err = iavf_init_adminq(hw);
1722         if (err) {
1723                 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n", err);
1724                 goto err;
1725         }
1726         err = iavf_send_api_ver(adapter);
1727         if (err) {
1728                 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", err);
1729                 iavf_shutdown_adminq(hw);
1730                 goto err;
1731         }
1732         adapter->state = __IAVF_INIT_VERSION_CHECK;
1733 err:
1734         return err;
1735 }
1736
1737 /**
1738  * iavf_init_version_check - second step of driver startup
1739  * @adapter: board private structure
1740  *
1741  * Function process __IAVF_INIT_VERSION_CHECK driver state.
1742  * When success the state is changed to __IAVF_INIT_GET_RESOURCES
1743  * when fails it returns -EAGAIN
1744  **/
1745 static int iavf_init_version_check(struct iavf_adapter *adapter)
1746 {
1747         struct pci_dev *pdev = adapter->pdev;
1748         struct iavf_hw *hw = &adapter->hw;
1749         int err = -EAGAIN;
1750
1751         WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK);
1752
1753         if (!iavf_asq_done(hw)) {
1754                 dev_err(&pdev->dev, "Admin queue command never completed\n");
1755                 iavf_shutdown_adminq(hw);
1756                 adapter->state = __IAVF_STARTUP;
1757                 goto err;
1758         }
1759
1760         /* aq msg sent, awaiting reply */
1761         err = iavf_verify_api_ver(adapter);
1762         if (err) {
1763                 if (err == IAVF_ERR_ADMIN_QUEUE_NO_WORK)
1764                         err = iavf_send_api_ver(adapter);
1765                 else
1766                         dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
1767                                 adapter->pf_version.major,
1768                                 adapter->pf_version.minor,
1769                                 VIRTCHNL_VERSION_MAJOR,
1770                                 VIRTCHNL_VERSION_MINOR);
1771                 goto err;
1772         }
1773         err = iavf_send_vf_config_msg(adapter);
1774         if (err) {
1775                 dev_err(&pdev->dev, "Unable to send config request (%d)\n",
1776                         err);
1777                 goto err;
1778         }
1779         adapter->state = __IAVF_INIT_GET_RESOURCES;
1780
1781 err:
1782         return err;
1783 }
1784
1785 /**
1786  * iavf_init_get_resources - third step of driver startup
1787  * @adapter: board private structure
1788  *
1789  * Function process __IAVF_INIT_GET_RESOURCES driver state and
1790  * finishes driver initialization procedure.
1791  * When success the state is changed to __IAVF_DOWN
1792  * when fails it returns -EAGAIN
1793  **/
1794 static int iavf_init_get_resources(struct iavf_adapter *adapter)
1795 {
1796         struct net_device *netdev = adapter->netdev;
1797         struct pci_dev *pdev = adapter->pdev;
1798         struct iavf_hw *hw = &adapter->hw;
1799         int err;
1800
1801         WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);
1802         /* aq msg sent, awaiting reply */
1803         if (!adapter->vf_res) {
1804                 adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,
1805                                           GFP_KERNEL);
1806                 if (!adapter->vf_res) {
1807                         err = -ENOMEM;
1808                         goto err;
1809                 }
1810         }
1811         err = iavf_get_vf_config(adapter);
1812         if (err == IAVF_ERR_ADMIN_QUEUE_NO_WORK) {
1813                 err = iavf_send_vf_config_msg(adapter);
1814                 goto err;
1815         } else if (err == IAVF_ERR_PARAM) {
1816                 /* We only get ERR_PARAM if the device is in a very bad
1817                  * state or if we've been disabled for previous bad
1818                  * behavior. Either way, we're done now.
1819                  */
1820                 iavf_shutdown_adminq(hw);
1821                 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
1822                 return 0;
1823         }
1824         if (err) {
1825                 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);
1826                 goto err_alloc;
1827         }
1828
1829         err = iavf_process_config(adapter);
1830         if (err)
1831                 goto err_alloc;
1832         adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1833
1834         adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;
1835
1836         netdev->netdev_ops = &iavf_netdev_ops;
1837         iavf_set_ethtool_ops(netdev);
1838         netdev->watchdog_timeo = 5 * HZ;
1839
1840         /* MTU range: 68 - 9710 */
1841         netdev->min_mtu = ETH_MIN_MTU;
1842         netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD;
1843
1844         if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
1845                 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
1846                          adapter->hw.mac.addr);
1847                 eth_hw_addr_random(netdev);
1848                 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
1849         } else {
1850                 eth_hw_addr_set(netdev, adapter->hw.mac.addr);
1851                 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
1852         }
1853
1854         adapter->tx_desc_count = IAVF_DEFAULT_TXD;
1855         adapter->rx_desc_count = IAVF_DEFAULT_RXD;
1856         err = iavf_init_interrupt_scheme(adapter);
1857         if (err)
1858                 goto err_sw_init;
1859         iavf_map_rings_to_vectors(adapter);
1860         if (adapter->vf_res->vf_cap_flags &
1861                 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
1862                 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;
1863
1864         err = iavf_request_misc_irq(adapter);
1865         if (err)
1866                 goto err_sw_init;
1867
1868         netif_carrier_off(netdev);
1869         adapter->link_up = false;
1870
1871         /* set the semaphore to prevent any callbacks after device registration
1872          * up to time when state of driver will be set to __IAVF_DOWN
1873          */
1874         rtnl_lock();
1875         if (!adapter->netdev_registered) {
1876                 err = register_netdevice(netdev);
1877                 if (err) {
1878                         rtnl_unlock();
1879                         goto err_register;
1880                 }
1881         }
1882
1883         adapter->netdev_registered = true;
1884
1885         netif_tx_stop_all_queues(netdev);
1886         if (CLIENT_ALLOWED(adapter)) {
1887                 err = iavf_lan_add_device(adapter);
1888                 if (err)
1889                         dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n",
1890                                  err);
1891         }
1892         dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
1893         if (netdev->features & NETIF_F_GRO)
1894                 dev_info(&pdev->dev, "GRO is enabled\n");
1895
1896         adapter->state = __IAVF_DOWN;
1897         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1898         rtnl_unlock();
1899
1900         iavf_misc_irq_enable(adapter);
1901         wake_up(&adapter->down_waitqueue);
1902
1903         adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
1904         adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
1905         if (!adapter->rss_key || !adapter->rss_lut) {
1906                 err = -ENOMEM;
1907                 goto err_mem;
1908         }
1909         if (RSS_AQ(adapter))
1910                 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
1911         else
1912                 iavf_init_rss(adapter);
1913
1914         return err;
1915 err_mem:
1916         iavf_free_rss(adapter);
1917 err_register:
1918         iavf_free_misc_irq(adapter);
1919 err_sw_init:
1920         iavf_reset_interrupt_capability(adapter);
1921 err_alloc:
1922         kfree(adapter->vf_res);
1923         adapter->vf_res = NULL;
1924 err:
1925         return err;
1926 }
1927
1928 /**
1929  * iavf_watchdog_task - Periodic call-back task
1930  * @work: pointer to work_struct
1931  **/
1932 static void iavf_watchdog_task(struct work_struct *work)
1933 {
1934         struct iavf_adapter *adapter = container_of(work,
1935                                                     struct iavf_adapter,
1936                                                     watchdog_task.work);
1937         struct iavf_hw *hw = &adapter->hw;
1938         u32 reg_val;
1939
1940         if (!mutex_trylock(&adapter->crit_lock))
1941                 goto restart_watchdog;
1942
1943         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
1944                 adapter->state = __IAVF_COMM_FAILED;
1945
1946         switch (adapter->state) {
1947         case __IAVF_COMM_FAILED:
1948                 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
1949                           IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
1950                 if (reg_val == VIRTCHNL_VFR_VFACTIVE ||
1951                     reg_val == VIRTCHNL_VFR_COMPLETED) {
1952                         /* A chance for redemption! */
1953                         dev_err(&adapter->pdev->dev,
1954                                 "Hardware came out of reset. Attempting reinit.\n");
1955                         adapter->state = __IAVF_STARTUP;
1956                         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
1957                         queue_delayed_work(iavf_wq, &adapter->init_task, 10);
1958                         mutex_unlock(&adapter->crit_lock);
1959                         /* Don't reschedule the watchdog, since we've restarted
1960                          * the init task. When init_task contacts the PF and
1961                          * gets everything set up again, it'll restart the
1962                          * watchdog for us. Down, boy. Sit. Stay. Woof.
1963                          */
1964                         return;
1965                 }
1966                 adapter->aq_required = 0;
1967                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1968                 mutex_unlock(&adapter->crit_lock);
1969                 queue_delayed_work(iavf_wq,
1970                                    &adapter->watchdog_task,
1971                                    msecs_to_jiffies(10));
1972                 goto watchdog_done;
1973         case __IAVF_RESETTING:
1974                 mutex_unlock(&adapter->crit_lock);
1975                 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2);
1976                 return;
1977         case __IAVF_DOWN:
1978         case __IAVF_DOWN_PENDING:
1979         case __IAVF_TESTING:
1980         case __IAVF_RUNNING:
1981                 if (adapter->current_op) {
1982                         if (!iavf_asq_done(hw)) {
1983                                 dev_dbg(&adapter->pdev->dev,
1984                                         "Admin queue timeout\n");
1985                                 iavf_send_api_ver(adapter);
1986                         }
1987                 } else {
1988                         /* An error will be returned if no commands were
1989                          * processed; use this opportunity to update stats
1990                          */
1991                         if (iavf_process_aq_command(adapter) &&
1992                             adapter->state == __IAVF_RUNNING)
1993                                 iavf_request_stats(adapter);
1994                 }
1995                 break;
1996         case __IAVF_REMOVE:
1997                 mutex_unlock(&adapter->crit_lock);
1998                 return;
1999         default:
2000                 goto restart_watchdog;
2001         }
2002
2003                 /* check for hw reset */
2004         reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2005         if (!reg_val) {
2006                 adapter->flags |= IAVF_FLAG_RESET_PENDING;
2007                 adapter->aq_required = 0;
2008                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2009                 dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
2010                 queue_work(iavf_wq, &adapter->reset_task);
2011                 goto watchdog_done;
2012         }
2013
2014         schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5));
2015 watchdog_done:
2016         if (adapter->state == __IAVF_RUNNING ||
2017             adapter->state == __IAVF_COMM_FAILED)
2018                 iavf_detect_recover_hung(&adapter->vsi);
2019         mutex_unlock(&adapter->crit_lock);
2020 restart_watchdog:
2021         if (adapter->aq_required)
2022                 queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2023                                    msecs_to_jiffies(20));
2024         else
2025                 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2);
2026         queue_work(iavf_wq, &adapter->adminq_task);
2027 }
2028
2029 static void iavf_disable_vf(struct iavf_adapter *adapter)
2030 {
2031         struct iavf_mac_filter *f, *ftmp;
2032         struct iavf_vlan_filter *fv, *fvtmp;
2033         struct iavf_cloud_filter *cf, *cftmp;
2034
2035         adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2036
2037         /* We don't use netif_running() because it may be true prior to
2038          * ndo_open() returning, so we can't assume it means all our open
2039          * tasks have finished, since we're not holding the rtnl_lock here.
2040          */
2041         if (adapter->state == __IAVF_RUNNING) {
2042                 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2043                 netif_carrier_off(adapter->netdev);
2044                 netif_tx_disable(adapter->netdev);
2045                 adapter->link_up = false;
2046                 iavf_napi_disable_all(adapter);
2047                 iavf_irq_disable(adapter);
2048                 iavf_free_traffic_irqs(adapter);
2049                 iavf_free_all_tx_resources(adapter);
2050                 iavf_free_all_rx_resources(adapter);
2051         }
2052
2053         spin_lock_bh(&adapter->mac_vlan_list_lock);
2054
2055         /* Delete all of the filters */
2056         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2057                 list_del(&f->list);
2058                 kfree(f);
2059         }
2060
2061         list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
2062                 list_del(&fv->list);
2063                 kfree(fv);
2064         }
2065
2066         spin_unlock_bh(&adapter->mac_vlan_list_lock);
2067
2068         spin_lock_bh(&adapter->cloud_filter_list_lock);
2069         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
2070                 list_del(&cf->list);
2071                 kfree(cf);
2072                 adapter->num_cloud_filters--;
2073         }
2074         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2075
2076         iavf_free_misc_irq(adapter);
2077         iavf_reset_interrupt_capability(adapter);
2078         iavf_free_queues(adapter);
2079         iavf_free_q_vectors(adapter);
2080         memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
2081         iavf_shutdown_adminq(&adapter->hw);
2082         adapter->netdev->flags &= ~IFF_UP;
2083         mutex_unlock(&adapter->crit_lock);
2084         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2085         adapter->state = __IAVF_DOWN;
2086         wake_up(&adapter->down_waitqueue);
2087         dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
2088 }
2089
2090 /**
2091  * iavf_reset_task - Call-back task to handle hardware reset
2092  * @work: pointer to work_struct
2093  *
2094  * During reset we need to shut down and reinitialize the admin queue
2095  * before we can use it to communicate with the PF again. We also clear
2096  * and reinit the rings because that context is lost as well.
2097  **/
2098 static void iavf_reset_task(struct work_struct *work)
2099 {
2100         struct iavf_adapter *adapter = container_of(work,
2101                                                       struct iavf_adapter,
2102                                                       reset_task);
2103         struct virtchnl_vf_resource *vfres = adapter->vf_res;
2104         struct net_device *netdev = adapter->netdev;
2105         struct iavf_hw *hw = &adapter->hw;
2106         struct iavf_mac_filter *f, *ftmp;
2107         struct iavf_vlan_filter *vlf;
2108         struct iavf_cloud_filter *cf;
2109         u32 reg_val;
2110         int i = 0, err;
2111         bool running;
2112
2113         /* When device is being removed it doesn't make sense to run the reset
2114          * task, just return in such a case.
2115          */
2116         if (mutex_is_locked(&adapter->remove_lock))
2117                 return;
2118
2119         if (iavf_lock_timeout(&adapter->crit_lock, 200)) {
2120                 schedule_work(&adapter->reset_task);
2121                 return;
2122         }
2123         while (!mutex_trylock(&adapter->client_lock))
2124                 usleep_range(500, 1000);
2125         if (CLIENT_ENABLED(adapter)) {
2126                 adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN |
2127                                     IAVF_FLAG_CLIENT_NEEDS_CLOSE |
2128                                     IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS |
2129                                     IAVF_FLAG_SERVICE_CLIENT_REQUESTED);
2130                 cancel_delayed_work_sync(&adapter->client_task);
2131                 iavf_notify_client_close(&adapter->vsi, true);
2132         }
2133         iavf_misc_irq_disable(adapter);
2134         if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
2135                 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;
2136                 /* Restart the AQ here. If we have been reset but didn't
2137                  * detect it, or if the PF had to reinit, our AQ will be hosed.
2138                  */
2139                 iavf_shutdown_adminq(hw);
2140                 iavf_init_adminq(hw);
2141                 iavf_request_reset(adapter);
2142         }
2143         adapter->flags |= IAVF_FLAG_RESET_PENDING;
2144
2145         /* poll until we see the reset actually happen */
2146         for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {
2147                 reg_val = rd32(hw, IAVF_VF_ARQLEN1) &
2148                           IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2149                 if (!reg_val)
2150                         break;
2151                 usleep_range(5000, 10000);
2152         }
2153         if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {
2154                 dev_info(&adapter->pdev->dev, "Never saw reset\n");
2155                 goto continue_reset; /* act like the reset happened */
2156         }
2157
2158         /* wait until the reset is complete and the PF is responding to us */
2159         for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
2160                 /* sleep first to make sure a minimum wait time is met */
2161                 msleep(IAVF_RESET_WAIT_MS);
2162
2163                 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
2164                           IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
2165                 if (reg_val == VIRTCHNL_VFR_VFACTIVE)
2166                         break;
2167         }
2168
2169         pci_set_master(adapter->pdev);
2170
2171         if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
2172                 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
2173                         reg_val);
2174                 iavf_disable_vf(adapter);
2175                 mutex_unlock(&adapter->client_lock);
2176                 return; /* Do not attempt to reinit. It's dead, Jim. */
2177         }
2178
2179 continue_reset:
2180         /* We don't use netif_running() because it may be true prior to
2181          * ndo_open() returning, so we can't assume it means all our open
2182          * tasks have finished, since we're not holding the rtnl_lock here.
2183          */
2184         running = ((adapter->state == __IAVF_RUNNING) ||
2185                    (adapter->state == __IAVF_RESETTING));
2186
2187         if (running) {
2188                 netif_carrier_off(netdev);
2189                 netif_tx_stop_all_queues(netdev);
2190                 adapter->link_up = false;
2191                 iavf_napi_disable_all(adapter);
2192         }
2193         iavf_irq_disable(adapter);
2194
2195         adapter->state = __IAVF_RESETTING;
2196         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2197
2198         /* free the Tx/Rx rings and descriptors, might be better to just
2199          * re-use them sometime in the future
2200          */
2201         iavf_free_all_rx_resources(adapter);
2202         iavf_free_all_tx_resources(adapter);
2203
2204         adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;
2205         /* kill and reinit the admin queue */
2206         iavf_shutdown_adminq(hw);
2207         adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2208         err = iavf_init_adminq(hw);
2209         if (err)
2210                 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
2211                          err);
2212         adapter->aq_required = 0;
2213
2214         if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) {
2215                 err = iavf_reinit_interrupt_scheme(adapter);
2216                 if (err)
2217                         goto reset_err;
2218         }
2219
2220         if (RSS_AQ(adapter)) {
2221                 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
2222         } else {
2223                 err = iavf_init_rss(adapter);
2224                 if (err)
2225                         goto reset_err;
2226         }
2227
2228         adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;
2229         adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
2230
2231         spin_lock_bh(&adapter->mac_vlan_list_lock);
2232
2233         /* Delete filter for the current MAC address, it could have
2234          * been changed by the PF via administratively set MAC.
2235          * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES.
2236          */
2237         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2238                 if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) {
2239                         list_del(&f->list);
2240                         kfree(f);
2241                 }
2242         }
2243         /* re-add all MAC filters */
2244         list_for_each_entry(f, &adapter->mac_filter_list, list) {
2245                 f->add = true;
2246         }
2247         /* re-add all VLAN filters */
2248         list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
2249                 vlf->add = true;
2250         }
2251
2252         spin_unlock_bh(&adapter->mac_vlan_list_lock);
2253
2254         /* check if TCs are running and re-add all cloud filters */
2255         spin_lock_bh(&adapter->cloud_filter_list_lock);
2256         if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
2257             adapter->num_tc) {
2258                 list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
2259                         cf->add = true;
2260                 }
2261         }
2262         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2263
2264         adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
2265         adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
2266         adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
2267         iavf_misc_irq_enable(adapter);
2268
2269         mod_delayed_work(iavf_wq, &adapter->watchdog_task, 2);
2270
2271         /* We were running when the reset started, so we need to restore some
2272          * state here.
2273          */
2274         if (running) {
2275                 /* allocate transmit descriptors */
2276                 err = iavf_setup_all_tx_resources(adapter);
2277                 if (err)
2278                         goto reset_err;
2279
2280                 /* allocate receive descriptors */
2281                 err = iavf_setup_all_rx_resources(adapter);
2282                 if (err)
2283                         goto reset_err;
2284
2285                 if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) {
2286                         err = iavf_request_traffic_irqs(adapter, netdev->name);
2287                         if (err)
2288                                 goto reset_err;
2289
2290                         adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
2291                 }
2292
2293                 iavf_configure(adapter);
2294
2295                 iavf_up_complete(adapter);
2296
2297                 iavf_irq_enable(adapter, true);
2298         } else {
2299                 adapter->state = __IAVF_DOWN;
2300                 wake_up(&adapter->down_waitqueue);
2301         }
2302         mutex_unlock(&adapter->client_lock);
2303         mutex_unlock(&adapter->crit_lock);
2304
2305         return;
2306 reset_err:
2307         mutex_unlock(&adapter->client_lock);
2308         mutex_unlock(&adapter->crit_lock);
2309         dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
2310         iavf_close(netdev);
2311 }
2312
2313 /**
2314  * iavf_adminq_task - worker thread to clean the admin queue
2315  * @work: pointer to work_struct containing our data
2316  **/
2317 static void iavf_adminq_task(struct work_struct *work)
2318 {
2319         struct iavf_adapter *adapter =
2320                 container_of(work, struct iavf_adapter, adminq_task);
2321         struct iavf_hw *hw = &adapter->hw;
2322         struct iavf_arq_event_info event;
2323         enum virtchnl_ops v_op;
2324         enum iavf_status ret, v_ret;
2325         u32 val, oldval;
2326         u16 pending;
2327
2328         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
2329                 goto out;
2330
2331         event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
2332         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
2333         if (!event.msg_buf)
2334                 goto out;
2335
2336         if (iavf_lock_timeout(&adapter->crit_lock, 200))
2337                 goto freedom;
2338         do {
2339                 ret = iavf_clean_arq_element(hw, &event, &pending);
2340                 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
2341                 v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low);
2342
2343                 if (ret || !v_op)
2344                         break; /* No event to process or error cleaning ARQ */
2345
2346                 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
2347                                          event.msg_len);
2348                 if (pending != 0)
2349                         memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);
2350         } while (pending);
2351         mutex_unlock(&adapter->crit_lock);
2352
2353         if ((adapter->flags &
2354              (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) ||
2355             adapter->state == __IAVF_RESETTING)
2356                 goto freedom;
2357
2358         /* check for error indications */
2359         val = rd32(hw, hw->aq.arq.len);
2360         if (val == 0xdeadbeef) /* indicates device in reset */
2361                 goto freedom;
2362         oldval = val;
2363         if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {
2364                 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
2365                 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK;
2366         }
2367         if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) {
2368                 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
2369                 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK;
2370         }
2371         if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) {
2372                 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
2373                 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK;
2374         }
2375         if (oldval != val)
2376                 wr32(hw, hw->aq.arq.len, val);
2377
2378         val = rd32(hw, hw->aq.asq.len);
2379         oldval = val;
2380         if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) {
2381                 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
2382                 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK;
2383         }
2384         if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) {
2385                 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
2386                 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK;
2387         }
2388         if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
2389                 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
2390                 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK;
2391         }
2392         if (oldval != val)
2393                 wr32(hw, hw->aq.asq.len, val);
2394
2395 freedom:
2396         kfree(event.msg_buf);
2397 out:
2398         /* re-enable Admin queue interrupt cause */
2399         iavf_misc_irq_enable(adapter);
2400 }
2401
2402 /**
2403  * iavf_client_task - worker thread to perform client work
2404  * @work: pointer to work_struct containing our data
2405  *
2406  * This task handles client interactions. Because client calls can be
2407  * reentrant, we can't handle them in the watchdog.
2408  **/
2409 static void iavf_client_task(struct work_struct *work)
2410 {
2411         struct iavf_adapter *adapter =
2412                 container_of(work, struct iavf_adapter, client_task.work);
2413
2414         /* If we can't get the client bit, just give up. We'll be rescheduled
2415          * later.
2416          */
2417
2418         if (!mutex_trylock(&adapter->client_lock))
2419                 return;
2420
2421         if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) {
2422                 iavf_client_subtask(adapter);
2423                 adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
2424                 goto out;
2425         }
2426         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) {
2427                 iavf_notify_client_l2_params(&adapter->vsi);
2428                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS;
2429                 goto out;
2430         }
2431         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) {
2432                 iavf_notify_client_close(&adapter->vsi, false);
2433                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE;
2434                 goto out;
2435         }
2436         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) {
2437                 iavf_notify_client_open(&adapter->vsi);
2438                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN;
2439         }
2440 out:
2441         mutex_unlock(&adapter->client_lock);
2442 }
2443
2444 /**
2445  * iavf_free_all_tx_resources - Free Tx Resources for All Queues
2446  * @adapter: board private structure
2447  *
2448  * Free all transmit software resources
2449  **/
2450 void iavf_free_all_tx_resources(struct iavf_adapter *adapter)
2451 {
2452         int i;
2453
2454         if (!adapter->tx_rings)
2455                 return;
2456
2457         for (i = 0; i < adapter->num_active_queues; i++)
2458                 if (adapter->tx_rings[i].desc)
2459                         iavf_free_tx_resources(&adapter->tx_rings[i]);
2460 }
2461
2462 /**
2463  * iavf_setup_all_tx_resources - allocate all queues Tx resources
2464  * @adapter: board private structure
2465  *
2466  * If this function returns with an error, then it's possible one or
2467  * more of the rings is populated (while the rest are not).  It is the
2468  * callers duty to clean those orphaned rings.
2469  *
2470  * Return 0 on success, negative on failure
2471  **/
2472 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
2473 {
2474         int i, err = 0;
2475
2476         for (i = 0; i < adapter->num_active_queues; i++) {
2477                 adapter->tx_rings[i].count = adapter->tx_desc_count;
2478                 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]);
2479                 if (!err)
2480                         continue;
2481                 dev_err(&adapter->pdev->dev,
2482                         "Allocation for Tx Queue %u failed\n", i);
2483                 break;
2484         }
2485
2486         return err;
2487 }
2488
2489 /**
2490  * iavf_setup_all_rx_resources - allocate all queues Rx resources
2491  * @adapter: board private structure
2492  *
2493  * If this function returns with an error, then it's possible one or
2494  * more of the rings is populated (while the rest are not).  It is the
2495  * callers duty to clean those orphaned rings.
2496  *
2497  * Return 0 on success, negative on failure
2498  **/
2499 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
2500 {
2501         int i, err = 0;
2502
2503         for (i = 0; i < adapter->num_active_queues; i++) {
2504                 adapter->rx_rings[i].count = adapter->rx_desc_count;
2505                 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);
2506                 if (!err)
2507                         continue;
2508                 dev_err(&adapter->pdev->dev,
2509                         "Allocation for Rx Queue %u failed\n", i);
2510                 break;
2511         }
2512         return err;
2513 }
2514
2515 /**
2516  * iavf_free_all_rx_resources - Free Rx Resources for All Queues
2517  * @adapter: board private structure
2518  *
2519  * Free all receive software resources
2520  **/
2521 void iavf_free_all_rx_resources(struct iavf_adapter *adapter)
2522 {
2523         int i;
2524
2525         if (!adapter->rx_rings)
2526                 return;
2527
2528         for (i = 0; i < adapter->num_active_queues; i++)
2529                 if (adapter->rx_rings[i].desc)
2530                         iavf_free_rx_resources(&adapter->rx_rings[i]);
2531 }
2532
2533 /**
2534  * iavf_validate_tx_bandwidth - validate the max Tx bandwidth
2535  * @adapter: board private structure
2536  * @max_tx_rate: max Tx bw for a tc
2537  **/
2538 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
2539                                       u64 max_tx_rate)
2540 {
2541         int speed = 0, ret = 0;
2542
2543         if (ADV_LINK_SUPPORT(adapter)) {
2544                 if (adapter->link_speed_mbps < U32_MAX) {
2545                         speed = adapter->link_speed_mbps;
2546                         goto validate_bw;
2547                 } else {
2548                         dev_err(&adapter->pdev->dev, "Unknown link speed\n");
2549                         return -EINVAL;
2550                 }
2551         }
2552
2553         switch (adapter->link_speed) {
2554         case VIRTCHNL_LINK_SPEED_40GB:
2555                 speed = SPEED_40000;
2556                 break;
2557         case VIRTCHNL_LINK_SPEED_25GB:
2558                 speed = SPEED_25000;
2559                 break;
2560         case VIRTCHNL_LINK_SPEED_20GB:
2561                 speed = SPEED_20000;
2562                 break;
2563         case VIRTCHNL_LINK_SPEED_10GB:
2564                 speed = SPEED_10000;
2565                 break;
2566         case VIRTCHNL_LINK_SPEED_5GB:
2567                 speed = SPEED_5000;
2568                 break;
2569         case VIRTCHNL_LINK_SPEED_2_5GB:
2570                 speed = SPEED_2500;
2571                 break;
2572         case VIRTCHNL_LINK_SPEED_1GB:
2573                 speed = SPEED_1000;
2574                 break;
2575         case VIRTCHNL_LINK_SPEED_100MB:
2576                 speed = SPEED_100;
2577                 break;
2578         default:
2579                 break;
2580         }
2581
2582 validate_bw:
2583         if (max_tx_rate > speed) {
2584                 dev_err(&adapter->pdev->dev,
2585                         "Invalid tx rate specified\n");
2586                 ret = -EINVAL;
2587         }
2588
2589         return ret;
2590 }
2591
2592 /**
2593  * iavf_validate_ch_config - validate queue mapping info
2594  * @adapter: board private structure
2595  * @mqprio_qopt: queue parameters
2596  *
2597  * This function validates if the config provided by the user to
2598  * configure queue channels is valid or not. Returns 0 on a valid
2599  * config.
2600  **/
2601 static int iavf_validate_ch_config(struct iavf_adapter *adapter,
2602                                    struct tc_mqprio_qopt_offload *mqprio_qopt)
2603 {
2604         u64 total_max_rate = 0;
2605         int i, num_qps = 0;
2606         u64 tx_rate = 0;
2607         int ret = 0;
2608
2609         if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||
2610             mqprio_qopt->qopt.num_tc < 1)
2611                 return -EINVAL;
2612
2613         for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
2614                 if (!mqprio_qopt->qopt.count[i] ||
2615                     mqprio_qopt->qopt.offset[i] != num_qps)
2616                         return -EINVAL;
2617                 if (mqprio_qopt->min_rate[i]) {
2618                         dev_err(&adapter->pdev->dev,
2619                                 "Invalid min tx rate (greater than 0) specified\n");
2620                         return -EINVAL;
2621                 }
2622                 /*convert to Mbps */
2623                 tx_rate = div_u64(mqprio_qopt->max_rate[i],
2624                                   IAVF_MBPS_DIVISOR);
2625                 total_max_rate += tx_rate;
2626                 num_qps += mqprio_qopt->qopt.count[i];
2627         }
2628         if (num_qps > IAVF_MAX_REQ_QUEUES)
2629                 return -EINVAL;
2630
2631         ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);
2632         return ret;
2633 }
2634
2635 /**
2636  * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes
2637  * @adapter: board private structure
2638  **/
2639 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)
2640 {
2641         struct iavf_cloud_filter *cf, *cftmp;
2642
2643         spin_lock_bh(&adapter->cloud_filter_list_lock);
2644         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
2645                                  list) {
2646                 list_del(&cf->list);
2647                 kfree(cf);
2648                 adapter->num_cloud_filters--;
2649         }
2650         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2651 }
2652
2653 /**
2654  * __iavf_setup_tc - configure multiple traffic classes
2655  * @netdev: network interface device structure
2656  * @type_data: tc offload data
2657  *
2658  * This function processes the config information provided by the
2659  * user to configure traffic classes/queue channels and packages the
2660  * information to request the PF to setup traffic classes.
2661  *
2662  * Returns 0 on success.
2663  **/
2664 static int __iavf_setup_tc(struct net_device *netdev, void *type_data)
2665 {
2666         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
2667         struct iavf_adapter *adapter = netdev_priv(netdev);
2668         struct virtchnl_vf_resource *vfres = adapter->vf_res;
2669         u8 num_tc = 0, total_qps = 0;
2670         int ret = 0, netdev_tc = 0;
2671         u64 max_tx_rate;
2672         u16 mode;
2673         int i;
2674
2675         num_tc = mqprio_qopt->qopt.num_tc;
2676         mode = mqprio_qopt->mode;
2677
2678         /* delete queue_channel */
2679         if (!mqprio_qopt->qopt.hw) {
2680                 if (adapter->ch_config.state == __IAVF_TC_RUNNING) {
2681                         /* reset the tc configuration */
2682                         netdev_reset_tc(netdev);
2683                         adapter->num_tc = 0;
2684                         netif_tx_stop_all_queues(netdev);
2685                         netif_tx_disable(netdev);
2686                         iavf_del_all_cloud_filters(adapter);
2687                         adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;
2688                         goto exit;
2689                 } else {
2690                         return -EINVAL;
2691                 }
2692         }
2693
2694         /* add queue channel */
2695         if (mode == TC_MQPRIO_MODE_CHANNEL) {
2696                 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
2697                         dev_err(&adapter->pdev->dev, "ADq not supported\n");
2698                         return -EOPNOTSUPP;
2699                 }
2700                 if (adapter->ch_config.state != __IAVF_TC_INVALID) {
2701                         dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
2702                         return -EINVAL;
2703                 }
2704
2705                 ret = iavf_validate_ch_config(adapter, mqprio_qopt);
2706                 if (ret)
2707                         return ret;
2708                 /* Return if same TC config is requested */
2709                 if (adapter->num_tc == num_tc)
2710                         return 0;
2711                 adapter->num_tc = num_tc;
2712
2713                 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
2714                         if (i < num_tc) {
2715                                 adapter->ch_config.ch_info[i].count =
2716                                         mqprio_qopt->qopt.count[i];
2717                                 adapter->ch_config.ch_info[i].offset =
2718                                         mqprio_qopt->qopt.offset[i];
2719                                 total_qps += mqprio_qopt->qopt.count[i];
2720                                 max_tx_rate = mqprio_qopt->max_rate[i];
2721                                 /* convert to Mbps */
2722                                 max_tx_rate = div_u64(max_tx_rate,
2723                                                       IAVF_MBPS_DIVISOR);
2724                                 adapter->ch_config.ch_info[i].max_tx_rate =
2725                                         max_tx_rate;
2726                         } else {
2727                                 adapter->ch_config.ch_info[i].count = 1;
2728                                 adapter->ch_config.ch_info[i].offset = 0;
2729                         }
2730                 }
2731                 adapter->ch_config.total_qps = total_qps;
2732                 netif_tx_stop_all_queues(netdev);
2733                 netif_tx_disable(netdev);
2734                 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;
2735                 netdev_reset_tc(netdev);
2736                 /* Report the tc mapping up the stack */
2737                 netdev_set_num_tc(adapter->netdev, num_tc);
2738                 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
2739                         u16 qcount = mqprio_qopt->qopt.count[i];
2740                         u16 qoffset = mqprio_qopt->qopt.offset[i];
2741
2742                         if (i < num_tc)
2743                                 netdev_set_tc_queue(netdev, netdev_tc++, qcount,
2744                                                     qoffset);
2745                 }
2746         }
2747 exit:
2748         return ret;
2749 }
2750
2751 /**
2752  * iavf_parse_cls_flower - Parse tc flower filters provided by kernel
2753  * @adapter: board private structure
2754  * @f: pointer to struct flow_cls_offload
2755  * @filter: pointer to cloud filter structure
2756  */
2757 static int iavf_parse_cls_flower(struct iavf_adapter *adapter,
2758                                  struct flow_cls_offload *f,
2759                                  struct iavf_cloud_filter *filter)
2760 {
2761         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
2762         struct flow_dissector *dissector = rule->match.dissector;
2763         u16 n_proto_mask = 0;
2764         u16 n_proto_key = 0;
2765         u8 field_flags = 0;
2766         u16 addr_type = 0;
2767         u16 n_proto = 0;
2768         int i = 0;
2769         struct virtchnl_filter *vf = &filter->f;
2770
2771         if (dissector->used_keys &
2772             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
2773               BIT(FLOW_DISSECTOR_KEY_BASIC) |
2774               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
2775               BIT(FLOW_DISSECTOR_KEY_VLAN) |
2776               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
2777               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
2778               BIT(FLOW_DISSECTOR_KEY_PORTS) |
2779               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
2780                 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n",
2781                         dissector->used_keys);
2782                 return -EOPNOTSUPP;
2783         }
2784
2785         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
2786                 struct flow_match_enc_keyid match;
2787
2788                 flow_rule_match_enc_keyid(rule, &match);
2789                 if (match.mask->keyid != 0)
2790                         field_flags |= IAVF_CLOUD_FIELD_TEN_ID;
2791         }
2792
2793         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
2794                 struct flow_match_basic match;
2795
2796                 flow_rule_match_basic(rule, &match);
2797                 n_proto_key = ntohs(match.key->n_proto);
2798                 n_proto_mask = ntohs(match.mask->n_proto);
2799
2800                 if (n_proto_key == ETH_P_ALL) {
2801                         n_proto_key = 0;
2802                         n_proto_mask = 0;
2803                 }
2804                 n_proto = n_proto_key & n_proto_mask;
2805                 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
2806                         return -EINVAL;
2807                 if (n_proto == ETH_P_IPV6) {
2808                         /* specify flow type as TCP IPv6 */
2809                         vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
2810                 }
2811
2812                 if (match.key->ip_proto != IPPROTO_TCP) {
2813                         dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
2814                         return -EINVAL;
2815                 }
2816         }
2817
2818         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
2819                 struct flow_match_eth_addrs match;
2820
2821                 flow_rule_match_eth_addrs(rule, &match);
2822
2823                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
2824                 if (!is_zero_ether_addr(match.mask->dst)) {
2825                         if (is_broadcast_ether_addr(match.mask->dst)) {
2826                                 field_flags |= IAVF_CLOUD_FIELD_OMAC;
2827                         } else {
2828                                 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
2829                                         match.mask->dst);
2830                                 return IAVF_ERR_CONFIG;
2831                         }
2832                 }
2833
2834                 if (!is_zero_ether_addr(match.mask->src)) {
2835                         if (is_broadcast_ether_addr(match.mask->src)) {
2836                                 field_flags |= IAVF_CLOUD_FIELD_IMAC;
2837                         } else {
2838                                 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
2839                                         match.mask->src);
2840                                 return IAVF_ERR_CONFIG;
2841                         }
2842                 }
2843
2844                 if (!is_zero_ether_addr(match.key->dst))
2845                         if (is_valid_ether_addr(match.key->dst) ||
2846                             is_multicast_ether_addr(match.key->dst)) {
2847                                 /* set the mask if a valid dst_mac address */
2848                                 for (i = 0; i < ETH_ALEN; i++)
2849                                         vf->mask.tcp_spec.dst_mac[i] |= 0xff;
2850                                 ether_addr_copy(vf->data.tcp_spec.dst_mac,
2851                                                 match.key->dst);
2852                         }
2853
2854                 if (!is_zero_ether_addr(match.key->src))
2855                         if (is_valid_ether_addr(match.key->src) ||
2856                             is_multicast_ether_addr(match.key->src)) {
2857                                 /* set the mask if a valid dst_mac address */
2858                                 for (i = 0; i < ETH_ALEN; i++)
2859                                         vf->mask.tcp_spec.src_mac[i] |= 0xff;
2860                                 ether_addr_copy(vf->data.tcp_spec.src_mac,
2861                                                 match.key->src);
2862                 }
2863         }
2864
2865         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
2866                 struct flow_match_vlan match;
2867
2868                 flow_rule_match_vlan(rule, &match);
2869                 if (match.mask->vlan_id) {
2870                         if (match.mask->vlan_id == VLAN_VID_MASK) {
2871                                 field_flags |= IAVF_CLOUD_FIELD_IVLAN;
2872                         } else {
2873                                 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
2874                                         match.mask->vlan_id);
2875                                 return IAVF_ERR_CONFIG;
2876                         }
2877                 }
2878                 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
2879                 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id);
2880         }
2881
2882         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
2883                 struct flow_match_control match;
2884
2885                 flow_rule_match_control(rule, &match);
2886                 addr_type = match.key->addr_type;
2887         }
2888
2889         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2890                 struct flow_match_ipv4_addrs match;
2891
2892                 flow_rule_match_ipv4_addrs(rule, &match);
2893                 if (match.mask->dst) {
2894                         if (match.mask->dst == cpu_to_be32(0xffffffff)) {
2895                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2896                         } else {
2897                                 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
2898                                         be32_to_cpu(match.mask->dst));
2899                                 return IAVF_ERR_CONFIG;
2900                         }
2901                 }
2902
2903                 if (match.mask->src) {
2904                         if (match.mask->src == cpu_to_be32(0xffffffff)) {
2905                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2906                         } else {
2907                                 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
2908                                         be32_to_cpu(match.mask->dst));
2909                                 return IAVF_ERR_CONFIG;
2910                         }
2911                 }
2912
2913                 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {
2914                         dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
2915                         return IAVF_ERR_CONFIG;
2916                 }
2917                 if (match.key->dst) {
2918                         vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
2919                         vf->data.tcp_spec.dst_ip[0] = match.key->dst;
2920                 }
2921                 if (match.key->src) {
2922                         vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
2923                         vf->data.tcp_spec.src_ip[0] = match.key->src;
2924                 }
2925         }
2926
2927         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2928                 struct flow_match_ipv6_addrs match;
2929
2930                 flow_rule_match_ipv6_addrs(rule, &match);
2931
2932                 /* validate mask, make sure it is not IPV6_ADDR_ANY */
2933                 if (ipv6_addr_any(&match.mask->dst)) {
2934                         dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
2935                                 IPV6_ADDR_ANY);
2936                         return IAVF_ERR_CONFIG;
2937                 }
2938
2939                 /* src and dest IPv6 address should not be LOOPBACK
2940                  * (0:0:0:0:0:0:0:1) which can be represented as ::1
2941                  */
2942                 if (ipv6_addr_loopback(&match.key->dst) ||
2943                     ipv6_addr_loopback(&match.key->src)) {
2944                         dev_err(&adapter->pdev->dev,
2945                                 "ipv6 addr should not be loopback\n");
2946                         return IAVF_ERR_CONFIG;
2947                 }
2948                 if (!ipv6_addr_any(&match.mask->dst) ||
2949                     !ipv6_addr_any(&match.mask->src))
2950                         field_flags |= IAVF_CLOUD_FIELD_IIP;
2951
2952                 for (i = 0; i < 4; i++)
2953                         vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
2954                 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32,
2955                        sizeof(vf->data.tcp_spec.dst_ip));
2956                 for (i = 0; i < 4; i++)
2957                         vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
2958                 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32,
2959                        sizeof(vf->data.tcp_spec.src_ip));
2960         }
2961         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
2962                 struct flow_match_ports match;
2963
2964                 flow_rule_match_ports(rule, &match);
2965                 if (match.mask->src) {
2966                         if (match.mask->src == cpu_to_be16(0xffff)) {
2967                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2968                         } else {
2969                                 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
2970                                         be16_to_cpu(match.mask->src));
2971                                 return IAVF_ERR_CONFIG;
2972                         }
2973                 }
2974
2975                 if (match.mask->dst) {
2976                         if (match.mask->dst == cpu_to_be16(0xffff)) {
2977                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
2978                         } else {
2979                                 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
2980                                         be16_to_cpu(match.mask->dst));
2981                                 return IAVF_ERR_CONFIG;
2982                         }
2983                 }
2984                 if (match.key->dst) {
2985                         vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
2986                         vf->data.tcp_spec.dst_port = match.key->dst;
2987                 }
2988
2989                 if (match.key->src) {
2990                         vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
2991                         vf->data.tcp_spec.src_port = match.key->src;
2992                 }
2993         }
2994         vf->field_flags = field_flags;
2995
2996         return 0;
2997 }
2998
2999 /**
3000  * iavf_handle_tclass - Forward to a traffic class on the device
3001  * @adapter: board private structure
3002  * @tc: traffic class index on the device
3003  * @filter: pointer to cloud filter structure
3004  */
3005 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,
3006                               struct iavf_cloud_filter *filter)
3007 {
3008         if (tc == 0)
3009                 return 0;
3010         if (tc < adapter->num_tc) {
3011                 if (!filter->f.data.tcp_spec.dst_port) {
3012                         dev_err(&adapter->pdev->dev,
3013                                 "Specify destination port to redirect to traffic class other than TC0\n");
3014                         return -EINVAL;
3015                 }
3016         }
3017         /* redirect to a traffic class on the same device */
3018         filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
3019         filter->f.action_meta = tc;
3020         return 0;
3021 }
3022
3023 /**
3024  * iavf_configure_clsflower - Add tc flower filters
3025  * @adapter: board private structure
3026  * @cls_flower: Pointer to struct flow_cls_offload
3027  */
3028 static int iavf_configure_clsflower(struct iavf_adapter *adapter,
3029                                     struct flow_cls_offload *cls_flower)
3030 {
3031         int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
3032         struct iavf_cloud_filter *filter = NULL;
3033         int err = -EINVAL, count = 50;
3034
3035         if (tc < 0) {
3036                 dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
3037                 return -EINVAL;
3038         }
3039
3040         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
3041         if (!filter)
3042                 return -ENOMEM;
3043
3044         while (!mutex_trylock(&adapter->crit_lock)) {
3045                 if (--count == 0)
3046                         goto err;
3047                 udelay(1);
3048         }
3049
3050         filter->cookie = cls_flower->cookie;
3051
3052         /* set the mask to all zeroes to begin with */
3053         memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
3054         /* start out with flow type and eth type IPv4 to begin with */
3055         filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
3056         err = iavf_parse_cls_flower(adapter, cls_flower, filter);
3057         if (err < 0)
3058                 goto err;
3059
3060         err = iavf_handle_tclass(adapter, tc, filter);
3061         if (err < 0)
3062                 goto err;
3063
3064         /* add filter to the list */
3065         spin_lock_bh(&adapter->cloud_filter_list_lock);
3066         list_add_tail(&filter->list, &adapter->cloud_filter_list);
3067         adapter->num_cloud_filters++;
3068         filter->add = true;
3069         adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
3070         spin_unlock_bh(&adapter->cloud_filter_list_lock);
3071 err:
3072         if (err)
3073                 kfree(filter);
3074
3075         mutex_unlock(&adapter->crit_lock);
3076         return err;
3077 }
3078
3079 /* iavf_find_cf - Find the cloud filter in the list
3080  * @adapter: Board private structure
3081  * @cookie: filter specific cookie
3082  *
3083  * Returns ptr to the filter object or NULL. Must be called while holding the
3084  * cloud_filter_list_lock.
3085  */
3086 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,
3087                                               unsigned long *cookie)
3088 {
3089         struct iavf_cloud_filter *filter = NULL;
3090
3091         if (!cookie)
3092                 return NULL;
3093
3094         list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
3095                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
3096                         return filter;
3097         }
3098         return NULL;
3099 }
3100
3101 /**
3102  * iavf_delete_clsflower - Remove tc flower filters
3103  * @adapter: board private structure
3104  * @cls_flower: Pointer to struct flow_cls_offload
3105  */
3106 static int iavf_delete_clsflower(struct iavf_adapter *adapter,
3107                                  struct flow_cls_offload *cls_flower)
3108 {
3109         struct iavf_cloud_filter *filter = NULL;
3110         int err = 0;
3111
3112         spin_lock_bh(&adapter->cloud_filter_list_lock);
3113         filter = iavf_find_cf(adapter, &cls_flower->cookie);
3114         if (filter) {
3115                 filter->del = true;
3116                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
3117         } else {
3118                 err = -EINVAL;
3119         }
3120         spin_unlock_bh(&adapter->cloud_filter_list_lock);
3121
3122         return err;
3123 }
3124
3125 /**
3126  * iavf_setup_tc_cls_flower - flower classifier offloads
3127  * @adapter: board private structure
3128  * @cls_flower: pointer to flow_cls_offload struct with flow info
3129  */
3130 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,
3131                                     struct flow_cls_offload *cls_flower)
3132 {
3133         switch (cls_flower->command) {
3134         case FLOW_CLS_REPLACE:
3135                 return iavf_configure_clsflower(adapter, cls_flower);
3136         case FLOW_CLS_DESTROY:
3137                 return iavf_delete_clsflower(adapter, cls_flower);
3138         case FLOW_CLS_STATS:
3139                 return -EOPNOTSUPP;
3140         default:
3141                 return -EOPNOTSUPP;
3142         }
3143 }
3144
3145 /**
3146  * iavf_setup_tc_block_cb - block callback for tc
3147  * @type: type of offload
3148  * @type_data: offload data
3149  * @cb_priv:
3150  *
3151  * This function is the block callback for traffic classes
3152  **/
3153 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
3154                                   void *cb_priv)
3155 {
3156         struct iavf_adapter *adapter = cb_priv;
3157
3158         if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
3159                 return -EOPNOTSUPP;
3160
3161         switch (type) {
3162         case TC_SETUP_CLSFLOWER:
3163                 return iavf_setup_tc_cls_flower(cb_priv, type_data);
3164         default:
3165                 return -EOPNOTSUPP;
3166         }
3167 }
3168
3169 static LIST_HEAD(iavf_block_cb_list);
3170
3171 /**
3172  * iavf_setup_tc - configure multiple traffic classes
3173  * @netdev: network interface device structure
3174  * @type: type of offload
3175  * @type_data: tc offload data
3176  *
3177  * This function is the callback to ndo_setup_tc in the
3178  * netdev_ops.
3179  *
3180  * Returns 0 on success
3181  **/
3182 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
3183                          void *type_data)
3184 {
3185         struct iavf_adapter *adapter = netdev_priv(netdev);
3186
3187         switch (type) {
3188         case TC_SETUP_QDISC_MQPRIO:
3189                 return __iavf_setup_tc(netdev, type_data);
3190         case TC_SETUP_BLOCK:
3191                 return flow_block_cb_setup_simple(type_data,
3192                                                   &iavf_block_cb_list,
3193                                                   iavf_setup_tc_block_cb,
3194                                                   adapter, adapter, true);
3195         default:
3196                 return -EOPNOTSUPP;
3197         }
3198 }
3199
3200 /**
3201  * iavf_open - Called when a network interface is made active
3202  * @netdev: network interface device structure
3203  *
3204  * Returns 0 on success, negative value on failure
3205  *
3206  * The open entry point is called when a network interface is made
3207  * active by the system (IFF_UP).  At this point all resources needed
3208  * for transmit and receive operations are allocated, the interrupt
3209  * handler is registered with the OS, the watchdog is started,
3210  * and the stack is notified that the interface is ready.
3211  **/
3212 static int iavf_open(struct net_device *netdev)
3213 {
3214         struct iavf_adapter *adapter = netdev_priv(netdev);
3215         int err;
3216
3217         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
3218                 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
3219                 return -EIO;
3220         }
3221
3222         while (!mutex_trylock(&adapter->crit_lock))
3223                 usleep_range(500, 1000);
3224
3225         if (adapter->state != __IAVF_DOWN) {
3226                 err = -EBUSY;
3227                 goto err_unlock;
3228         }
3229
3230         /* allocate transmit descriptors */
3231         err = iavf_setup_all_tx_resources(adapter);
3232         if (err)
3233                 goto err_setup_tx;
3234
3235         /* allocate receive descriptors */
3236         err = iavf_setup_all_rx_resources(adapter);
3237         if (err)
3238                 goto err_setup_rx;
3239
3240         /* clear any pending interrupts, may auto mask */
3241         err = iavf_request_traffic_irqs(adapter, netdev->name);
3242         if (err)
3243                 goto err_req_irq;
3244
3245         spin_lock_bh(&adapter->mac_vlan_list_lock);
3246
3247         iavf_add_filter(adapter, adapter->hw.mac.addr);
3248
3249         spin_unlock_bh(&adapter->mac_vlan_list_lock);
3250
3251         iavf_configure(adapter);
3252
3253         iavf_up_complete(adapter);
3254
3255         iavf_irq_enable(adapter, true);
3256
3257         mutex_unlock(&adapter->crit_lock);
3258
3259         return 0;
3260
3261 err_req_irq:
3262         iavf_down(adapter);
3263         iavf_free_traffic_irqs(adapter);
3264 err_setup_rx:
3265         iavf_free_all_rx_resources(adapter);
3266 err_setup_tx:
3267         iavf_free_all_tx_resources(adapter);
3268 err_unlock:
3269         mutex_unlock(&adapter->crit_lock);
3270
3271         return err;
3272 }
3273
3274 /**
3275  * iavf_close - Disables a network interface
3276  * @netdev: network interface device structure
3277  *
3278  * Returns 0, this is not allowed to fail
3279  *
3280  * The close entry point is called when an interface is de-activated
3281  * by the OS.  The hardware is still under the drivers control, but
3282  * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
3283  * are freed, along with all transmit and receive resources.
3284  **/
3285 static int iavf_close(struct net_device *netdev)
3286 {
3287         struct iavf_adapter *adapter = netdev_priv(netdev);
3288         int status;
3289
3290         if (adapter->state <= __IAVF_DOWN_PENDING)
3291                 return 0;
3292
3293         while (!mutex_trylock(&adapter->crit_lock))
3294                 usleep_range(500, 1000);
3295
3296         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
3297         if (CLIENT_ENABLED(adapter))
3298                 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE;
3299
3300         iavf_down(adapter);
3301         adapter->state = __IAVF_DOWN_PENDING;
3302         iavf_free_traffic_irqs(adapter);
3303
3304         mutex_unlock(&adapter->crit_lock);
3305
3306         /* We explicitly don't free resources here because the hardware is
3307          * still active and can DMA into memory. Resources are cleared in
3308          * iavf_virtchnl_completion() after we get confirmation from the PF
3309          * driver that the rings have been stopped.
3310          *
3311          * Also, we wait for state to transition to __IAVF_DOWN before
3312          * returning. State change occurs in iavf_virtchnl_completion() after
3313          * VF resources are released (which occurs after PF driver processes and
3314          * responds to admin queue commands).
3315          */
3316
3317         status = wait_event_timeout(adapter->down_waitqueue,
3318                                     adapter->state == __IAVF_DOWN,
3319                                     msecs_to_jiffies(500));
3320         if (!status)
3321                 netdev_warn(netdev, "Device resources not yet released\n");
3322         return 0;
3323 }
3324
3325 /**
3326  * iavf_change_mtu - Change the Maximum Transfer Unit
3327  * @netdev: network interface device structure
3328  * @new_mtu: new value for maximum frame size
3329  *
3330  * Returns 0 on success, negative on failure
3331  **/
3332 static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
3333 {
3334         struct iavf_adapter *adapter = netdev_priv(netdev);
3335
3336         netdev->mtu = new_mtu;
3337         if (CLIENT_ENABLED(adapter)) {
3338                 iavf_notify_client_l2_params(&adapter->vsi);
3339                 adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
3340         }
3341         adapter->flags |= IAVF_FLAG_RESET_NEEDED;
3342         queue_work(iavf_wq, &adapter->reset_task);
3343
3344         return 0;
3345 }
3346
3347 /**
3348  * iavf_set_features - set the netdev feature flags
3349  * @netdev: ptr to the netdev being adjusted
3350  * @features: the feature set that the stack is suggesting
3351  * Note: expects to be called while under rtnl_lock()
3352  **/
3353 static int iavf_set_features(struct net_device *netdev,
3354                              netdev_features_t features)
3355 {
3356         struct iavf_adapter *adapter = netdev_priv(netdev);
3357
3358         /* Don't allow changing VLAN_RX flag when adapter is not capable
3359          * of VLAN offload
3360          */
3361         if (!VLAN_ALLOWED(adapter)) {
3362                 if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX)
3363                         return -EINVAL;
3364         } else if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) {
3365                 if (features & NETIF_F_HW_VLAN_CTAG_RX)
3366                         adapter->aq_required |=
3367                                 IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
3368                 else
3369                         adapter->aq_required |=
3370                                 IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
3371         }
3372
3373         return 0;
3374 }
3375
3376 /**
3377  * iavf_features_check - Validate encapsulated packet conforms to limits
3378  * @skb: skb buff
3379  * @dev: This physical port's netdev
3380  * @features: Offload features that the stack believes apply
3381  **/
3382 static netdev_features_t iavf_features_check(struct sk_buff *skb,
3383                                              struct net_device *dev,
3384                                              netdev_features_t features)
3385 {
3386         size_t len;
3387
3388         /* No point in doing any of this if neither checksum nor GSO are
3389          * being requested for this frame.  We can rule out both by just
3390          * checking for CHECKSUM_PARTIAL
3391          */
3392         if (skb->ip_summed != CHECKSUM_PARTIAL)
3393                 return features;
3394
3395         /* We cannot support GSO if the MSS is going to be less than
3396          * 64 bytes.  If it is then we need to drop support for GSO.
3397          */
3398         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
3399                 features &= ~NETIF_F_GSO_MASK;
3400
3401         /* MACLEN can support at most 63 words */
3402         len = skb_network_header(skb) - skb->data;
3403         if (len & ~(63 * 2))
3404                 goto out_err;
3405
3406         /* IPLEN and EIPLEN can support at most 127 dwords */
3407         len = skb_transport_header(skb) - skb_network_header(skb);
3408         if (len & ~(127 * 4))
3409                 goto out_err;
3410
3411         if (skb->encapsulation) {
3412                 /* L4TUNLEN can support 127 words */
3413                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
3414                 if (len & ~(127 * 2))
3415                         goto out_err;
3416
3417                 /* IPLEN can support at most 127 dwords */
3418                 len = skb_inner_transport_header(skb) -
3419                       skb_inner_network_header(skb);
3420                 if (len & ~(127 * 4))
3421                         goto out_err;
3422         }
3423
3424         /* No need to validate L4LEN as TCP is the only protocol with a
3425          * a flexible value and we support all possible values supported
3426          * by TCP, which is at most 15 dwords
3427          */
3428
3429         return features;
3430 out_err:
3431         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3432 }
3433
3434 /**
3435  * iavf_fix_features - fix up the netdev feature bits
3436  * @netdev: our net device
3437  * @features: desired feature bits
3438  *
3439  * Returns fixed-up features bits
3440  **/
3441 static netdev_features_t iavf_fix_features(struct net_device *netdev,
3442                                            netdev_features_t features)
3443 {
3444         struct iavf_adapter *adapter = netdev_priv(netdev);
3445
3446         if (!(adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN))
3447                 features &= ~(NETIF_F_HW_VLAN_CTAG_TX |
3448                               NETIF_F_HW_VLAN_CTAG_RX |
3449                               NETIF_F_HW_VLAN_CTAG_FILTER);
3450
3451         return features;
3452 }
3453
3454 static const struct net_device_ops iavf_netdev_ops = {
3455         .ndo_open               = iavf_open,
3456         .ndo_stop               = iavf_close,
3457         .ndo_start_xmit         = iavf_xmit_frame,
3458         .ndo_set_rx_mode        = iavf_set_rx_mode,
3459         .ndo_validate_addr      = eth_validate_addr,
3460         .ndo_set_mac_address    = iavf_set_mac,
3461         .ndo_change_mtu         = iavf_change_mtu,
3462         .ndo_tx_timeout         = iavf_tx_timeout,
3463         .ndo_vlan_rx_add_vid    = iavf_vlan_rx_add_vid,
3464         .ndo_vlan_rx_kill_vid   = iavf_vlan_rx_kill_vid,
3465         .ndo_features_check     = iavf_features_check,
3466         .ndo_fix_features       = iavf_fix_features,
3467         .ndo_set_features       = iavf_set_features,
3468         .ndo_setup_tc           = iavf_setup_tc,
3469 };
3470
3471 /**
3472  * iavf_check_reset_complete - check that VF reset is complete
3473  * @hw: pointer to hw struct
3474  *
3475  * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
3476  **/
3477 static int iavf_check_reset_complete(struct iavf_hw *hw)
3478 {
3479         u32 rstat;
3480         int i;
3481
3482         for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
3483                 rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
3484                              IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
3485                 if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
3486                     (rstat == VIRTCHNL_VFR_COMPLETED))
3487                         return 0;
3488                 usleep_range(10, 20);
3489         }
3490         return -EBUSY;
3491 }
3492
3493 /**
3494  * iavf_process_config - Process the config information we got from the PF
3495  * @adapter: board private structure
3496  *
3497  * Verify that we have a valid config struct, and set up our netdev features
3498  * and our VSI struct.
3499  **/
3500 int iavf_process_config(struct iavf_adapter *adapter)
3501 {
3502         struct virtchnl_vf_resource *vfres = adapter->vf_res;
3503         int i, num_req_queues = adapter->num_req_queues;
3504         struct net_device *netdev = adapter->netdev;
3505         struct iavf_vsi *vsi = &adapter->vsi;
3506         netdev_features_t hw_enc_features;
3507         netdev_features_t hw_features;
3508
3509         /* got VF config message back from PF, now we can parse it */
3510         for (i = 0; i < vfres->num_vsis; i++) {
3511                 if (vfres->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
3512                         adapter->vsi_res = &vfres->vsi_res[i];
3513         }
3514         if (!adapter->vsi_res) {
3515                 dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
3516                 return -ENODEV;
3517         }
3518
3519         if (num_req_queues &&
3520             num_req_queues > adapter->vsi_res->num_queue_pairs) {
3521                 /* Problem.  The PF gave us fewer queues than what we had
3522                  * negotiated in our request.  Need a reset to see if we can't
3523                  * get back to a working state.
3524                  */
3525                 dev_err(&adapter->pdev->dev,
3526                         "Requested %d queues, but PF only gave us %d.\n",
3527                         num_req_queues,
3528                         adapter->vsi_res->num_queue_pairs);
3529                 adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED;
3530                 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
3531                 iavf_schedule_reset(adapter);
3532                 return -ENODEV;
3533         }
3534         adapter->num_req_queues = 0;
3535
3536         hw_enc_features = NETIF_F_SG                    |
3537                           NETIF_F_IP_CSUM               |
3538                           NETIF_F_IPV6_CSUM             |
3539                           NETIF_F_HIGHDMA               |
3540                           NETIF_F_SOFT_FEATURES |
3541                           NETIF_F_TSO                   |
3542                           NETIF_F_TSO_ECN               |
3543                           NETIF_F_TSO6                  |
3544                           NETIF_F_SCTP_CRC              |
3545                           NETIF_F_RXHASH                |
3546                           NETIF_F_RXCSUM                |
3547                           0;
3548
3549         /* advertise to stack only if offloads for encapsulated packets is
3550          * supported
3551          */
3552         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
3553                 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL       |
3554                                    NETIF_F_GSO_GRE              |
3555                                    NETIF_F_GSO_GRE_CSUM         |
3556                                    NETIF_F_GSO_IPXIP4           |
3557                                    NETIF_F_GSO_IPXIP6           |
3558                                    NETIF_F_GSO_UDP_TUNNEL_CSUM  |
3559                                    NETIF_F_GSO_PARTIAL          |
3560                                    0;
3561
3562                 if (!(vfres->vf_cap_flags &
3563                       VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
3564                         netdev->gso_partial_features |=
3565                                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
3566
3567                 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
3568                 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
3569                 netdev->hw_enc_features |= hw_enc_features;
3570         }
3571         /* record features VLANs can make use of */
3572         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
3573
3574         /* Write features and hw_features separately to avoid polluting
3575          * with, or dropping, features that are set when we registered.
3576          */
3577         hw_features = hw_enc_features;
3578
3579         /* Enable VLAN features if supported */
3580         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
3581                 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
3582                                 NETIF_F_HW_VLAN_CTAG_RX);
3583         /* Enable cloud filter if ADQ is supported */
3584         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)
3585                 hw_features |= NETIF_F_HW_TC;
3586         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO)
3587                 hw_features |= NETIF_F_GSO_UDP_L4;
3588
3589         netdev->hw_features |= hw_features;
3590
3591         netdev->features |= hw_features;
3592
3593         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
3594                 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
3595
3596         netdev->priv_flags |= IFF_UNICAST_FLT;
3597
3598         /* Do not turn on offloads when they are requested to be turned off.
3599          * TSO needs minimum 576 bytes to work correctly.
3600          */
3601         if (netdev->wanted_features) {
3602                 if (!(netdev->wanted_features & NETIF_F_TSO) ||
3603                     netdev->mtu < 576)
3604                         netdev->features &= ~NETIF_F_TSO;
3605                 if (!(netdev->wanted_features & NETIF_F_TSO6) ||
3606                     netdev->mtu < 576)
3607                         netdev->features &= ~NETIF_F_TSO6;
3608                 if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
3609                         netdev->features &= ~NETIF_F_TSO_ECN;
3610                 if (!(netdev->wanted_features & NETIF_F_GRO))
3611                         netdev->features &= ~NETIF_F_GRO;
3612                 if (!(netdev->wanted_features & NETIF_F_GSO))
3613                         netdev->features &= ~NETIF_F_GSO;
3614         }
3615
3616         adapter->vsi.id = adapter->vsi_res->vsi_id;
3617
3618         adapter->vsi.back = adapter;
3619         adapter->vsi.base_vector = 1;
3620         adapter->vsi.work_limit = IAVF_DEFAULT_IRQ_WORK;
3621         vsi->netdev = adapter->netdev;
3622         vsi->qs_handle = adapter->vsi_res->qset_handle;
3623         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
3624                 adapter->rss_key_size = vfres->rss_key_size;
3625                 adapter->rss_lut_size = vfres->rss_lut_size;
3626         } else {
3627                 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;
3628                 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;
3629         }
3630
3631         return 0;
3632 }
3633
3634 /**
3635  * iavf_init_task - worker thread to perform delayed initialization
3636  * @work: pointer to work_struct containing our data
3637  *
3638  * This task completes the work that was begun in probe. Due to the nature
3639  * of VF-PF communications, we may need to wait tens of milliseconds to get
3640  * responses back from the PF. Rather than busy-wait in probe and bog down the
3641  * whole system, we'll do it in a task so we can sleep.
3642  * This task only runs during driver init. Once we've established
3643  * communications with the PF driver and set up our netdev, the watchdog
3644  * takes over.
3645  **/
3646 static void iavf_init_task(struct work_struct *work)
3647 {
3648         struct iavf_adapter *adapter = container_of(work,
3649                                                     struct iavf_adapter,
3650                                                     init_task.work);
3651         struct iavf_hw *hw = &adapter->hw;
3652
3653         if (iavf_lock_timeout(&adapter->crit_lock, 5000)) {
3654                 dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
3655                 return;
3656         }
3657         switch (adapter->state) {
3658         case __IAVF_STARTUP:
3659                 if (iavf_startup(adapter) < 0)
3660                         goto init_failed;
3661                 break;
3662         case __IAVF_INIT_VERSION_CHECK:
3663                 if (iavf_init_version_check(adapter) < 0)
3664                         goto init_failed;
3665                 break;
3666         case __IAVF_INIT_GET_RESOURCES:
3667                 if (iavf_init_get_resources(adapter) < 0)
3668                         goto init_failed;
3669                 goto out;
3670         default:
3671                 goto init_failed;
3672         }
3673
3674         queue_delayed_work(iavf_wq, &adapter->init_task,
3675                            msecs_to_jiffies(30));
3676         goto out;
3677 init_failed:
3678         if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
3679                 dev_err(&adapter->pdev->dev,
3680                         "Failed to communicate with PF; waiting before retry\n");
3681                 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
3682                 iavf_shutdown_adminq(hw);
3683                 adapter->state = __IAVF_STARTUP;
3684                 queue_delayed_work(iavf_wq, &adapter->init_task, HZ * 5);
3685                 goto out;
3686         }
3687         queue_delayed_work(iavf_wq, &adapter->init_task, HZ);
3688 out:
3689         mutex_unlock(&adapter->crit_lock);
3690 }
3691
3692 /**
3693  * iavf_shutdown - Shutdown the device in preparation for a reboot
3694  * @pdev: pci device structure
3695  **/
3696 static void iavf_shutdown(struct pci_dev *pdev)
3697 {
3698         struct net_device *netdev = pci_get_drvdata(pdev);
3699         struct iavf_adapter *adapter = netdev_priv(netdev);
3700
3701         netif_device_detach(netdev);
3702
3703         if (netif_running(netdev))
3704                 iavf_close(netdev);
3705
3706         if (iavf_lock_timeout(&adapter->crit_lock, 5000))
3707                 dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
3708         /* Prevent the watchdog from running. */
3709         adapter->state = __IAVF_REMOVE;
3710         adapter->aq_required = 0;
3711         mutex_unlock(&adapter->crit_lock);
3712
3713 #ifdef CONFIG_PM
3714         pci_save_state(pdev);
3715
3716 #endif
3717         pci_disable_device(pdev);
3718 }
3719
3720 /**
3721  * iavf_probe - Device Initialization Routine
3722  * @pdev: PCI device information struct
3723  * @ent: entry in iavf_pci_tbl
3724  *
3725  * Returns 0 on success, negative on failure
3726  *
3727  * iavf_probe initializes an adapter identified by a pci_dev structure.
3728  * The OS initialization, configuring of the adapter private structure,
3729  * and a hardware reset occur.
3730  **/
3731 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3732 {
3733         struct net_device *netdev;
3734         struct iavf_adapter *adapter = NULL;
3735         struct iavf_hw *hw = NULL;
3736         int err;
3737
3738         err = pci_enable_device(pdev);
3739         if (err)
3740                 return err;
3741
3742         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3743         if (err) {
3744                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3745                 if (err) {
3746                         dev_err(&pdev->dev,
3747                                 "DMA configuration failed: 0x%x\n", err);
3748                         goto err_dma;
3749                 }
3750         }
3751
3752         err = pci_request_regions(pdev, iavf_driver_name);
3753         if (err) {
3754                 dev_err(&pdev->dev,
3755                         "pci_request_regions failed 0x%x\n", err);
3756                 goto err_pci_reg;
3757         }
3758
3759         pci_enable_pcie_error_reporting(pdev);
3760
3761         pci_set_master(pdev);
3762
3763         netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),
3764                                    IAVF_MAX_REQ_QUEUES);
3765         if (!netdev) {
3766                 err = -ENOMEM;
3767                 goto err_alloc_etherdev;
3768         }
3769
3770         SET_NETDEV_DEV(netdev, &pdev->dev);
3771
3772         pci_set_drvdata(pdev, netdev);
3773         adapter = netdev_priv(netdev);
3774
3775         adapter->netdev = netdev;
3776         adapter->pdev = pdev;
3777
3778         hw = &adapter->hw;
3779         hw->back = adapter;
3780
3781         adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
3782         adapter->state = __IAVF_STARTUP;
3783
3784         /* Call save state here because it relies on the adapter struct. */
3785         pci_save_state(pdev);
3786
3787         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3788                               pci_resource_len(pdev, 0));
3789         if (!hw->hw_addr) {
3790                 err = -EIO;
3791                 goto err_ioremap;
3792         }
3793         hw->vendor_id = pdev->vendor;
3794         hw->device_id = pdev->device;
3795         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
3796         hw->subsystem_vendor_id = pdev->subsystem_vendor;
3797         hw->subsystem_device_id = pdev->subsystem_device;
3798         hw->bus.device = PCI_SLOT(pdev->devfn);
3799         hw->bus.func = PCI_FUNC(pdev->devfn);
3800         hw->bus.bus_id = pdev->bus->number;
3801
3802         /* set up the locks for the AQ, do this only once in probe
3803          * and destroy them only once in remove
3804          */
3805         mutex_init(&adapter->crit_lock);
3806         mutex_init(&adapter->client_lock);
3807         mutex_init(&adapter->remove_lock);
3808         mutex_init(&hw->aq.asq_mutex);
3809         mutex_init(&hw->aq.arq_mutex);
3810
3811         spin_lock_init(&adapter->mac_vlan_list_lock);
3812         spin_lock_init(&adapter->cloud_filter_list_lock);
3813         spin_lock_init(&adapter->fdir_fltr_lock);
3814         spin_lock_init(&adapter->adv_rss_lock);
3815
3816         INIT_LIST_HEAD(&adapter->mac_filter_list);
3817         INIT_LIST_HEAD(&adapter->vlan_filter_list);
3818         INIT_LIST_HEAD(&adapter->cloud_filter_list);
3819         INIT_LIST_HEAD(&adapter->fdir_list_head);
3820         INIT_LIST_HEAD(&adapter->adv_rss_list_head);
3821
3822         INIT_WORK(&adapter->reset_task, iavf_reset_task);
3823         INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
3824         INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);
3825         INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task);
3826         INIT_DELAYED_WORK(&adapter->init_task, iavf_init_task);
3827         queue_delayed_work(iavf_wq, &adapter->init_task,
3828                            msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
3829
3830         /* Setup the wait queue for indicating transition to down status */
3831         init_waitqueue_head(&adapter->down_waitqueue);
3832
3833         return 0;
3834
3835 err_ioremap:
3836         free_netdev(netdev);
3837 err_alloc_etherdev:
3838         pci_disable_pcie_error_reporting(pdev);
3839         pci_release_regions(pdev);
3840 err_pci_reg:
3841 err_dma:
3842         pci_disable_device(pdev);
3843         return err;
3844 }
3845
3846 /**
3847  * iavf_suspend - Power management suspend routine
3848  * @dev_d: device info pointer
3849  *
3850  * Called when the system (VM) is entering sleep/suspend.
3851  **/
3852 static int __maybe_unused iavf_suspend(struct device *dev_d)
3853 {
3854         struct net_device *netdev = dev_get_drvdata(dev_d);
3855         struct iavf_adapter *adapter = netdev_priv(netdev);
3856
3857         netif_device_detach(netdev);
3858
3859         while (!mutex_trylock(&adapter->crit_lock))
3860                 usleep_range(500, 1000);
3861
3862         if (netif_running(netdev)) {
3863                 rtnl_lock();
3864                 iavf_down(adapter);
3865                 rtnl_unlock();
3866         }
3867         iavf_free_misc_irq(adapter);
3868         iavf_reset_interrupt_capability(adapter);
3869
3870         mutex_unlock(&adapter->crit_lock);
3871
3872         return 0;
3873 }
3874
3875 /**
3876  * iavf_resume - Power management resume routine
3877  * @dev_d: device info pointer
3878  *
3879  * Called when the system (VM) is resumed from sleep/suspend.
3880  **/
3881 static int __maybe_unused iavf_resume(struct device *dev_d)
3882 {
3883         struct pci_dev *pdev = to_pci_dev(dev_d);
3884         struct net_device *netdev = pci_get_drvdata(pdev);
3885         struct iavf_adapter *adapter = netdev_priv(netdev);
3886         u32 err;
3887
3888         pci_set_master(pdev);
3889
3890         rtnl_lock();
3891         err = iavf_set_interrupt_capability(adapter);
3892         if (err) {
3893                 rtnl_unlock();
3894                 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
3895                 return err;
3896         }
3897         err = iavf_request_misc_irq(adapter);
3898         rtnl_unlock();
3899         if (err) {
3900                 dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
3901                 return err;
3902         }
3903
3904         queue_work(iavf_wq, &adapter->reset_task);
3905
3906         netif_device_attach(netdev);
3907
3908         return err;
3909 }
3910
3911 /**
3912  * iavf_remove - Device Removal Routine
3913  * @pdev: PCI device information struct
3914  *
3915  * iavf_remove is called by the PCI subsystem to alert the driver
3916  * that it should release a PCI device.  The could be caused by a
3917  * Hot-Plug event, or because the driver is going to be removed from
3918  * memory.
3919  **/
3920 static void iavf_remove(struct pci_dev *pdev)
3921 {
3922         struct net_device *netdev = pci_get_drvdata(pdev);
3923         struct iavf_adapter *adapter = netdev_priv(netdev);
3924         struct iavf_fdir_fltr *fdir, *fdirtmp;
3925         struct iavf_vlan_filter *vlf, *vlftmp;
3926         struct iavf_adv_rss *rss, *rsstmp;
3927         struct iavf_mac_filter *f, *ftmp;
3928         struct iavf_cloud_filter *cf, *cftmp;
3929         struct iavf_hw *hw = &adapter->hw;
3930         int err;
3931         /* Indicate we are in remove and not to run reset_task */
3932         mutex_lock(&adapter->remove_lock);
3933         cancel_delayed_work_sync(&adapter->init_task);
3934         cancel_work_sync(&adapter->reset_task);
3935         cancel_delayed_work_sync(&adapter->client_task);
3936         if (adapter->netdev_registered) {
3937                 unregister_netdev(netdev);
3938                 adapter->netdev_registered = false;
3939         }
3940         if (CLIENT_ALLOWED(adapter)) {
3941                 err = iavf_lan_del_device(adapter);
3942                 if (err)
3943                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
3944                                  err);
3945         }
3946
3947         iavf_request_reset(adapter);
3948         msleep(50);
3949         /* If the FW isn't responding, kick it once, but only once. */
3950         if (!iavf_asq_done(hw)) {
3951                 iavf_request_reset(adapter);
3952                 msleep(50);
3953         }
3954         if (iavf_lock_timeout(&adapter->crit_lock, 5000))
3955                 dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
3956
3957         /* Shut down all the garbage mashers on the detention level */
3958         adapter->state = __IAVF_REMOVE;
3959         adapter->aq_required = 0;
3960         adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
3961         iavf_free_all_tx_resources(adapter);
3962         iavf_free_all_rx_resources(adapter);
3963         iavf_misc_irq_disable(adapter);
3964         iavf_free_misc_irq(adapter);
3965         iavf_reset_interrupt_capability(adapter);
3966         iavf_free_q_vectors(adapter);
3967
3968         cancel_delayed_work_sync(&adapter->watchdog_task);
3969
3970         cancel_work_sync(&adapter->adminq_task);
3971
3972         iavf_free_rss(adapter);
3973
3974         if (hw->aq.asq.count)
3975                 iavf_shutdown_adminq(hw);
3976
3977         /* destroy the locks only once, here */
3978         mutex_destroy(&hw->aq.arq_mutex);
3979         mutex_destroy(&hw->aq.asq_mutex);
3980         mutex_destroy(&adapter->client_lock);
3981         mutex_unlock(&adapter->crit_lock);
3982         mutex_destroy(&adapter->crit_lock);
3983         mutex_unlock(&adapter->remove_lock);
3984         mutex_destroy(&adapter->remove_lock);
3985
3986         iounmap(hw->hw_addr);
3987         pci_release_regions(pdev);
3988         iavf_free_queues(adapter);
3989         kfree(adapter->vf_res);
3990         spin_lock_bh(&adapter->mac_vlan_list_lock);
3991         /* If we got removed before an up/down sequence, we've got a filter
3992          * hanging out there that we need to get rid of.
3993          */
3994         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
3995                 list_del(&f->list);
3996                 kfree(f);
3997         }
3998         list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
3999                                  list) {
4000                 list_del(&vlf->list);
4001                 kfree(vlf);
4002         }
4003
4004         spin_unlock_bh(&adapter->mac_vlan_list_lock);
4005
4006         spin_lock_bh(&adapter->cloud_filter_list_lock);
4007         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
4008                 list_del(&cf->list);
4009                 kfree(cf);
4010         }
4011         spin_unlock_bh(&adapter->cloud_filter_list_lock);
4012
4013         spin_lock_bh(&adapter->fdir_fltr_lock);
4014         list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) {
4015                 list_del(&fdir->list);
4016                 kfree(fdir);
4017         }
4018         spin_unlock_bh(&adapter->fdir_fltr_lock);
4019
4020         spin_lock_bh(&adapter->adv_rss_lock);
4021         list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
4022                                  list) {
4023                 list_del(&rss->list);
4024                 kfree(rss);
4025         }
4026         spin_unlock_bh(&adapter->adv_rss_lock);
4027
4028         free_netdev(netdev);
4029
4030         pci_disable_pcie_error_reporting(pdev);
4031
4032         pci_disable_device(pdev);
4033 }
4034
4035 static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume);
4036
4037 static struct pci_driver iavf_driver = {
4038         .name      = iavf_driver_name,
4039         .id_table  = iavf_pci_tbl,
4040         .probe     = iavf_probe,
4041         .remove    = iavf_remove,
4042         .driver.pm = &iavf_pm_ops,
4043         .shutdown  = iavf_shutdown,
4044 };
4045
4046 /**
4047  * iavf_init_module - Driver Registration Routine
4048  *
4049  * iavf_init_module is the first routine called when the driver is
4050  * loaded. All it does is register with the PCI subsystem.
4051  **/
4052 static int __init iavf_init_module(void)
4053 {
4054         int ret;
4055
4056         pr_info("iavf: %s\n", iavf_driver_string);
4057
4058         pr_info("%s\n", iavf_copyright);
4059
4060         iavf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
4061                                   iavf_driver_name);
4062         if (!iavf_wq) {
4063                 pr_err("%s: Failed to create workqueue\n", iavf_driver_name);
4064                 return -ENOMEM;
4065         }
4066         ret = pci_register_driver(&iavf_driver);
4067         return ret;
4068 }
4069
4070 module_init(iavf_init_module);
4071
4072 /**
4073  * iavf_exit_module - Driver Exit Cleanup Routine
4074  *
4075  * iavf_exit_module is called just before the driver is removed
4076  * from memory.
4077  **/
4078 static void __exit iavf_exit_module(void)
4079 {
4080         pci_unregister_driver(&iavf_driver);
4081         destroy_workqueue(iavf_wq);
4082 }
4083
4084 module_exit(iavf_exit_module);
4085
4086 /* iavf_main.c */