drivers/net/ethernet/intel/igc/igc_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c)  2018 Intel Corporation */
   3
   4 #include <linux/module.h>
   5 #include <linux/types.h>
   6 #include <linux/if_vlan.h>
   7 #include <linux/aer.h>
   8 #include <linux/tcp.h>
   9 #include <linux/udp.h>
  10 #include <linux/ip.h>
  11 #include <linux/pm_runtime.h>
  12 #include <net/pkt_sched.h>
  13 #include <linux/bpf_trace.h>
  14 #include <net/xdp_sock_drv.h>
  15 #include <linux/pci.h>
  16
  17 #include <net/ipv6.h>
  18
  19 #include "igc.h"
  20 #include "igc_hw.h"
  21 #include "igc_tsn.h"
  22 #include "igc_xdp.h"
  23
  24 #define DRV_SUMMARY     "Intel(R) 2.5G Ethernet Linux Driver"
  25
  26 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  27
  28 #define IGC_XDP_PASS            0
  29 #define IGC_XDP_CONSUMED        BIT(0)
  30 #define IGC_XDP_TX              BIT(1)
  31 #define IGC_XDP_REDIRECT        BIT(2)
  32
  33 static int debug = -1;
  34
  35 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  36 MODULE_DESCRIPTION(DRV_SUMMARY);
  37 MODULE_LICENSE("GPL v2");
  38 module_param(debug, int, 0);
  39 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  40
  41 char igc_driver_name[] = "igc";
  42 static const char igc_driver_string[] = DRV_SUMMARY;
  43 static const char igc_copyright[] =
  44         "Copyright(c) 2018 Intel Corporation.";
  45
  46 static const struct igc_info *igc_info_tbl[] = {
  47         [board_base] = &igc_base_info,
  48 };
  49
  50 static const struct pci_device_id igc_pci_tbl[] = {
  51         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
  52         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
  53         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base },
  54         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base },
  55         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base },
  56         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base },
  57         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base },
  58         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base },
  59         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), board_base },
  60         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base },
  61         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base },
  62         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base },
  63         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base },
  64         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base },
  65         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base },
  66         { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base },
  67         /* required last entry */
  68         {0, }
  69 };
  70
  71 MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
  72
  73 enum latency_range {
  74         lowest_latency = 0,
  75         low_latency = 1,
  76         bulk_latency = 2,
  77         latency_invalid = 255
  78 };
  79
  80 void igc_reset(struct igc_adapter *adapter)
  81 {
  82         struct net_device *dev = adapter->netdev;
  83         struct igc_hw *hw = &adapter->hw;
  84         struct igc_fc_info *fc = &hw->fc;
  85         u32 pba, hwm;
  86
  87         /* Repartition PBA for greater than 9k MTU if required */
  88         pba = IGC_PBA_34K;
  89
  90         /* flow control settings
  91          * The high water mark must be low enough to fit one full frame
  92          * after transmitting the pause frame.  As such we must have enough
  93          * space to allow for us to complete our current transmit and then
  94          * receive the frame that is in progress from the link partner.
  95          * Set it to:
  96          * - the full Rx FIFO size minus one full Tx plus one full Rx frame
  97          */
  98         hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
  99
 100         fc->high_water = hwm & 0xFFFFFFF0;      /* 16-byte granularity */
 101         fc->low_water = fc->high_water - 16;
 102         fc->pause_time = 0xFFFF;
 103         fc->send_xon = 1;
 104         fc->current_mode = fc->requested_mode;
 105
 106         hw->mac.ops.reset_hw(hw);
 107
 108         if (hw->mac.ops.init_hw(hw))
 109                 netdev_err(dev, "Error on hardware initialization\n");
 110
 111         /* Re-establish EEE setting */
 112         igc_set_eee_i225(hw, true, true, true);
 113
 114         if (!netif_running(adapter->netdev))
 115                 igc_power_down_phy_copper_base(&adapter->hw);
 116
 117         /* Enable HW to recognize an 802.1Q VLAN Ethernet packet */
 118         wr32(IGC_VET, ETH_P_8021Q);
 119
 120         /* Re-enable PTP, where applicable. */
 121         igc_ptp_reset(adapter);
 122
 123         /* Re-enable TSN offloading, where applicable. */
 124         igc_tsn_reset(adapter);
 125
 126         igc_get_phy_info(hw);
 127 }
 128
 129 /**
 130  * igc_power_up_link - Power up the phy link
 131  * @adapter: address of board private structure
 132  */
 133 static void igc_power_up_link(struct igc_adapter *adapter)
 134 {
 135         igc_reset_phy(&adapter->hw);
 136
 137         igc_power_up_phy_copper(&adapter->hw);
 138
 139         igc_setup_link(&adapter->hw);
 140 }
 141
 142 /**
 143  * igc_release_hw_control - release control of the h/w to f/w
 144  * @adapter: address of board private structure
 145  *
 146  * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 147  * For ASF and Pass Through versions of f/w this means that the
 148  * driver is no longer loaded.
 149  */
 150 static void igc_release_hw_control(struct igc_adapter *adapter)
 151 {
 152         struct igc_hw *hw = &adapter->hw;
 153         u32 ctrl_ext;
 154
 155         if (!pci_device_is_present(adapter->pdev))
 156                 return;
 157
 158         /* Let firmware take over control of h/w */
 159         ctrl_ext = rd32(IGC_CTRL_EXT);
 160         wr32(IGC_CTRL_EXT,
 161              ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
 162 }
 163
 164 /**
 165  * igc_get_hw_control - get control of the h/w from f/w
 166  * @adapter: address of board private structure
 167  *
 168  * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 169  * For ASF and Pass Through versions of f/w this means that
 170  * the driver is loaded.
 171  */
 172 static void igc_get_hw_control(struct igc_adapter *adapter)
 173 {
 174         struct igc_hw *hw = &adapter->hw;
 175         u32 ctrl_ext;
 176
 177         /* Let firmware know the driver has taken over */
 178         ctrl_ext = rd32(IGC_CTRL_EXT);
 179         wr32(IGC_CTRL_EXT,
 180              ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
 181 }
 182
 183 static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf)
 184 {
 185         dma_unmap_single(dev, dma_unmap_addr(buf, dma),
 186                          dma_unmap_len(buf, len), DMA_TO_DEVICE);
 187
 188         dma_unmap_len_set(buf, len, 0);
 189 }
 190
 191 /**
 192  * igc_clean_tx_ring - Free Tx Buffers
 193  * @tx_ring: ring to be cleaned
 194  */
 195 static void igc_clean_tx_ring(struct igc_ring *tx_ring)
 196 {
 197         u16 i = tx_ring->next_to_clean;
 198         struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
 199         u32 xsk_frames = 0;
 200
 201         while (i != tx_ring->next_to_use) {
 202                 union igc_adv_tx_desc *eop_desc, *tx_desc;
 203
 204                 switch (tx_buffer->type) {
 205                 case IGC_TX_BUFFER_TYPE_XSK:
 206                         xsk_frames++;
 207                         break;
 208                 case IGC_TX_BUFFER_TYPE_XDP:
 209                         xdp_return_frame(tx_buffer->xdpf);
 210                         igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
 211                         break;
 212                 case IGC_TX_BUFFER_TYPE_SKB:
 213                         dev_kfree_skb_any(tx_buffer->skb);
 214                         igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
 215                         break;
 216                 default:
 217                         netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
 218                         break;
 219                 }
 220
 221                 /* check for eop_desc to determine the end of the packet */
 222                 eop_desc = tx_buffer->next_to_watch;
 223                 tx_desc = IGC_TX_DESC(tx_ring, i);
 224
 225                 /* unmap remaining buffers */
 226                 while (tx_desc != eop_desc) {
 227                         tx_buffer++;
 228                         tx_desc++;
 229                         i++;
 230                         if (unlikely(i == tx_ring->count)) {
 231                                 i = 0;
 232                                 tx_buffer = tx_ring->tx_buffer_info;
 233                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
 234                         }
 235
 236                         /* unmap any remaining paged data */
 237                         if (dma_unmap_len(tx_buffer, len))
 238                                 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
 239                 }
 240
 241                 tx_buffer->next_to_watch = NULL;
 242
 243                 /* move us one more past the eop_desc for start of next pkt */
 244                 tx_buffer++;
 245                 i++;
 246                 if (unlikely(i == tx_ring->count)) {
 247                         i = 0;
 248                         tx_buffer = tx_ring->tx_buffer_info;
 249                 }
 250         }
 251
 252         if (tx_ring->xsk_pool && xsk_frames)
 253                 xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
 254
 255         /* reset BQL for queue */
 256         netdev_tx_reset_queue(txring_txq(tx_ring));
 257
 258         /* reset next_to_use and next_to_clean */
 259         tx_ring->next_to_use = 0;
 260         tx_ring->next_to_clean = 0;
 261 }
 262
 263 /**
 264  * igc_free_tx_resources - Free Tx Resources per Queue
 265  * @tx_ring: Tx descriptor ring for a specific queue
 266  *
 267  * Free all transmit software resources
 268  */
 269 void igc_free_tx_resources(struct igc_ring *tx_ring)
 270 {
 271         igc_clean_tx_ring(tx_ring);
 272
 273         vfree(tx_ring->tx_buffer_info);
 274         tx_ring->tx_buffer_info = NULL;
 275
 276         /* if not set, then don't free */
 277         if (!tx_ring->desc)
 278                 return;
 279
 280         dma_free_coherent(tx_ring->dev, tx_ring->size,
 281                           tx_ring->desc, tx_ring->dma);
 282
 283         tx_ring->desc = NULL;
 284 }
 285
 286 /**
 287  * igc_free_all_tx_resources - Free Tx Resources for All Queues
 288  * @adapter: board private structure
 289  *
 290  * Free all transmit software resources
 291  */
 292 static void igc_free_all_tx_resources(struct igc_adapter *adapter)
 293 {
 294         int i;
 295
 296         for (i = 0; i < adapter->num_tx_queues; i++)
 297                 igc_free_tx_resources(adapter->tx_ring[i]);
 298 }
 299
 300 /**
 301  * igc_clean_all_tx_rings - Free Tx Buffers for all queues
 302  * @adapter: board private structure
 303  */
 304 static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
 305 {
 306         int i;
 307
 308         for (i = 0; i < adapter->num_tx_queues; i++)
 309                 if (adapter->tx_ring[i])
 310                         igc_clean_tx_ring(adapter->tx_ring[i]);
 311 }
 312
 313 /**
 314  * igc_setup_tx_resources - allocate Tx resources (Descriptors)
 315  * @tx_ring: tx descriptor ring (for a specific queue) to setup
 316  *
 317  * Return 0 on success, negative on failure
 318  */
 319 int igc_setup_tx_resources(struct igc_ring *tx_ring)
 320 {
 321         struct net_device *ndev = tx_ring->netdev;
 322         struct device *dev = tx_ring->dev;
 323         int size = 0;
 324
 325         size = sizeof(struct igc_tx_buffer) * tx_ring->count;
 326         tx_ring->tx_buffer_info = vzalloc(size);
 327         if (!tx_ring->tx_buffer_info)
 328                 goto err;
 329
 330         /* round up to nearest 4K */
 331         tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc);
 332         tx_ring->size = ALIGN(tx_ring->size, 4096);
 333
 334         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
 335                                            &tx_ring->dma, GFP_KERNEL);
 336
 337         if (!tx_ring->desc)
 338                 goto err;
 339
 340         tx_ring->next_to_use = 0;
 341         tx_ring->next_to_clean = 0;
 342
 343         return 0;
 344
 345 err:
 346         vfree(tx_ring->tx_buffer_info);
 347         netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n");
 348         return -ENOMEM;
 349 }
 350
 351 /**
 352  * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
 353  * @adapter: board private structure
 354  *
 355  * Return 0 on success, negative on failure
 356  */
 357 static int igc_setup_all_tx_resources(struct igc_adapter *adapter)
 358 {
 359         struct net_device *dev = adapter->netdev;
 360         int i, err = 0;
 361
 362         for (i = 0; i < adapter->num_tx_queues; i++) {
 363                 err = igc_setup_tx_resources(adapter->tx_ring[i]);
 364                 if (err) {
 365                         netdev_err(dev, "Error on Tx queue %u setup\n", i);
 366                         for (i--; i >= 0; i--)
 367                                 igc_free_tx_resources(adapter->tx_ring[i]);
 368                         break;
 369                 }
 370         }
 371
 372         return err;
 373 }
 374
 375 static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring)
 376 {
 377         u16 i = rx_ring->next_to_clean;
 378
 379         dev_kfree_skb(rx_ring->skb);
 380         rx_ring->skb = NULL;
 381
 382         /* Free all the Rx ring sk_buffs */
 383         while (i != rx_ring->next_to_alloc) {
 384                 struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
 385
 386                 /* Invalidate cache lines that may have been written to by
 387                  * device so that we avoid corrupting memory.
 388                  */
 389                 dma_sync_single_range_for_cpu(rx_ring->dev,
 390                                               buffer_info->dma,
 391                                               buffer_info->page_offset,
 392                                               igc_rx_bufsz(rx_ring),
 393                                               DMA_FROM_DEVICE);
 394
 395                 /* free resources associated with mapping */
 396                 dma_unmap_page_attrs(rx_ring->dev,
 397                                      buffer_info->dma,
 398                                      igc_rx_pg_size(rx_ring),
 399                                      DMA_FROM_DEVICE,
 400                                      IGC_RX_DMA_ATTR);
 401                 __page_frag_cache_drain(buffer_info->page,
 402                                         buffer_info->pagecnt_bias);
 403
 404                 i++;
 405                 if (i == rx_ring->count)
 406                         i = 0;
 407         }
 408 }
 409
 410 static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring)
 411 {
 412         struct igc_rx_buffer *bi;
 413         u16 i;
 414
 415         for (i = 0; i < ring->count; i++) {
 416                 bi = &ring->rx_buffer_info[i];
 417                 if (!bi->xdp)
 418                         continue;
 419
 420                 xsk_buff_free(bi->xdp);
 421                 bi->xdp = NULL;
 422         }
 423 }
 424
 425 /**
 426  * igc_clean_rx_ring - Free Rx Buffers per Queue
 427  * @ring: ring to free buffers from
 428  */
 429 static void igc_clean_rx_ring(struct igc_ring *ring)
 430 {
 431         if (ring->xsk_pool)
 432                 igc_clean_rx_ring_xsk_pool(ring);
 433         else
 434                 igc_clean_rx_ring_page_shared(ring);
 435
 436         clear_ring_uses_large_buffer(ring);
 437
 438         ring->next_to_alloc = 0;
 439         ring->next_to_clean = 0;
 440         ring->next_to_use = 0;
 441 }
 442
 443 /**
 444  * igc_clean_all_rx_rings - Free Rx Buffers for all queues
 445  * @adapter: board private structure
 446  */
 447 static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
 448 {
 449         int i;
 450
 451         for (i = 0; i < adapter->num_rx_queues; i++)
 452                 if (adapter->rx_ring[i])
 453                         igc_clean_rx_ring(adapter->rx_ring[i]);
 454 }
 455
 456 /**
 457  * igc_free_rx_resources - Free Rx Resources
 458  * @rx_ring: ring to clean the resources from
 459  *
 460  * Free all receive software resources
 461  */
 462 void igc_free_rx_resources(struct igc_ring *rx_ring)
 463 {
 464         igc_clean_rx_ring(rx_ring);
 465
 466         xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 467
 468         vfree(rx_ring->rx_buffer_info);
 469         rx_ring->rx_buffer_info = NULL;
 470
 471         /* if not set, then don't free */
 472         if (!rx_ring->desc)
 473                 return;
 474
 475         dma_free_coherent(rx_ring->dev, rx_ring->size,
 476                           rx_ring->desc, rx_ring->dma);
 477
 478         rx_ring->desc = NULL;
 479 }
 480
 481 /**
 482  * igc_free_all_rx_resources - Free Rx Resources for All Queues
 483  * @adapter: board private structure
 484  *
 485  * Free all receive software resources
 486  */
 487 static void igc_free_all_rx_resources(struct igc_adapter *adapter)
 488 {
 489         int i;
 490
 491         for (i = 0; i < adapter->num_rx_queues; i++)
 492                 igc_free_rx_resources(adapter->rx_ring[i]);
 493 }
 494
 495 /**
 496  * igc_setup_rx_resources - allocate Rx resources (Descriptors)
 497  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
 498  *
 499  * Returns 0 on success, negative on failure
 500  */
 501 int igc_setup_rx_resources(struct igc_ring *rx_ring)
 502 {
 503         struct net_device *ndev = rx_ring->netdev;
 504         struct device *dev = rx_ring->dev;
 505         u8 index = rx_ring->queue_index;
 506         int size, desc_len, res;
 507
 508         /* XDP RX-queue info */
 509         if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
 510                 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 511         res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index,
 512                                rx_ring->q_vector->napi.napi_id);
 513         if (res < 0) {
 514                 netdev_err(ndev, "Failed to register xdp_rxq index %u\n",
 515                            index);
 516                 return res;
 517         }
 518
 519         size = sizeof(struct igc_rx_buffer) * rx_ring->count;
 520         rx_ring->rx_buffer_info = vzalloc(size);
 521         if (!rx_ring->rx_buffer_info)
 522                 goto err;
 523
 524         desc_len = sizeof(union igc_adv_rx_desc);
 525
 526         /* Round up to nearest 4K */
 527         rx_ring->size = rx_ring->count * desc_len;
 528         rx_ring->size = ALIGN(rx_ring->size, 4096);
 529
 530         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
 531                                            &rx_ring->dma, GFP_KERNEL);
 532
 533         if (!rx_ring->desc)
 534                 goto err;
 535
 536         rx_ring->next_to_alloc = 0;
 537         rx_ring->next_to_clean = 0;
 538         rx_ring->next_to_use = 0;
 539
 540         return 0;
 541
 542 err:
 543         xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 544         vfree(rx_ring->rx_buffer_info);
 545         rx_ring->rx_buffer_info = NULL;
 546         netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n");
 547         return -ENOMEM;
 548 }
 549
 550 /**
 551  * igc_setup_all_rx_resources - wrapper to allocate Rx resources
 552  *                                (Descriptors) for all queues
 553  * @adapter: board private structure
 554  *
 555  * Return 0 on success, negative on failure
 556  */
 557 static int igc_setup_all_rx_resources(struct igc_adapter *adapter)
 558 {
 559         struct net_device *dev = adapter->netdev;
 560         int i, err = 0;
 561
 562         for (i = 0; i < adapter->num_rx_queues; i++) {
 563                 err = igc_setup_rx_resources(adapter->rx_ring[i]);
 564                 if (err) {
 565                         netdev_err(dev, "Error on Rx queue %u setup\n", i);
 566                         for (i--; i >= 0; i--)
 567                                 igc_free_rx_resources(adapter->rx_ring[i]);
 568                         break;
 569                 }
 570         }
 571
 572         return err;
 573 }
 574
 575 static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter,
 576                                               struct igc_ring *ring)
 577 {
 578         if (!igc_xdp_is_enabled(adapter) ||
 579             !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags))
 580                 return NULL;
 581
 582         return xsk_get_pool_from_qid(ring->netdev, ring->queue_index);
 583 }
 584
 585 /**
 586  * igc_configure_rx_ring - Configure a receive ring after Reset
 587  * @adapter: board private structure
 588  * @ring: receive ring to be configured
 589  *
 590  * Configure the Rx unit of the MAC after a reset.
 591  */
 592 static void igc_configure_rx_ring(struct igc_adapter *adapter,
 593                                   struct igc_ring *ring)
 594 {
 595         struct igc_hw *hw = &adapter->hw;
 596         union igc_adv_rx_desc *rx_desc;
 597         int reg_idx = ring->reg_idx;
 598         u32 srrctl = 0, rxdctl = 0;
 599         u64 rdba = ring->dma;
 600         u32 buf_size;
 601
 602         xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
 603         ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
 604         if (ring->xsk_pool) {
 605                 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
 606                                                    MEM_TYPE_XSK_BUFF_POOL,
 607                                                    NULL));
 608                 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
 609         } else {
 610                 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
 611                                                    MEM_TYPE_PAGE_SHARED,
 612                                                    NULL));
 613         }
 614
 615         if (igc_xdp_is_enabled(adapter))
 616                 set_ring_uses_large_buffer(ring);
 617
 618         /* disable the queue */
 619         wr32(IGC_RXDCTL(reg_idx), 0);
 620
 621         /* Set DMA base address registers */
 622         wr32(IGC_RDBAL(reg_idx),
 623              rdba & 0x00000000ffffffffULL);
 624         wr32(IGC_RDBAH(reg_idx), rdba >> 32);
 625         wr32(IGC_RDLEN(reg_idx),
 626              ring->count * sizeof(union igc_adv_rx_desc));
 627
 628         /* initialize head and tail */
 629         ring->tail = adapter->io_addr + IGC_RDT(reg_idx);
 630         wr32(IGC_RDH(reg_idx), 0);
 631         writel(0, ring->tail);
 632
 633         /* reset next-to- use/clean to place SW in sync with hardware */
 634         ring->next_to_clean = 0;
 635         ring->next_to_use = 0;
 636
 637         if (ring->xsk_pool)
 638                 buf_size = xsk_pool_get_rx_frame_size(ring->xsk_pool);
 639         else if (ring_uses_large_buffer(ring))
 640                 buf_size = IGC_RXBUFFER_3072;
 641         else
 642                 buf_size = IGC_RXBUFFER_2048;
 643
 644         srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT;
 645         srrctl |= buf_size >> IGC_SRRCTL_BSIZEPKT_SHIFT;
 646         srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
 647
 648         wr32(IGC_SRRCTL(reg_idx), srrctl);
 649
 650         rxdctl |= IGC_RX_PTHRESH;
 651         rxdctl |= IGC_RX_HTHRESH << 8;
 652         rxdctl |= IGC_RX_WTHRESH << 16;
 653
 654         /* initialize rx_buffer_info */
 655         memset(ring->rx_buffer_info, 0,
 656                sizeof(struct igc_rx_buffer) * ring->count);
 657
 658         /* initialize Rx descriptor 0 */
 659         rx_desc = IGC_RX_DESC(ring, 0);
 660         rx_desc->wb.upper.length = 0;
 661
 662         /* enable receive descriptor fetching */
 663         rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
 664
 665         wr32(IGC_RXDCTL(reg_idx), rxdctl);
 666 }
 667
 668 /**
 669  * igc_configure_rx - Configure receive Unit after Reset
 670  * @adapter: board private structure
 671  *
 672  * Configure the Rx unit of the MAC after a reset.
 673  */
 674 static void igc_configure_rx(struct igc_adapter *adapter)
 675 {
 676         int i;
 677
 678         /* Setup the HW Rx Head and Tail Descriptor Pointers and
 679          * the Base and Length of the Rx Descriptor Ring
 680          */
 681         for (i = 0; i < adapter->num_rx_queues; i++)
 682                 igc_configure_rx_ring(adapter, adapter->rx_ring[i]);
 683 }
 684
 685 /**
 686  * igc_configure_tx_ring - Configure transmit ring after Reset
 687  * @adapter: board private structure
 688  * @ring: tx ring to configure
 689  *
 690  * Configure a transmit ring after a reset.
 691  */
 692 static void igc_configure_tx_ring(struct igc_adapter *adapter,
 693                                   struct igc_ring *ring)
 694 {
 695         struct igc_hw *hw = &adapter->hw;
 696         int reg_idx = ring->reg_idx;
 697         u64 tdba = ring->dma;
 698         u32 txdctl = 0;
 699
 700         ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
 701
 702         /* disable the queue */
 703         wr32(IGC_TXDCTL(reg_idx), 0);
 704         wrfl();
 705         mdelay(10);
 706
 707         wr32(IGC_TDLEN(reg_idx),
 708              ring->count * sizeof(union igc_adv_tx_desc));
 709         wr32(IGC_TDBAL(reg_idx),
 710              tdba & 0x00000000ffffffffULL);
 711         wr32(IGC_TDBAH(reg_idx), tdba >> 32);
 712
 713         ring->tail = adapter->io_addr + IGC_TDT(reg_idx);
 714         wr32(IGC_TDH(reg_idx), 0);
 715         writel(0, ring->tail);
 716
 717         txdctl |= IGC_TX_PTHRESH;
 718         txdctl |= IGC_TX_HTHRESH << 8;
 719         txdctl |= IGC_TX_WTHRESH << 16;
 720
 721         txdctl |= IGC_TXDCTL_QUEUE_ENABLE;
 722         wr32(IGC_TXDCTL(reg_idx), txdctl);
 723 }
 724
 725 /**
 726  * igc_configure_tx - Configure transmit Unit after Reset
 727  * @adapter: board private structure
 728  *
 729  * Configure the Tx unit of the MAC after a reset.
 730  */
 731 static void igc_configure_tx(struct igc_adapter *adapter)
 732 {
 733         int i;
 734
 735         for (i = 0; i < adapter->num_tx_queues; i++)
 736                 igc_configure_tx_ring(adapter, adapter->tx_ring[i]);
 737 }
 738
 739 /**
 740  * igc_setup_mrqc - configure the multiple receive queue control registers
 741  * @adapter: Board private structure
 742  */
 743 static void igc_setup_mrqc(struct igc_adapter *adapter)
 744 {
 745         struct igc_hw *hw = &adapter->hw;
 746         u32 j, num_rx_queues;
 747         u32 mrqc, rxcsum;
 748         u32 rss_key[10];
 749
 750         netdev_rss_key_fill(rss_key, sizeof(rss_key));
 751         for (j = 0; j < 10; j++)
 752                 wr32(IGC_RSSRK(j), rss_key[j]);
 753
 754         num_rx_queues = adapter->rss_queues;
 755
 756         if (adapter->rss_indir_tbl_init != num_rx_queues) {
 757                 for (j = 0; j < IGC_RETA_SIZE; j++)
 758                         adapter->rss_indir_tbl[j] =
 759                         (j * num_rx_queues) / IGC_RETA_SIZE;
 760                 adapter->rss_indir_tbl_init = num_rx_queues;
 761         }
 762         igc_write_rss_indir_tbl(adapter);
 763
 764         /* Disable raw packet checksumming so that RSS hash is placed in
 765          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
 766          * offloads as they are enabled by default
 767          */
 768         rxcsum = rd32(IGC_RXCSUM);
 769         rxcsum |= IGC_RXCSUM_PCSD;
 770
 771         /* Enable Receive Checksum Offload for SCTP */
 772         rxcsum |= IGC_RXCSUM_CRCOFL;
 773
 774         /* Don't need to set TUOFL or IPOFL, they default to 1 */
 775         wr32(IGC_RXCSUM, rxcsum);
 776
 777         /* Generate RSS hash based on packet types, TCP/UDP
 778          * port numbers and/or IPv4/v6 src and dst addresses
 779          */
 780         mrqc = IGC_MRQC_RSS_FIELD_IPV4 |
 781                IGC_MRQC_RSS_FIELD_IPV4_TCP |
 782                IGC_MRQC_RSS_FIELD_IPV6 |
 783                IGC_MRQC_RSS_FIELD_IPV6_TCP |
 784                IGC_MRQC_RSS_FIELD_IPV6_TCP_EX;
 785
 786         if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP)
 787                 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP;
 788         if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP)
 789                 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP;
 790
 791         mrqc |= IGC_MRQC_ENABLE_RSS_MQ;
 792
 793         wr32(IGC_MRQC, mrqc);
 794 }
 795
 796 /**
 797  * igc_setup_rctl - configure the receive control registers
 798  * @adapter: Board private structure
 799  */
 800 static void igc_setup_rctl(struct igc_adapter *adapter)
 801 {
 802         struct igc_hw *hw = &adapter->hw;
 803         u32 rctl;
 804
 805         rctl = rd32(IGC_RCTL);
 806
 807         rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
 808         rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC);
 809
 810         rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF |
 811                 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
 812
 813         /* enable stripping of CRC. Newer features require
 814          * that the HW strips the CRC.
 815          */
 816         rctl |= IGC_RCTL_SECRC;
 817
 818         /* disable store bad packets and clear size bits. */
 819         rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256);
 820
 821         /* enable LPE to allow for reception of jumbo frames */
 822         rctl |= IGC_RCTL_LPE;
 823
 824         /* disable queue 0 to prevent tail write w/o re-config */
 825         wr32(IGC_RXDCTL(0), 0);
 826
 827         /* This is useful for sniffing bad packets. */
 828         if (adapter->netdev->features & NETIF_F_RXALL) {
 829                 /* UPE and MPE will be handled by normal PROMISC logic
 830                  * in set_rx_mode
 831                  */
 832                 rctl |= (IGC_RCTL_SBP | /* Receive bad packets */
 833                          IGC_RCTL_BAM | /* RX All Bcast Pkts */
 834                          IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
 835
 836                 rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */
 837                           IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */
 838         }
 839
 840         wr32(IGC_RCTL, rctl);
 841 }
 842
 843 /**
 844  * igc_setup_tctl - configure the transmit control registers
 845  * @adapter: Board private structure
 846  */
 847 static void igc_setup_tctl(struct igc_adapter *adapter)
 848 {
 849         struct igc_hw *hw = &adapter->hw;
 850         u32 tctl;
 851
 852         /* disable queue 0 which icould be enabled by default */
 853         wr32(IGC_TXDCTL(0), 0);
 854
 855         /* Program the Transmit Control Register */
 856         tctl = rd32(IGC_TCTL);
 857         tctl &= ~IGC_TCTL_CT;
 858         tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC |
 859                 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT);
 860
 861         /* Enable transmits */
 862         tctl |= IGC_TCTL_EN;
 863
 864         wr32(IGC_TCTL, tctl);
 865 }
 866
 867 /**
 868  * igc_set_mac_filter_hw() - Set MAC address filter in hardware
 869  * @adapter: Pointer to adapter where the filter should be set
 870  * @index: Filter index
 871  * @type: MAC address filter type (source or destination)
 872  * @addr: MAC address
 873  * @queue: If non-negative, queue assignment feature is enabled and frames
 874  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
 875  *         assignment is disabled.
 876  */
 877 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index,
 878                                   enum igc_mac_filter_type type,
 879                                   const u8 *addr, int queue)
 880 {
 881         struct net_device *dev = adapter->netdev;
 882         struct igc_hw *hw = &adapter->hw;
 883         u32 ral, rah;
 884
 885         if (WARN_ON(index >= hw->mac.rar_entry_count))
 886                 return;
 887
 888         ral = le32_to_cpup((__le32 *)(addr));
 889         rah = le16_to_cpup((__le16 *)(addr + 4));
 890
 891         if (type == IGC_MAC_FILTER_TYPE_SRC) {
 892                 rah &= ~IGC_RAH_ASEL_MASK;
 893                 rah |= IGC_RAH_ASEL_SRC_ADDR;
 894         }
 895
 896         if (queue >= 0) {
 897                 rah &= ~IGC_RAH_QSEL_MASK;
 898                 rah |= (queue << IGC_RAH_QSEL_SHIFT);
 899                 rah |= IGC_RAH_QSEL_ENABLE;
 900         }
 901
 902         rah |= IGC_RAH_AV;
 903
 904         wr32(IGC_RAL(index), ral);
 905         wr32(IGC_RAH(index), rah);
 906
 907         netdev_dbg(dev, "MAC address filter set in HW: index %d", index);
 908 }
 909
 910 /**
 911  * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware
 912  * @adapter: Pointer to adapter where the filter should be cleared
 913  * @index: Filter index
 914  */
 915 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index)
 916 {
 917         struct net_device *dev = adapter->netdev;
 918         struct igc_hw *hw = &adapter->hw;
 919
 920         if (WARN_ON(index >= hw->mac.rar_entry_count))
 921                 return;
 922
 923         wr32(IGC_RAL(index), 0);
 924         wr32(IGC_RAH(index), 0);
 925
 926         netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index);
 927 }
 928
 929 /* Set default MAC address for the PF in the first RAR entry */
 930 static void igc_set_default_mac_filter(struct igc_adapter *adapter)
 931 {
 932         struct net_device *dev = adapter->netdev;
 933         u8 *addr = adapter->hw.mac.addr;
 934
 935         netdev_dbg(dev, "Set default MAC address filter: address %pM", addr);
 936
 937         igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1);
 938 }
 939
 940 /**
 941  * igc_set_mac - Change the Ethernet Address of the NIC
 942  * @netdev: network interface device structure
 943  * @p: pointer to an address structure
 944  *
 945  * Returns 0 on success, negative on failure
 946  */
 947 static int igc_set_mac(struct net_device *netdev, void *p)
 948 {
 949         struct igc_adapter *adapter = netdev_priv(netdev);
 950         struct igc_hw *hw = &adapter->hw;
 951         struct sockaddr *addr = p;
 952
 953         if (!is_valid_ether_addr(addr->sa_data))
 954                 return -EADDRNOTAVAIL;
 955
 956         eth_hw_addr_set(netdev, addr->sa_data);
 957         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
 958
 959         /* set the correct pool for the new PF MAC address in entry 0 */
 960         igc_set_default_mac_filter(adapter);
 961
 962         return 0;
 963 }
 964
 965 /**
 966  *  igc_write_mc_addr_list - write multicast addresses to MTA
 967  *  @netdev: network interface device structure
 968  *
 969  *  Writes multicast address list to the MTA hash table.
 970  *  Returns: -ENOMEM on failure
 971  *           0 on no addresses written
 972  *           X on writing X addresses to MTA
 973  **/
 974 static int igc_write_mc_addr_list(struct net_device *netdev)
 975 {
 976         struct igc_adapter *adapter = netdev_priv(netdev);
 977         struct igc_hw *hw = &adapter->hw;
 978         struct netdev_hw_addr *ha;
 979         u8  *mta_list;
 980         int i;
 981
 982         if (netdev_mc_empty(netdev)) {
 983                 /* nothing to program, so clear mc list */
 984                 igc_update_mc_addr_list(hw, NULL, 0);
 985                 return 0;
 986         }
 987
 988         mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
 989         if (!mta_list)
 990                 return -ENOMEM;
 991
 992         /* The shared function expects a packed array of only addresses. */
 993         i = 0;
 994         netdev_for_each_mc_addr(ha, netdev)
 995                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
 996
 997         igc_update_mc_addr_list(hw, mta_list, i);
 998         kfree(mta_list);
 999
1000         return netdev_mc_count(netdev);
1001 }
1002
1003 static __le32 igc_tx_launchtime(struct igc_ring *ring, ktime_t txtime,
1004                                 bool *first_flag, bool *insert_empty)
1005 {
1006         struct igc_adapter *adapter = netdev_priv(ring->netdev);
1007         ktime_t cycle_time = adapter->cycle_time;
1008         ktime_t base_time = adapter->base_time;
1009         ktime_t now = ktime_get_clocktai();
1010         ktime_t baset_est, end_of_cycle;
1011         u32 launchtime;
1012         s64 n;
1013
1014         n = div64_s64(ktime_sub_ns(now, base_time), cycle_time);
1015
1016         baset_est = ktime_add_ns(base_time, cycle_time * (n));
1017         end_of_cycle = ktime_add_ns(baset_est, cycle_time);
1018
1019         if (ktime_compare(txtime, end_of_cycle) >= 0) {
1020                 if (baset_est != ring->last_ff_cycle) {
1021                         *first_flag = true;
1022                         ring->last_ff_cycle = baset_est;
1023
1024                         if (ktime_compare(txtime, ring->last_tx_cycle) > 0)
1025                                 *insert_empty = true;
1026                 }
1027         }
1028
1029         /* Introducing a window at end of cycle on which packets
1030          * potentially not honor launchtime. Window of 5us chosen
1031          * considering software update the tail pointer and packets
1032          * are dma'ed to packet buffer.
1033          */
1034         if ((ktime_sub_ns(end_of_cycle, now) < 5 * NSEC_PER_USEC))
1035                 netdev_warn(ring->netdev, "Packet with txtime=%llu may not be honoured\n",
1036                             txtime);
1037
1038         ring->last_tx_cycle = end_of_cycle;
1039
1040         launchtime = ktime_sub_ns(txtime, baset_est);
1041         if (launchtime > 0)
1042                 div_s64_rem(launchtime, cycle_time, &launchtime);
1043         else
1044                 launchtime = 0;
1045
1046         return cpu_to_le32(launchtime);
1047 }
1048
1049 static int igc_init_empty_frame(struct igc_ring *ring,
1050                                 struct igc_tx_buffer *buffer,
1051                                 struct sk_buff *skb)
1052 {
1053         unsigned int size;
1054         dma_addr_t dma;
1055
1056         size = skb_headlen(skb);
1057
1058         dma = dma_map_single(ring->dev, skb->data, size, DMA_TO_DEVICE);
1059         if (dma_mapping_error(ring->dev, dma)) {
1060                 netdev_err_once(ring->netdev, "Failed to map DMA for TX\n");
1061                 return -ENOMEM;
1062         }
1063
1064         buffer->skb = skb;
1065         buffer->protocol = 0;
1066         buffer->bytecount = skb->len;
1067         buffer->gso_segs = 1;
1068         buffer->time_stamp = jiffies;
1069         dma_unmap_len_set(buffer, len, skb->len);
1070         dma_unmap_addr_set(buffer, dma, dma);
1071
1072         return 0;
1073 }
1074
1075 static int igc_init_tx_empty_descriptor(struct igc_ring *ring,
1076                                         struct sk_buff *skb,
1077                                         struct igc_tx_buffer *first)
1078 {
1079         union igc_adv_tx_desc *desc;
1080         u32 cmd_type, olinfo_status;
1081         int err;
1082
1083         if (!igc_desc_unused(ring))
1084                 return -EBUSY;
1085
1086         err = igc_init_empty_frame(ring, first, skb);
1087         if (err)
1088                 return err;
1089
1090         cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
1091                    IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
1092                    first->bytecount;
1093         olinfo_status = first->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
1094
1095         desc = IGC_TX_DESC(ring, ring->next_to_use);
1096         desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1097         desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1098         desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(first, dma));
1099
1100         netdev_tx_sent_queue(txring_txq(ring), skb->len);
1101
1102         first->next_to_watch = desc;
1103
1104         ring->next_to_use++;
1105         if (ring->next_to_use == ring->count)
1106                 ring->next_to_use = 0;
1107
1108         return 0;
1109 }
1110
1111 #define IGC_EMPTY_FRAME_SIZE 60
1112
1113 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring,
1114                             __le32 launch_time, bool first_flag,
1115                             u32 vlan_macip_lens, u32 type_tucmd,
1116                             u32 mss_l4len_idx)
1117 {
1118         struct igc_adv_tx_context_desc *context_desc;
1119         u16 i = tx_ring->next_to_use;
1120
1121         context_desc = IGC_TX_CTXTDESC(tx_ring, i);
1122
1123         i++;
1124         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1125
1126         /* set bits to identify this as an advanced context descriptor */
1127         type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT;
1128
1129         /* For i225, context index must be unique per ring. */
1130         if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
1131                 mss_l4len_idx |= tx_ring->reg_idx << 4;
1132
1133         if (first_flag)
1134                 mss_l4len_idx |= IGC_ADVTXD_TSN_CNTX_FIRST;
1135
1136         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
1137         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
1138         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
1139         context_desc->launch_time       = launch_time;
1140 }
1141
1142 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first,
1143                         __le32 launch_time, bool first_flag)
1144 {
1145         struct sk_buff *skb = first->skb;
1146         u32 vlan_macip_lens = 0;
1147         u32 type_tucmd = 0;
1148
1149         if (skb->ip_summed != CHECKSUM_PARTIAL) {
1150 csum_failed:
1151                 if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) &&
1152                     !tx_ring->launchtime_enable)
1153                         return;
1154                 goto no_csum;
1155         }
1156
1157         switch (skb->csum_offset) {
1158         case offsetof(struct tcphdr, check):
1159                 type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1160                 fallthrough;
1161         case offsetof(struct udphdr, check):
1162                 break;
1163         case offsetof(struct sctphdr, checksum):
1164                 /* validate that this is actually an SCTP request */
1165                 if (skb_csum_is_sctp(skb)) {
1166                         type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP;
1167                         break;
1168                 }
1169                 fallthrough;
1170         default:
1171                 skb_checksum_help(skb);
1172                 goto csum_failed;
1173         }
1174
1175         /* update TX checksum flag */
1176         first->tx_flags |= IGC_TX_FLAGS_CSUM;
1177         vlan_macip_lens = skb_checksum_start_offset(skb) -
1178                           skb_network_offset(skb);
1179 no_csum:
1180         vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT;
1181         vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1182
1183         igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1184                         vlan_macip_lens, type_tucmd, 0);
1185 }
1186
1187 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1188 {
1189         struct net_device *netdev = tx_ring->netdev;
1190
1191         netif_stop_subqueue(netdev, tx_ring->queue_index);
1192
1193         /* memory barriier comment */
1194         smp_mb();
1195
1196         /* We need to check again in a case another CPU has just
1197          * made room available.
1198          */
1199         if (igc_desc_unused(tx_ring) < size)
1200                 return -EBUSY;
1201
1202         /* A reprieve! */
1203         netif_wake_subqueue(netdev, tx_ring->queue_index);
1204
1205         u64_stats_update_begin(&tx_ring->tx_syncp2);
1206         tx_ring->tx_stats.restart_queue2++;
1207         u64_stats_update_end(&tx_ring->tx_syncp2);
1208
1209         return 0;
1210 }
1211
1212 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1213 {
1214         if (igc_desc_unused(tx_ring) >= size)
1215                 return 0;
1216         return __igc_maybe_stop_tx(tx_ring, size);
1217 }
1218
1219 #define IGC_SET_FLAG(_input, _flag, _result) \
1220         (((_flag) <= (_result)) ?                               \
1221          ((u32)((_input) & (_flag)) * ((_result) / (_flag))) :  \
1222          ((u32)((_input) & (_flag)) / ((_flag) / (_result))))
1223
1224 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
1225 {
1226         /* set type for advanced descriptor with frame checksum insertion */
1227         u32 cmd_type = IGC_ADVTXD_DTYP_DATA |
1228                        IGC_ADVTXD_DCMD_DEXT |
1229                        IGC_ADVTXD_DCMD_IFCS;
1230
1231         /* set HW vlan bit if vlan is present */
1232         cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN,
1233                                  IGC_ADVTXD_DCMD_VLE);
1234
1235         /* set segmentation bits for TSO */
1236         cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO,
1237                                  (IGC_ADVTXD_DCMD_TSE));
1238
1239         /* set timestamp bit if present */
1240         cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP,
1241                                  (IGC_ADVTXD_MAC_TSTAMP));
1242
1243         /* insert frame checksum */
1244         cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS);
1245
1246         return cmd_type;
1247 }
1248
1249 static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
1250                                  union igc_adv_tx_desc *tx_desc,
1251                                  u32 tx_flags, unsigned int paylen)
1252 {
1253         u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
1254
1255         /* insert L4 checksum */
1256         olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
1257                           ((IGC_TXD_POPTS_TXSM << 8) /
1258                           IGC_TX_FLAGS_CSUM);
1259
1260         /* insert IPv4 checksum */
1261         olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
1262                           (((IGC_TXD_POPTS_IXSM << 8)) /
1263                           IGC_TX_FLAGS_IPV4);
1264
1265         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1266 }
1267
1268 static int igc_tx_map(struct igc_ring *tx_ring,
1269                       struct igc_tx_buffer *first,
1270                       const u8 hdr_len)
1271 {
1272         struct sk_buff *skb = first->skb;
1273         struct igc_tx_buffer *tx_buffer;
1274         union igc_adv_tx_desc *tx_desc;
1275         u32 tx_flags = first->tx_flags;
1276         skb_frag_t *frag;
1277         u16 i = tx_ring->next_to_use;
1278         unsigned int data_len, size;
1279         dma_addr_t dma;
1280         u32 cmd_type;
1281
1282         cmd_type = igc_tx_cmd_type(skb, tx_flags);
1283         tx_desc = IGC_TX_DESC(tx_ring, i);
1284
1285         igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
1286
1287         size = skb_headlen(skb);
1288         data_len = skb->data_len;
1289
1290         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1291
1292         tx_buffer = first;
1293
1294         for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1295                 if (dma_mapping_error(tx_ring->dev, dma))
1296                         goto dma_error;
1297
1298                 /* record length, and DMA address */
1299                 dma_unmap_len_set(tx_buffer, len, size);
1300                 dma_unmap_addr_set(tx_buffer, dma, dma);
1301
1302                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
1303
1304                 while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
1305                         tx_desc->read.cmd_type_len =
1306                                 cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
1307
1308                         i++;
1309                         tx_desc++;
1310                         if (i == tx_ring->count) {
1311                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
1312                                 i = 0;
1313                         }
1314                         tx_desc->read.olinfo_status = 0;
1315
1316                         dma += IGC_MAX_DATA_PER_TXD;
1317                         size -= IGC_MAX_DATA_PER_TXD;
1318
1319                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
1320                 }
1321
1322                 if (likely(!data_len))
1323                         break;
1324
1325                 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1326
1327                 i++;
1328                 tx_desc++;
1329                 if (i == tx_ring->count) {
1330                         tx_desc = IGC_TX_DESC(tx_ring, 0);
1331                         i = 0;
1332                 }
1333                 tx_desc->read.olinfo_status = 0;
1334
1335                 size = skb_frag_size(frag);
1336                 data_len -= size;
1337
1338                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
1339                                        size, DMA_TO_DEVICE);
1340
1341                 tx_buffer = &tx_ring->tx_buffer_info[i];
1342         }
1343
1344         /* write last descriptor with RS and EOP bits */
1345         cmd_type |= size | IGC_TXD_DCMD;
1346         tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1347
1348         netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
1349
1350         /* set the timestamp */
1351         first->time_stamp = jiffies;
1352
1353         skb_tx_timestamp(skb);
1354
1355         /* Force memory writes to complete before letting h/w know there
1356          * are new descriptors to fetch.  (Only applicable for weak-ordered
1357          * memory model archs, such as IA-64).
1358          *
1359          * We also need this memory barrier to make certain all of the
1360          * status bits have been updated before next_to_watch is written.
1361          */
1362         wmb();
1363
1364         /* set next_to_watch value indicating a packet is present */
1365         first->next_to_watch = tx_desc;
1366
1367         i++;
1368         if (i == tx_ring->count)
1369                 i = 0;
1370
1371         tx_ring->next_to_use = i;
1372
1373         /* Make sure there is space in the ring for the next send. */
1374         igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
1375
1376         if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
1377                 writel(i, tx_ring->tail);
1378         }
1379
1380         return 0;
1381 dma_error:
1382         netdev_err(tx_ring->netdev, "TX DMA map failed\n");
1383         tx_buffer = &tx_ring->tx_buffer_info[i];
1384
1385         /* clear dma mappings for failed tx_buffer_info map */
1386         while (tx_buffer != first) {
1387                 if (dma_unmap_len(tx_buffer, len))
1388                         igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
1389
1390                 if (i-- == 0)
1391                         i += tx_ring->count;
1392                 tx_buffer = &tx_ring->tx_buffer_info[i];
1393         }
1394
1395         if (dma_unmap_len(tx_buffer, len))
1396                 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
1397
1398         dev_kfree_skb_any(tx_buffer->skb);
1399         tx_buffer->skb = NULL;
1400
1401         tx_ring->next_to_use = i;
1402
1403         return -1;
1404 }
1405
1406 static int igc_tso(struct igc_ring *tx_ring,
1407                    struct igc_tx_buffer *first,
1408                    __le32 launch_time, bool first_flag,
1409                    u8 *hdr_len)
1410 {
1411         u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1412         struct sk_buff *skb = first->skb;
1413         union {
1414                 struct iphdr *v4;
1415                 struct ipv6hdr *v6;
1416                 unsigned char *hdr;
1417         } ip;
1418         union {
1419                 struct tcphdr *tcp;
1420                 struct udphdr *udp;
1421                 unsigned char *hdr;
1422         } l4;
1423         u32 paylen, l4_offset;
1424         int err;
1425
1426         if (skb->ip_summed != CHECKSUM_PARTIAL)
1427                 return 0;
1428
1429         if (!skb_is_gso(skb))
1430                 return 0;
1431
1432         err = skb_cow_head(skb, 0);
1433         if (err < 0)
1434                 return err;
1435
1436         ip.hdr = skb_network_header(skb);
1437         l4.hdr = skb_checksum_start(skb);
1438
1439         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
1440         type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1441
1442         /* initialize outer IP header fields */
1443         if (ip.v4->version == 4) {
1444                 unsigned char *csum_start = skb_checksum_start(skb);
1445                 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
1446
1447                 /* IP header will have to cancel out any data that
1448                  * is not a part of the outer IP header
1449                  */
1450                 ip.v4->check = csum_fold(csum_partial(trans_start,
1451                                                       csum_start - trans_start,
1452                                                       0));
1453                 type_tucmd |= IGC_ADVTXD_TUCMD_IPV4;
1454
1455                 ip.v4->tot_len = 0;
1456                 first->tx_flags |= IGC_TX_FLAGS_TSO |
1457                                    IGC_TX_FLAGS_CSUM |
1458                                    IGC_TX_FLAGS_IPV4;
1459         } else {
1460                 ip.v6->payload_len = 0;
1461                 first->tx_flags |= IGC_TX_FLAGS_TSO |
1462                                    IGC_TX_FLAGS_CSUM;
1463         }
1464
1465         /* determine offset of inner transport header */
1466         l4_offset = l4.hdr - skb->data;
1467
1468         /* remove payload length from inner checksum */
1469         paylen = skb->len - l4_offset;
1470         if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) {
1471                 /* compute length of segmentation header */
1472                 *hdr_len = (l4.tcp->doff * 4) + l4_offset;
1473                 csum_replace_by_diff(&l4.tcp->check,
1474                                      (__force __wsum)htonl(paylen));
1475         } else {
1476                 /* compute length of segmentation header */
1477                 *hdr_len = sizeof(*l4.udp) + l4_offset;
1478                 csum_replace_by_diff(&l4.udp->check,
1479                                      (__force __wsum)htonl(paylen));
1480         }
1481
1482         /* update gso size and bytecount with header size */
1483         first->gso_segs = skb_shinfo(skb)->gso_segs;
1484         first->bytecount += (first->gso_segs - 1) * *hdr_len;
1485
1486         /* MSS L4LEN IDX */
1487         mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT;
1488         mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT;
1489
1490         /* VLAN MACLEN IPLEN */
1491         vlan_macip_lens = l4.hdr - ip.hdr;
1492         vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT;
1493         vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1494
1495         igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1496                         vlan_macip_lens, type_tucmd, mss_l4len_idx);
1497
1498         return 1;
1499 }
1500
1501 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
1502                                        struct igc_ring *tx_ring)
1503 {
1504         bool first_flag = false, insert_empty = false;
1505         u16 count = TXD_USE_COUNT(skb_headlen(skb));
1506         __be16 protocol = vlan_get_protocol(skb);
1507         struct igc_tx_buffer *first;
1508         __le32 launch_time = 0;
1509         u32 tx_flags = 0;
1510         unsigned short f;
1511         ktime_t txtime;
1512         u8 hdr_len = 0;
1513         int tso = 0;
1514
1515         /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1516          *      + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1517          *      + 2 desc gap to keep tail from touching head,
1518          *      + 1 desc for context descriptor,
1519          * otherwise try next time
1520          */
1521         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1522                 count += TXD_USE_COUNT(skb_frag_size(
1523                                                 &skb_shinfo(skb)->frags[f]));
1524
1525         if (igc_maybe_stop_tx(tx_ring, count + 5)) {
1526                 /* this is a hard error */
1527                 return NETDEV_TX_BUSY;
1528         }
1529
1530         if (!tx_ring->launchtime_enable)
1531                 goto done;
1532
1533         txtime = skb->tstamp;
1534         skb->tstamp = ktime_set(0, 0);
1535         launch_time = igc_tx_launchtime(tx_ring, txtime, &first_flag, &insert_empty);
1536
1537         if (insert_empty) {
1538                 struct igc_tx_buffer *empty_info;
1539                 struct sk_buff *empty;
1540                 void *data;
1541
1542                 empty_info = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1543                 empty = alloc_skb(IGC_EMPTY_FRAME_SIZE, GFP_ATOMIC);
1544                 if (!empty)
1545                         goto done;
1546
1547                 data = skb_put(empty, IGC_EMPTY_FRAME_SIZE);
1548                 memset(data, 0, IGC_EMPTY_FRAME_SIZE);
1549
1550                 igc_tx_ctxtdesc(tx_ring, 0, false, 0, 0, 0);
1551
1552                 if (igc_init_tx_empty_descriptor(tx_ring,
1553                                                  empty,
1554                                                  empty_info) < 0)
1555                         dev_kfree_skb_any(empty);
1556         }
1557
1558 done:
1559         /* record the location of the first descriptor for this packet */
1560         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1561         first->type = IGC_TX_BUFFER_TYPE_SKB;
1562         first->skb = skb;
1563         first->bytecount = skb->len;
1564         first->gso_segs = 1;
1565
1566         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
1567                 struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
1568
1569                 /* FIXME: add support for retrieving timestamps from
1570                  * the other timer registers before skipping the
1571                  * timestamping request.
1572                  */
1573                 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
1574                     !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS,
1575                                            &adapter->state)) {
1576                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1577                         tx_flags |= IGC_TX_FLAGS_TSTAMP;
1578
1579                         adapter->ptp_tx_skb = skb_get(skb);
1580                         adapter->ptp_tx_start = jiffies;
1581                 } else {
1582                         adapter->tx_hwtstamp_skipped++;
1583                 }
1584         }
1585
1586         if (skb_vlan_tag_present(skb)) {
1587                 tx_flags |= IGC_TX_FLAGS_VLAN;
1588                 tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT);
1589         }
1590
1591         /* record initial flags and protocol */
1592         first->tx_flags = tx_flags;
1593         first->protocol = protocol;
1594
1595         tso = igc_tso(tx_ring, first, launch_time, first_flag, &hdr_len);
1596         if (tso < 0)
1597                 goto out_drop;
1598         else if (!tso)
1599                 igc_tx_csum(tx_ring, first, launch_time, first_flag);
1600
1601         igc_tx_map(tx_ring, first, hdr_len);
1602
1603         return NETDEV_TX_OK;
1604
1605 out_drop:
1606         dev_kfree_skb_any(first->skb);
1607         first->skb = NULL;
1608
1609         return NETDEV_TX_OK;
1610 }
1611
1612 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
1613                                                     struct sk_buff *skb)
1614 {
1615         unsigned int r_idx = skb->queue_mapping;
1616
1617         if (r_idx >= adapter->num_tx_queues)
1618                 r_idx = r_idx % adapter->num_tx_queues;
1619
1620         return adapter->tx_ring[r_idx];
1621 }
1622
1623 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
1624                                   struct net_device *netdev)
1625 {
1626         struct igc_adapter *adapter = netdev_priv(netdev);
1627
1628         /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1629          * in order to meet this minimum size requirement.
1630          */
1631         if (skb->len < 17) {
1632                 if (skb_padto(skb, 17))
1633                         return NETDEV_TX_OK;
1634                 skb->len = 17;
1635         }
1636
1637         return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
1638 }
1639
1640 static void igc_rx_checksum(struct igc_ring *ring,
1641                             union igc_adv_rx_desc *rx_desc,
1642                             struct sk_buff *skb)
1643 {
1644         skb_checksum_none_assert(skb);
1645
1646         /* Ignore Checksum bit is set */
1647         if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM))
1648                 return;
1649
1650         /* Rx checksum disabled via ethtool */
1651         if (!(ring->netdev->features & NETIF_F_RXCSUM))
1652                 return;
1653
1654         /* TCP/UDP checksum error bit is set */
1655         if (igc_test_staterr(rx_desc,
1656                              IGC_RXDEXT_STATERR_L4E |
1657                              IGC_RXDEXT_STATERR_IPE)) {
1658                 /* work around errata with sctp packets where the TCPE aka
1659                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
1660                  * packets (aka let the stack check the crc32c)
1661                  */
1662                 if (!(skb->len == 60 &&
1663                       test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
1664                         u64_stats_update_begin(&ring->rx_syncp);
1665                         ring->rx_stats.csum_err++;
1666                         u64_stats_update_end(&ring->rx_syncp);
1667                 }
1668                 /* let the stack verify checksum errors */
1669                 return;
1670         }
1671         /* It must be a TCP or UDP packet with a valid checksum */
1672         if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS |
1673                                       IGC_RXD_STAT_UDPCS))
1674                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1675
1676         netdev_dbg(ring->netdev, "cksum success: bits %08X\n",
1677                    le32_to_cpu(rx_desc->wb.upper.status_error));
1678 }
1679
1680 static inline void igc_rx_hash(struct igc_ring *ring,
1681                                union igc_adv_rx_desc *rx_desc,
1682                                struct sk_buff *skb)
1683 {
1684         if (ring->netdev->features & NETIF_F_RXHASH)
1685                 skb_set_hash(skb,
1686                              le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
1687                              PKT_HASH_TYPE_L3);
1688 }
1689
1690 static void igc_rx_vlan(struct igc_ring *rx_ring,
1691                         union igc_adv_rx_desc *rx_desc,
1692                         struct sk_buff *skb)
1693 {
1694         struct net_device *dev = rx_ring->netdev;
1695         u16 vid;
1696
1697         if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1698             igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) {
1699                 if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) &&
1700                     test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
1701                         vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan);
1702                 else
1703                         vid = le16_to_cpu(rx_desc->wb.upper.vlan);
1704
1705                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1706         }
1707 }
1708
1709 /**
1710  * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1711  * @rx_ring: rx descriptor ring packet is being transacted on
1712  * @rx_desc: pointer to the EOP Rx descriptor
1713  * @skb: pointer to current skb being populated
1714  *
1715  * This function checks the ring, descriptor, and packet information in order
1716  * to populate the hash, checksum, VLAN, protocol, and other fields within the
1717  * skb.
1718  */
1719 static void igc_process_skb_fields(struct igc_ring *rx_ring,
1720                                    union igc_adv_rx_desc *rx_desc,
1721                                    struct sk_buff *skb)
1722 {
1723         igc_rx_hash(rx_ring, rx_desc, skb);
1724
1725         igc_rx_checksum(rx_ring, rx_desc, skb);
1726
1727         igc_rx_vlan(rx_ring, rx_desc, skb);
1728
1729         skb_record_rx_queue(skb, rx_ring->queue_index);
1730
1731         skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1732 }
1733
1734 static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features)
1735 {
1736         bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
1737         struct igc_adapter *adapter = netdev_priv(netdev);
1738         struct igc_hw *hw = &adapter->hw;
1739         u32 ctrl;
1740
1741         ctrl = rd32(IGC_CTRL);
1742
1743         if (enable) {
1744                 /* enable VLAN tag insert/strip */
1745                 ctrl |= IGC_CTRL_VME;
1746         } else {
1747                 /* disable VLAN tag insert/strip */
1748                 ctrl &= ~IGC_CTRL_VME;
1749         }
1750         wr32(IGC_CTRL, ctrl);
1751 }
1752
1753 static void igc_restore_vlan(struct igc_adapter *adapter)
1754 {
1755         igc_vlan_mode(adapter->netdev, adapter->netdev->features);
1756 }
1757
1758 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
1759                                                const unsigned int size,
1760                                                int *rx_buffer_pgcnt)
1761 {
1762         struct igc_rx_buffer *rx_buffer;
1763
1764         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1765         *rx_buffer_pgcnt =
1766 #if (PAGE_SIZE < 8192)
1767                 page_count(rx_buffer->page);
1768 #else
1769                 0;
1770 #endif
1771         prefetchw(rx_buffer->page);
1772
1773         /* we are reusing so sync this buffer for CPU use */
1774         dma_sync_single_range_for_cpu(rx_ring->dev,
1775                                       rx_buffer->dma,
1776                                       rx_buffer->page_offset,
1777                                       size,
1778                                       DMA_FROM_DEVICE);
1779
1780         rx_buffer->pagecnt_bias--;
1781
1782         return rx_buffer;
1783 }
1784
1785 static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer,
1786                                unsigned int truesize)
1787 {
1788 #if (PAGE_SIZE < 8192)
1789         buffer->page_offset ^= truesize;
1790 #else
1791         buffer->page_offset += truesize;
1792 #endif
1793 }
1794
1795 static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring,
1796                                               unsigned int size)
1797 {
1798         unsigned int truesize;
1799
1800 #if (PAGE_SIZE < 8192)
1801         truesize = igc_rx_pg_size(ring) / 2;
1802 #else
1803         truesize = ring_uses_build_skb(ring) ?
1804                    SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1805                    SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1806                    SKB_DATA_ALIGN(size);
1807 #endif
1808         return truesize;
1809 }
1810
1811 /**
1812  * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1813  * @rx_ring: rx descriptor ring to transact packets on
1814  * @rx_buffer: buffer containing page to add
1815  * @skb: sk_buff to place the data into
1816  * @size: size of buffer to be added
1817  *
1818  * This function will add the data contained in rx_buffer->page to the skb.
1819  */
1820 static void igc_add_rx_frag(struct igc_ring *rx_ring,
1821                             struct igc_rx_buffer *rx_buffer,
1822                             struct sk_buff *skb,
1823                             unsigned int size)
1824 {
1825         unsigned int truesize;
1826
1827 #if (PAGE_SIZE < 8192)
1828         truesize = igc_rx_pg_size(rx_ring) / 2;
1829 #else
1830         truesize = ring_uses_build_skb(rx_ring) ?
1831                    SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1832                    SKB_DATA_ALIGN(size);
1833 #endif
1834         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1835                         rx_buffer->page_offset, size, truesize);
1836
1837         igc_rx_buffer_flip(rx_buffer, truesize);
1838 }
1839
1840 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
1841                                      struct igc_rx_buffer *rx_buffer,
1842                                      struct xdp_buff *xdp)
1843 {
1844         unsigned int size = xdp->data_end - xdp->data;
1845         unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1846         unsigned int metasize = xdp->data - xdp->data_meta;
1847         struct sk_buff *skb;
1848
1849         /* prefetch first cache line of first page */
1850         net_prefetch(xdp->data_meta);
1851
1852         /* build an skb around the page buffer */
1853         skb = napi_build_skb(xdp->data_hard_start, truesize);
1854         if (unlikely(!skb))
1855                 return NULL;
1856
1857         /* update pointers within the skb to store the data */
1858         skb_reserve(skb, xdp->data - xdp->data_hard_start);
1859         __skb_put(skb, size);
1860         if (metasize)
1861                 skb_metadata_set(skb, metasize);
1862
1863         igc_rx_buffer_flip(rx_buffer, truesize);
1864         return skb;
1865 }
1866
1867 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
1868                                          struct igc_rx_buffer *rx_buffer,
1869                                          struct xdp_buff *xdp,
1870                                          ktime_t timestamp)
1871 {
1872         unsigned int metasize = xdp->data - xdp->data_meta;
1873         unsigned int size = xdp->data_end - xdp->data;
1874         unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1875         void *va = xdp->data;
1876         unsigned int headlen;
1877         struct sk_buff *skb;
1878
1879         /* prefetch first cache line of first page */
1880         net_prefetch(xdp->data_meta);
1881
1882         /* allocate a skb to store the frags */
1883         skb = napi_alloc_skb(&rx_ring->q_vector->napi,
1884                              IGC_RX_HDR_LEN + metasize);
1885         if (unlikely(!skb))
1886                 return NULL;
1887
1888         if (timestamp)
1889                 skb_hwtstamps(skb)->hwtstamp = timestamp;
1890
1891         /* Determine available headroom for copy */
1892         headlen = size;
1893         if (headlen > IGC_RX_HDR_LEN)
1894                 headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN);
1895
1896         /* align pull length to size of long to optimize memcpy performance */
1897         memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta,
1898                ALIGN(headlen + metasize, sizeof(long)));
1899
1900         if (metasize) {
1901                 skb_metadata_set(skb, metasize);
1902                 __skb_pull(skb, metasize);
1903         }
1904
1905         /* update all of the pointers */
1906         size -= headlen;
1907         if (size) {
1908                 skb_add_rx_frag(skb, 0, rx_buffer->page,
1909                                 (va + headlen) - page_address(rx_buffer->page),
1910                                 size, truesize);
1911                 igc_rx_buffer_flip(rx_buffer, truesize);
1912         } else {
1913                 rx_buffer->pagecnt_bias++;
1914         }
1915
1916         return skb;
1917 }
1918
1919 /**
1920  * igc_reuse_rx_page - page flip buffer and store it back on the ring
1921  * @rx_ring: rx descriptor ring to store buffers on
1922  * @old_buff: donor buffer to have page reused
1923  *
1924  * Synchronizes page for reuse by the adapter
1925  */
1926 static void igc_reuse_rx_page(struct igc_ring *rx_ring,
1927                               struct igc_rx_buffer *old_buff)
1928 {
1929         u16 nta = rx_ring->next_to_alloc;
1930         struct igc_rx_buffer *new_buff;
1931
1932         new_buff = &rx_ring->rx_buffer_info[nta];
1933
1934         /* update, and store next to alloc */
1935         nta++;
1936         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1937
1938         /* Transfer page from old buffer to new buffer.
1939          * Move each member individually to avoid possible store
1940          * forwarding stalls.
1941          */
1942         new_buff->dma           = old_buff->dma;
1943         new_buff->page          = old_buff->page;
1944         new_buff->page_offset   = old_buff->page_offset;
1945         new_buff->pagecnt_bias  = old_buff->pagecnt_bias;
1946 }
1947
1948 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer,
1949                                   int rx_buffer_pgcnt)
1950 {
1951         unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1952         struct page *page = rx_buffer->page;
1953
1954         /* avoid re-using remote and pfmemalloc pages */
1955         if (!dev_page_is_reusable(page))
1956                 return false;
1957
1958 #if (PAGE_SIZE < 8192)
1959         /* if we are only owner of page we can reuse it */
1960         if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1))
1961                 return false;
1962 #else
1963 #define IGC_LAST_OFFSET \
1964         (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
1965
1966         if (rx_buffer->page_offset > IGC_LAST_OFFSET)
1967                 return false;
1968 #endif
1969
1970         /* If we have drained the page fragment pool we need to update
1971          * the pagecnt_bias and page count so that we fully restock the
1972          * number of references the driver holds.
1973          */
1974         if (unlikely(pagecnt_bias == 1)) {
1975                 page_ref_add(page, USHRT_MAX - 1);
1976                 rx_buffer->pagecnt_bias = USHRT_MAX;
1977         }
1978
1979         return true;
1980 }
1981
1982 /**
1983  * igc_is_non_eop - process handling of non-EOP buffers
1984  * @rx_ring: Rx ring being processed
1985  * @rx_desc: Rx descriptor for current buffer
1986  *
1987  * This function updates next to clean.  If the buffer is an EOP buffer
1988  * this function exits returning false, otherwise it will place the
1989  * sk_buff in the next buffer to be chained and return true indicating
1990  * that this is in fact a non-EOP buffer.
1991  */
1992 static bool igc_is_non_eop(struct igc_ring *rx_ring,
1993                            union igc_adv_rx_desc *rx_desc)
1994 {
1995         u32 ntc = rx_ring->next_to_clean + 1;
1996
1997         /* fetch, update, and store next to clean */
1998         ntc = (ntc < rx_ring->count) ? ntc : 0;
1999         rx_ring->next_to_clean = ntc;
2000
2001         prefetch(IGC_RX_DESC(rx_ring, ntc));
2002
2003         if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP)))
2004                 return false;
2005
2006         return true;
2007 }
2008
2009 /**
2010  * igc_cleanup_headers - Correct corrupted or empty headers
2011  * @rx_ring: rx descriptor ring packet is being transacted on
2012  * @rx_desc: pointer to the EOP Rx descriptor
2013  * @skb: pointer to current skb being fixed
2014  *
2015  * Address the case where we are pulling data in on pages only
2016  * and as such no data is present in the skb header.
2017  *
2018  * In addition if skb is not at least 60 bytes we need to pad it so that
2019  * it is large enough to qualify as a valid Ethernet frame.
2020  *
2021  * Returns true if an error was encountered and skb was freed.
2022  */
2023 static bool igc_cleanup_headers(struct igc_ring *rx_ring,
2024                                 union igc_adv_rx_desc *rx_desc,
2025                                 struct sk_buff *skb)
2026 {
2027         /* XDP packets use error pointer so abort at this point */
2028         if (IS_ERR(skb))
2029                 return true;
2030
2031         if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) {
2032                 struct net_device *netdev = rx_ring->netdev;
2033
2034                 if (!(netdev->features & NETIF_F_RXALL)) {
2035                         dev_kfree_skb_any(skb);
2036                         return true;
2037                 }
2038         }
2039
2040         /* if eth_skb_pad returns an error the skb was freed */
2041         if (eth_skb_pad(skb))
2042                 return true;
2043
2044         return false;
2045 }
2046
2047 static void igc_put_rx_buffer(struct igc_ring *rx_ring,
2048                               struct igc_rx_buffer *rx_buffer,
2049                               int rx_buffer_pgcnt)
2050 {
2051         if (igc_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {
2052                 /* hand second half of page back to the ring */
2053                 igc_reuse_rx_page(rx_ring, rx_buffer);
2054         } else {
2055                 /* We are not reusing the buffer so unmap it and free
2056                  * any references we are holding to it
2057                  */
2058                 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
2059                                      igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
2060                                      IGC_RX_DMA_ATTR);
2061                 __page_frag_cache_drain(rx_buffer->page,
2062                                         rx_buffer->pagecnt_bias);
2063         }
2064
2065         /* clear contents of rx_buffer */
2066         rx_buffer->page = NULL;
2067 }
2068
2069 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
2070 {
2071         struct igc_adapter *adapter = rx_ring->q_vector->adapter;
2072
2073         if (ring_uses_build_skb(rx_ring))
2074                 return IGC_SKB_PAD;
2075         if (igc_xdp_is_enabled(adapter))
2076                 return XDP_PACKET_HEADROOM;
2077
2078         return 0;
2079 }
2080
2081 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
2082                                   struct igc_rx_buffer *bi)
2083 {
2084         struct page *page = bi->page;
2085         dma_addr_t dma;
2086
2087         /* since we are recycling buffers we should seldom need to alloc */
2088         if (likely(page))
2089                 return true;
2090
2091         /* alloc new page for storage */
2092         page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
2093         if (unlikely(!page)) {
2094                 rx_ring->rx_stats.alloc_failed++;
2095                 return false;
2096         }
2097
2098         /* map page for use */
2099         dma = dma_map_page_attrs(rx_ring->dev, page, 0,
2100                                  igc_rx_pg_size(rx_ring),
2101                                  DMA_FROM_DEVICE,
2102                                  IGC_RX_DMA_ATTR);
2103
2104         /* if mapping failed free memory back to system since
2105          * there isn't much point in holding memory we can't use
2106          */
2107         if (dma_mapping_error(rx_ring->dev, dma)) {
2108                 __free_page(page);
2109
2110                 rx_ring->rx_stats.alloc_failed++;
2111                 return false;
2112         }
2113
2114         bi->dma = dma;
2115         bi->page = page;
2116         bi->page_offset = igc_rx_offset(rx_ring);
2117         page_ref_add(page, USHRT_MAX - 1);
2118         bi->pagecnt_bias = USHRT_MAX;
2119
2120         return true;
2121 }
2122
2123 /**
2124  * igc_alloc_rx_buffers - Replace used receive buffers; packet split
2125  * @rx_ring: rx descriptor ring
2126  * @cleaned_count: number of buffers to clean
2127  */
2128 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
2129 {
2130         union igc_adv_rx_desc *rx_desc;
2131         u16 i = rx_ring->next_to_use;
2132         struct igc_rx_buffer *bi;
2133         u16 bufsz;
2134
2135         /* nothing to do */
2136         if (!cleaned_count)
2137                 return;
2138
2139         rx_desc = IGC_RX_DESC(rx_ring, i);
2140         bi = &rx_ring->rx_buffer_info[i];
2141         i -= rx_ring->count;
2142
2143         bufsz = igc_rx_bufsz(rx_ring);
2144
2145         do {
2146                 if (!igc_alloc_mapped_page(rx_ring, bi))
2147                         break;
2148
2149                 /* sync the buffer for use by the device */
2150                 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
2151                                                  bi->page_offset, bufsz,
2152                                                  DMA_FROM_DEVICE);
2153
2154                 /* Refresh the desc even if buffer_addrs didn't change
2155                  * because each write-back erases this info.
2156                  */
2157                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
2158
2159                 rx_desc++;
2160                 bi++;
2161                 i++;
2162                 if (unlikely(!i)) {
2163                         rx_desc = IGC_RX_DESC(rx_ring, 0);
2164                         bi = rx_ring->rx_buffer_info;
2165                         i -= rx_ring->count;
2166                 }
2167
2168                 /* clear the length for the next_to_use descriptor */
2169                 rx_desc->wb.upper.length = 0;
2170
2171                 cleaned_count--;
2172         } while (cleaned_count);
2173
2174         i += rx_ring->count;
2175
2176         if (rx_ring->next_to_use != i) {
2177                 /* record the next descriptor to use */
2178                 rx_ring->next_to_use = i;
2179
2180                 /* update next to alloc since we have filled the ring */
2181                 rx_ring->next_to_alloc = i;
2182
2183                 /* Force memory writes to complete before letting h/w
2184                  * know there are new descriptors to fetch.  (Only
2185                  * applicable for weak-ordered memory model archs,
2186                  * such as IA-64).
2187                  */
2188                 wmb();
2189                 writel(i, rx_ring->tail);
2190         }
2191 }
2192
2193 static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count)
2194 {
2195         union igc_adv_rx_desc *desc;
2196         u16 i = ring->next_to_use;
2197         struct igc_rx_buffer *bi;
2198         dma_addr_t dma;
2199         bool ok = true;
2200
2201         if (!count)
2202                 return ok;
2203
2204         desc = IGC_RX_DESC(ring, i);
2205         bi = &ring->rx_buffer_info[i];
2206         i -= ring->count;
2207
2208         do {
2209                 bi->xdp = xsk_buff_alloc(ring->xsk_pool);
2210                 if (!bi->xdp) {
2211                         ok = false;
2212                         break;
2213                 }
2214
2215                 dma = xsk_buff_xdp_get_dma(bi->xdp);
2216                 desc->read.pkt_addr = cpu_to_le64(dma);
2217
2218                 desc++;
2219                 bi++;
2220                 i++;
2221                 if (unlikely(!i)) {
2222                         desc = IGC_RX_DESC(ring, 0);
2223                         bi = ring->rx_buffer_info;
2224                         i -= ring->count;
2225                 }
2226
2227                 /* Clear the length for the next_to_use descriptor. */
2228                 desc->wb.upper.length = 0;
2229
2230                 count--;
2231         } while (count);
2232
2233         i += ring->count;
2234
2235         if (ring->next_to_use != i) {
2236                 ring->next_to_use = i;
2237
2238                 /* Force memory writes to complete before letting h/w
2239                  * know there are new descriptors to fetch.  (Only
2240                  * applicable for weak-ordered memory model archs,
2241                  * such as IA-64).
2242                  */
2243                 wmb();
2244                 writel(i, ring->tail);
2245         }
2246
2247         return ok;
2248 }
2249
2250 /* This function requires __netif_tx_lock is held by the caller. */
2251 static int igc_xdp_init_tx_descriptor(struct igc_ring *ring,
2252                                       struct xdp_frame *xdpf)
2253 {
2254         struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
2255         u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
2256         u16 count, index = ring->next_to_use;
2257         struct igc_tx_buffer *head = &ring->tx_buffer_info[index];
2258         struct igc_tx_buffer *buffer = head;
2259         union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index);
2260         u32 olinfo_status, len = xdpf->len, cmd_type;
2261         void *data = xdpf->data;
2262         u16 i;
2263
2264         count = TXD_USE_COUNT(len);
2265         for (i = 0; i < nr_frags; i++)
2266                 count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
2267
2268         if (igc_maybe_stop_tx(ring, count + 3)) {
2269                 /* this is a hard error */
2270                 return -EBUSY;
2271         }
2272
2273         i = 0;
2274         head->bytecount = xdp_get_frame_len(xdpf);
2275         head->type = IGC_TX_BUFFER_TYPE_XDP;
2276         head->gso_segs = 1;
2277         head->xdpf = xdpf;
2278
2279         olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
2280         desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2281
2282         for (;;) {
2283                 dma_addr_t dma;
2284
2285                 dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE);
2286                 if (dma_mapping_error(ring->dev, dma)) {
2287                         netdev_err_once(ring->netdev,
2288                                         "Failed to map DMA for TX\n");
2289                         goto unmap;
2290                 }
2291
2292                 dma_unmap_len_set(buffer, len, len);
2293                 dma_unmap_addr_set(buffer, dma, dma);
2294
2295                 cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
2296                            IGC_ADVTXD_DCMD_IFCS | len;
2297
2298                 desc->read.cmd_type_len = cpu_to_le32(cmd_type);
2299                 desc->read.buffer_addr = cpu_to_le64(dma);
2300
2301                 buffer->protocol = 0;
2302
2303                 if (++index == ring->count)
2304                         index = 0;
2305
2306                 if (i == nr_frags)
2307                         break;
2308
2309                 buffer = &ring->tx_buffer_info[index];
2310                 desc = IGC_TX_DESC(ring, index);
2311                 desc->read.olinfo_status = 0;
2312
2313                 data = skb_frag_address(&sinfo->frags[i]);
2314                 len = skb_frag_size(&sinfo->frags[i]);
2315                 i++;
2316         }
2317         desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD);
2318
2319         netdev_tx_sent_queue(txring_txq(ring), head->bytecount);
2320         /* set the timestamp */
2321         head->time_stamp = jiffies;
2322         /* set next_to_watch value indicating a packet is present */
2323         head->next_to_watch = desc;
2324         ring->next_to_use = index;
2325
2326         return 0;
2327
2328 unmap:
2329         for (;;) {
2330                 buffer = &ring->tx_buffer_info[index];
2331                 if (dma_unmap_len(buffer, len))
2332                         dma_unmap_page(ring->dev,
2333                                        dma_unmap_addr(buffer, dma),
2334                                        dma_unmap_len(buffer, len),
2335                                        DMA_TO_DEVICE);
2336                 dma_unmap_len_set(buffer, len, 0);
2337                 if (buffer == head)
2338                         break;
2339
2340                 if (!index)
2341                         index += ring->count;
2342                 index--;
2343         }
2344
2345         return -ENOMEM;
2346 }
2347
2348 static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter,
2349                                             int cpu)
2350 {
2351         int index = cpu;
2352
2353         if (unlikely(index < 0))
2354                 index = 0;
2355
2356         while (index >= adapter->num_tx_queues)
2357                 index -= adapter->num_tx_queues;
2358
2359         return adapter->tx_ring[index];
2360 }
2361
2362 static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp)
2363 {
2364         struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2365         int cpu = smp_processor_id();
2366         struct netdev_queue *nq;
2367         struct igc_ring *ring;
2368         int res;
2369
2370         if (unlikely(!xdpf))
2371                 return -EFAULT;
2372
2373         ring = igc_xdp_get_tx_ring(adapter, cpu);
2374         nq = txring_txq(ring);
2375
2376         __netif_tx_lock(nq, cpu);
2377         res = igc_xdp_init_tx_descriptor(ring, xdpf);
2378         __netif_tx_unlock(nq);
2379         return res;
2380 }
2381
2382 /* This function assumes rcu_read_lock() is held by the caller. */
2383 static int __igc_xdp_run_prog(struct igc_adapter *adapter,
2384                               struct bpf_prog *prog,
2385                               struct xdp_buff *xdp)
2386 {
2387         u32 act = bpf_prog_run_xdp(prog, xdp);
2388
2389         switch (act) {
2390         case XDP_PASS:
2391                 return IGC_XDP_PASS;
2392         case XDP_TX:
2393                 if (igc_xdp_xmit_back(adapter, xdp) < 0)
2394                         goto out_failure;
2395                 return IGC_XDP_TX;
2396         case XDP_REDIRECT:
2397                 if (xdp_do_redirect(adapter->netdev, xdp, prog) < 0)
2398                         goto out_failure;
2399                 return IGC_XDP_REDIRECT;
2400                 break;
2401         default:
2402                 bpf_warn_invalid_xdp_action(adapter->netdev, prog, act);
2403                 fallthrough;
2404         case XDP_ABORTED:
2405 out_failure:
2406                 trace_xdp_exception(adapter->netdev, prog, act);
2407                 fallthrough;
2408         case XDP_DROP:
2409                 return IGC_XDP_CONSUMED;
2410         }
2411 }
2412
2413 static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter,
2414                                         struct xdp_buff *xdp)
2415 {
2416         struct bpf_prog *prog;
2417         int res;
2418
2419         prog = READ_ONCE(adapter->xdp_prog);
2420         if (!prog) {
2421                 res = IGC_XDP_PASS;
2422                 goto out;
2423         }
2424
2425         res = __igc_xdp_run_prog(adapter, prog, xdp);
2426
2427 out:
2428         return ERR_PTR(-res);
2429 }
2430
2431 /* This function assumes __netif_tx_lock is held by the caller. */
2432 static void igc_flush_tx_descriptors(struct igc_ring *ring)
2433 {
2434         /* Once tail pointer is updated, hardware can fetch the descriptors
2435          * any time so we issue a write membar here to ensure all memory
2436          * writes are complete before the tail pointer is updated.
2437          */
2438         wmb();
2439         writel(ring->next_to_use, ring->tail);
2440 }
2441
2442 static void igc_finalize_xdp(struct igc_adapter *adapter, int status)
2443 {
2444         int cpu = smp_processor_id();
2445         struct netdev_queue *nq;
2446         struct igc_ring *ring;
2447
2448         if (status & IGC_XDP_TX) {
2449                 ring = igc_xdp_get_tx_ring(adapter, cpu);
2450                 nq = txring_txq(ring);
2451
2452                 __netif_tx_lock(nq, cpu);
2453                 igc_flush_tx_descriptors(ring);
2454                 __netif_tx_unlock(nq);
2455         }
2456
2457         if (status & IGC_XDP_REDIRECT)
2458                 xdp_do_flush();
2459 }
2460
2461 static void igc_update_rx_stats(struct igc_q_vector *q_vector,
2462                                 unsigned int packets, unsigned int bytes)
2463 {
2464         struct igc_ring *ring = q_vector->rx.ring;
2465
2466         u64_stats_update_begin(&ring->rx_syncp);
2467         ring->rx_stats.packets += packets;
2468         ring->rx_stats.bytes += bytes;
2469         u64_stats_update_end(&ring->rx_syncp);
2470
2471         q_vector->rx.total_packets += packets;
2472         q_vector->rx.total_bytes += bytes;
2473 }
2474
2475 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
2476 {
2477         unsigned int total_bytes = 0, total_packets = 0;
2478         struct igc_adapter *adapter = q_vector->adapter;
2479         struct igc_ring *rx_ring = q_vector->rx.ring;
2480         struct sk_buff *skb = rx_ring->skb;
2481         u16 cleaned_count = igc_desc_unused(rx_ring);
2482         int xdp_status = 0, rx_buffer_pgcnt;
2483
2484         while (likely(total_packets < budget)) {
2485                 union igc_adv_rx_desc *rx_desc;
2486                 struct igc_rx_buffer *rx_buffer;
2487                 unsigned int size, truesize;
2488                 ktime_t timestamp = 0;
2489                 struct xdp_buff xdp;
2490                 int pkt_offset = 0;
2491                 void *pktbuf;
2492
2493                 /* return some buffers to hardware, one at a time is too slow */
2494                 if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
2495                         igc_alloc_rx_buffers(rx_ring, cleaned_count);
2496                         cleaned_count = 0;
2497                 }
2498
2499                 rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
2500                 size = le16_to_cpu(rx_desc->wb.upper.length);
2501                 if (!size)
2502                         break;
2503
2504                 /* This memory barrier is needed to keep us from reading
2505                  * any other fields out of the rx_desc until we know the
2506                  * descriptor has been written back
2507                  */
2508                 dma_rmb();
2509
2510                 rx_buffer = igc_get_rx_buffer(rx_ring, size, &rx_buffer_pgcnt);
2511                 truesize = igc_get_rx_frame_truesize(rx_ring, size);
2512
2513                 pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
2514
2515                 if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) {
2516                         timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
2517                                                         pktbuf);
2518                         pkt_offset = IGC_TS_HDR_LEN;
2519                         size -= IGC_TS_HDR_LEN;
2520                 }
2521
2522                 if (!skb) {
2523                         xdp_init_buff(&xdp, truesize, &rx_ring->xdp_rxq);
2524                         xdp_prepare_buff(&xdp, pktbuf - igc_rx_offset(rx_ring),
2525                                          igc_rx_offset(rx_ring) + pkt_offset,
2526                                          size, true);
2527                         xdp_buff_clear_frags_flag(&xdp);
2528
2529                         skb = igc_xdp_run_prog(adapter, &xdp);
2530                 }
2531
2532                 if (IS_ERR(skb)) {
2533                         unsigned int xdp_res = -PTR_ERR(skb);
2534
2535                         switch (xdp_res) {
2536                         case IGC_XDP_CONSUMED:
2537                                 rx_buffer->pagecnt_bias++;
2538                                 break;
2539                         case IGC_XDP_TX:
2540                         case IGC_XDP_REDIRECT:
2541                                 igc_rx_buffer_flip(rx_buffer, truesize);
2542                                 xdp_status |= xdp_res;
2543                                 break;
2544                         }
2545
2546                         total_packets++;
2547                         total_bytes += size;
2548                 } else if (skb)
2549                         igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
2550                 else if (ring_uses_build_skb(rx_ring))
2551                         skb = igc_build_skb(rx_ring, rx_buffer, &xdp);
2552                 else
2553                         skb = igc_construct_skb(rx_ring, rx_buffer, &xdp,
2554                                                 timestamp);
2555
2556                 /* exit if we failed to retrieve a buffer */
2557                 if (!skb) {
2558                         rx_ring->rx_stats.alloc_failed++;
2559                         rx_buffer->pagecnt_bias++;
2560                         break;
2561                 }
2562
2563                 igc_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt);
2564                 cleaned_count++;
2565
2566                 /* fetch next buffer in frame if non-eop */
2567                 if (igc_is_non_eop(rx_ring, rx_desc))
2568                         continue;
2569
2570                 /* verify the packet layout is correct */
2571                 if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
2572                         skb = NULL;
2573                         continue;
2574                 }
2575
2576                 /* probably a little skewed due to removing CRC */
2577                 total_bytes += skb->len;
2578
2579                 /* populate checksum, VLAN, and protocol */
2580                 igc_process_skb_fields(rx_ring, rx_desc, skb);
2581
2582                 napi_gro_receive(&q_vector->napi, skb);
2583
2584                 /* reset skb pointer */
2585                 skb = NULL;
2586
2587                 /* update budget accounting */
2588                 total_packets++;
2589         }
2590
2591         if (xdp_status)
2592                 igc_finalize_xdp(adapter, xdp_status);
2593
2594         /* place incomplete frames back on ring for completion */
2595         rx_ring->skb = skb;
2596
2597         igc_update_rx_stats(q_vector, total_packets, total_bytes);
2598
2599         if (cleaned_count)
2600                 igc_alloc_rx_buffers(rx_ring, cleaned_count);
2601
2602         return total_packets;
2603 }
2604
2605 static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring,
2606                                             struct xdp_buff *xdp)
2607 {
2608         unsigned int totalsize = xdp->data_end - xdp->data_meta;
2609         unsigned int metasize = xdp->data - xdp->data_meta;
2610         struct sk_buff *skb;
2611
2612         net_prefetch(xdp->data_meta);
2613
2614         skb = __napi_alloc_skb(&ring->q_vector->napi, totalsize,
2615                                GFP_ATOMIC | __GFP_NOWARN);
2616         if (unlikely(!skb))
2617                 return NULL;
2618
2619         memcpy(__skb_put(skb, totalsize), xdp->data_meta,
2620                ALIGN(totalsize, sizeof(long)));
2621
2622         if (metasize) {
2623                 skb_metadata_set(skb, metasize);
2624                 __skb_pull(skb, metasize);
2625         }
2626
2627         return skb;
2628 }
2629
2630 static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector,
2631                                 union igc_adv_rx_desc *desc,
2632                                 struct xdp_buff *xdp,
2633                                 ktime_t timestamp)
2634 {
2635         struct igc_ring *ring = q_vector->rx.ring;
2636         struct sk_buff *skb;
2637
2638         skb = igc_construct_skb_zc(ring, xdp);
2639         if (!skb) {
2640                 ring->rx_stats.alloc_failed++;
2641                 return;
2642         }
2643
2644         if (timestamp)
2645                 skb_hwtstamps(skb)->hwtstamp = timestamp;
2646
2647         if (igc_cleanup_headers(ring, desc, skb))
2648                 return;
2649
2650         igc_process_skb_fields(ring, desc, skb);
2651         napi_gro_receive(&q_vector->napi, skb);
2652 }
2653
2654 static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget)
2655 {
2656         struct igc_adapter *adapter = q_vector->adapter;
2657         struct igc_ring *ring = q_vector->rx.ring;
2658         u16 cleaned_count = igc_desc_unused(ring);
2659         int total_bytes = 0, total_packets = 0;
2660         u16 ntc = ring->next_to_clean;
2661         struct bpf_prog *prog;
2662         bool failure = false;
2663         int xdp_status = 0;
2664
2665         rcu_read_lock();
2666
2667         prog = READ_ONCE(adapter->xdp_prog);
2668
2669         while (likely(total_packets < budget)) {
2670                 union igc_adv_rx_desc *desc;
2671                 struct igc_rx_buffer *bi;
2672                 ktime_t timestamp = 0;
2673                 unsigned int size;
2674                 int res;
2675
2676                 desc = IGC_RX_DESC(ring, ntc);
2677                 size = le16_to_cpu(desc->wb.upper.length);
2678                 if (!size)
2679                         break;
2680
2681                 /* This memory barrier is needed to keep us from reading
2682                  * any other fields out of the rx_desc until we know the
2683                  * descriptor has been written back
2684                  */
2685                 dma_rmb();
2686
2687                 bi = &ring->rx_buffer_info[ntc];
2688
2689                 if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) {
2690                         timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
2691                                                         bi->xdp->data);
2692
2693                         bi->xdp->data += IGC_TS_HDR_LEN;
2694
2695                         /* HW timestamp has been copied into local variable. Metadata
2696                          * length when XDP program is called should be 0.
2697                          */
2698                         bi->xdp->data_meta += IGC_TS_HDR_LEN;
2699                         size -= IGC_TS_HDR_LEN;
2700                 }
2701
2702                 bi->xdp->data_end = bi->xdp->data + size;
2703                 xsk_buff_dma_sync_for_cpu(bi->xdp, ring->xsk_pool);
2704
2705                 res = __igc_xdp_run_prog(adapter, prog, bi->xdp);
2706                 switch (res) {
2707                 case IGC_XDP_PASS:
2708                         igc_dispatch_skb_zc(q_vector, desc, bi->xdp, timestamp);
2709                         fallthrough;
2710                 case IGC_XDP_CONSUMED:
2711                         xsk_buff_free(bi->xdp);
2712                         break;
2713                 case IGC_XDP_TX:
2714                 case IGC_XDP_REDIRECT:
2715                         xdp_status |= res;
2716                         break;
2717                 }
2718
2719                 bi->xdp = NULL;
2720                 total_bytes += size;
2721                 total_packets++;
2722                 cleaned_count++;
2723                 ntc++;
2724                 if (ntc == ring->count)
2725                         ntc = 0;
2726         }
2727
2728         ring->next_to_clean = ntc;
2729         rcu_read_unlock();
2730
2731         if (cleaned_count >= IGC_RX_BUFFER_WRITE)
2732                 failure = !igc_alloc_rx_buffers_zc(ring, cleaned_count);
2733
2734         if (xdp_status)
2735                 igc_finalize_xdp(adapter, xdp_status);
2736
2737         igc_update_rx_stats(q_vector, total_packets, total_bytes);
2738
2739         if (xsk_uses_need_wakeup(ring->xsk_pool)) {
2740                 if (failure || ring->next_to_clean == ring->next_to_use)
2741                         xsk_set_rx_need_wakeup(ring->xsk_pool);
2742                 else
2743                         xsk_clear_rx_need_wakeup(ring->xsk_pool);
2744                 return total_packets;
2745         }
2746
2747         return failure ? budget : total_packets;
2748 }
2749
2750 static void igc_update_tx_stats(struct igc_q_vector *q_vector,
2751                                 unsigned int packets, unsigned int bytes)
2752 {
2753         struct igc_ring *ring = q_vector->tx.ring;
2754
2755         u64_stats_update_begin(&ring->tx_syncp);
2756         ring->tx_stats.bytes += bytes;
2757         ring->tx_stats.packets += packets;
2758         u64_stats_update_end(&ring->tx_syncp);
2759
2760         q_vector->tx.total_bytes += bytes;
2761         q_vector->tx.total_packets += packets;
2762 }
2763
2764 static void igc_xdp_xmit_zc(struct igc_ring *ring)
2765 {
2766         struct xsk_buff_pool *pool = ring->xsk_pool;
2767         struct netdev_queue *nq = txring_txq(ring);
2768         union igc_adv_tx_desc *tx_desc = NULL;
2769         int cpu = smp_processor_id();
2770         u16 ntu = ring->next_to_use;
2771         struct xdp_desc xdp_desc;
2772         u16 budget;
2773
2774         if (!netif_carrier_ok(ring->netdev))
2775                 return;
2776
2777         __netif_tx_lock(nq, cpu);
2778
2779         budget = igc_desc_unused(ring);
2780
2781         while (xsk_tx_peek_desc(pool, &xdp_desc) && budget--) {
2782                 u32 cmd_type, olinfo_status;
2783                 struct igc_tx_buffer *bi;
2784                 dma_addr_t dma;
2785
2786                 cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
2787                            IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
2788                            xdp_desc.len;
2789                 olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT;
2790
2791                 dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
2792                 xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len);
2793
2794                 tx_desc = IGC_TX_DESC(ring, ntu);
2795                 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
2796                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2797                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
2798
2799                 bi = &ring->tx_buffer_info[ntu];
2800                 bi->type = IGC_TX_BUFFER_TYPE_XSK;
2801                 bi->protocol = 0;
2802                 bi->bytecount = xdp_desc.len;
2803                 bi->gso_segs = 1;
2804                 bi->time_stamp = jiffies;
2805                 bi->next_to_watch = tx_desc;
2806
2807                 netdev_tx_sent_queue(txring_txq(ring), xdp_desc.len);
2808
2809                 ntu++;
2810                 if (ntu == ring->count)
2811                         ntu = 0;
2812         }
2813
2814         ring->next_to_use = ntu;
2815         if (tx_desc) {
2816                 igc_flush_tx_descriptors(ring);
2817                 xsk_tx_release(pool);
2818         }
2819
2820         __netif_tx_unlock(nq);
2821 }
2822
2823 /**
2824  * igc_clean_tx_irq - Reclaim resources after transmit completes
2825  * @q_vector: pointer to q_vector containing needed info
2826  * @napi_budget: Used to determine if we are in netpoll
2827  *
2828  * returns true if ring is completely cleaned
2829  */
2830 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
2831 {
2832         struct igc_adapter *adapter = q_vector->adapter;
2833         unsigned int total_bytes = 0, total_packets = 0;
2834         unsigned int budget = q_vector->tx.work_limit;
2835         struct igc_ring *tx_ring = q_vector->tx.ring;
2836         unsigned int i = tx_ring->next_to_clean;
2837         struct igc_tx_buffer *tx_buffer;
2838         union igc_adv_tx_desc *tx_desc;
2839         u32 xsk_frames = 0;
2840
2841         if (test_bit(__IGC_DOWN, &adapter->state))
2842                 return true;
2843
2844         tx_buffer = &tx_ring->tx_buffer_info[i];
2845         tx_desc = IGC_TX_DESC(tx_ring, i);
2846         i -= tx_ring->count;
2847
2848         do {
2849                 union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
2850
2851                 /* if next_to_watch is not set then there is no work pending */
2852                 if (!eop_desc)
2853                         break;
2854
2855                 /* prevent any other reads prior to eop_desc */
2856                 smp_rmb();
2857
2858                 /* if DD is not set pending work has not been completed */
2859                 if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD)))
2860                         break;
2861
2862                 /* clear next_to_watch to prevent false hangs */
2863                 tx_buffer->next_to_watch = NULL;
2864
2865                 /* update the statistics for this packet */
2866                 total_bytes += tx_buffer->bytecount;
2867                 total_packets += tx_buffer->gso_segs;
2868
2869                 switch (tx_buffer->type) {
2870                 case IGC_TX_BUFFER_TYPE_XSK:
2871                         xsk_frames++;
2872                         break;
2873                 case IGC_TX_BUFFER_TYPE_XDP:
2874                         xdp_return_frame(tx_buffer->xdpf);
2875                         igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2876                         break;
2877                 case IGC_TX_BUFFER_TYPE_SKB:
2878                         napi_consume_skb(tx_buffer->skb, napi_budget);
2879                         igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2880                         break;
2881                 default:
2882                         netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
2883                         break;
2884                 }
2885
2886                 /* clear last DMA location and unmap remaining buffers */
2887                 while (tx_desc != eop_desc) {
2888                         tx_buffer++;
2889                         tx_desc++;
2890                         i++;
2891                         if (unlikely(!i)) {
2892                                 i -= tx_ring->count;
2893                                 tx_buffer = tx_ring->tx_buffer_info;
2894                                 tx_desc = IGC_TX_DESC(tx_ring, 0);
2895                         }
2896
2897                         /* unmap any remaining paged data */
2898                         if (dma_unmap_len(tx_buffer, len))
2899                                 igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2900                 }
2901
2902                 /* move us one more past the eop_desc for start of next pkt */
2903                 tx_buffer++;
2904                 tx_desc++;
2905                 i++;
2906                 if (unlikely(!i)) {
2907                         i -= tx_ring->count;
2908                         tx_buffer = tx_ring->tx_buffer_info;
2909                         tx_desc = IGC_TX_DESC(tx_ring, 0);
2910                 }
2911
2912                 /* issue prefetch for next Tx descriptor */
2913                 prefetch(tx_desc);
2914
2915                 /* update budget accounting */
2916                 budget--;
2917         } while (likely(budget));
2918
2919         netdev_tx_completed_queue(txring_txq(tx_ring),
2920                                   total_packets, total_bytes);
2921
2922         i += tx_ring->count;
2923         tx_ring->next_to_clean = i;
2924
2925         igc_update_tx_stats(q_vector, total_packets, total_bytes);
2926
2927         if (tx_ring->xsk_pool) {
2928                 if (xsk_frames)
2929                         xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
2930                 if (xsk_uses_need_wakeup(tx_ring->xsk_pool))
2931                         xsk_set_tx_need_wakeup(tx_ring->xsk_pool);
2932                 igc_xdp_xmit_zc(tx_ring);
2933         }
2934
2935         if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
2936                 struct igc_hw *hw = &adapter->hw;
2937
2938                 /* Detect a transmit hang in hardware, this serializes the
2939                  * check with the clearing of time_stamp and movement of i
2940                  */
2941                 clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
2942                 if (tx_buffer->next_to_watch &&
2943                     time_after(jiffies, tx_buffer->time_stamp +
2944                     (adapter->tx_timeout_factor * HZ)) &&
2945                     !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) {
2946                         /* detected Tx unit hang */
2947                         netdev_err(tx_ring->netdev,
2948                                    "Detected Tx Unit Hang\n"
2949                                    "  Tx Queue             <%d>\n"
2950                                    "  TDH                  <%x>\n"
2951                                    "  TDT                  <%x>\n"
2952                                    "  next_to_use          <%x>\n"
2953                                    "  next_to_clean        <%x>\n"
2954                                    "buffer_info[next_to_clean]\n"
2955                                    "  time_stamp           <%lx>\n"
2956                                    "  next_to_watch        <%p>\n"
2957                                    "  jiffies              <%lx>\n"
2958                                    "  desc.status          <%x>\n",
2959                                    tx_ring->queue_index,
2960                                    rd32(IGC_TDH(tx_ring->reg_idx)),
2961                                    readl(tx_ring->tail),
2962                                    tx_ring->next_to_use,
2963                                    tx_ring->next_to_clean,
2964                                    tx_buffer->time_stamp,
2965                                    tx_buffer->next_to_watch,
2966                                    jiffies,
2967                                    tx_buffer->next_to_watch->wb.status);
2968                         netif_stop_subqueue(tx_ring->netdev,
2969                                             tx_ring->queue_index);
2970
2971                         /* we are about to reset, no point in enabling stuff */
2972                         return true;
2973                 }
2974         }
2975
2976 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
2977         if (unlikely(total_packets &&
2978                      netif_carrier_ok(tx_ring->netdev) &&
2979                      igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
2980                 /* Make sure that anybody stopping the queue after this
2981                  * sees the new next_to_clean.
2982                  */
2983                 smp_mb();
2984                 if (__netif_subqueue_stopped(tx_ring->netdev,
2985                                              tx_ring->queue_index) &&
2986                     !(test_bit(__IGC_DOWN, &adapter->state))) {
2987                         netif_wake_subqueue(tx_ring->netdev,
2988                                             tx_ring->queue_index);
2989
2990                         u64_stats_update_begin(&tx_ring->tx_syncp);
2991                         tx_ring->tx_stats.restart_queue++;
2992                         u64_stats_update_end(&tx_ring->tx_syncp);
2993                 }
2994         }
2995
2996         return !!budget;
2997 }
2998
2999 static int igc_find_mac_filter(struct igc_adapter *adapter,
3000                                enum igc_mac_filter_type type, const u8 *addr)
3001 {
3002         struct igc_hw *hw = &adapter->hw;
3003         int max_entries = hw->mac.rar_entry_count;
3004         u32 ral, rah;
3005         int i;
3006
3007         for (i = 0; i < max_entries; i++) {
3008                 ral = rd32(IGC_RAL(i));
3009                 rah = rd32(IGC_RAH(i));
3010
3011                 if (!(rah & IGC_RAH_AV))
3012                         continue;
3013                 if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type)
3014                         continue;
3015                 if ((rah & IGC_RAH_RAH_MASK) !=
3016                     le16_to_cpup((__le16 *)(addr + 4)))
3017                         continue;
3018                 if (ral != le32_to_cpup((__le32 *)(addr)))
3019                         continue;
3020
3021                 return i;
3022         }
3023
3024         return -1;
3025 }
3026
3027 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter)
3028 {
3029         struct igc_hw *hw = &adapter->hw;
3030         int max_entries = hw->mac.rar_entry_count;
3031         u32 rah;
3032         int i;
3033
3034         for (i = 0; i < max_entries; i++) {
3035                 rah = rd32(IGC_RAH(i));
3036
3037                 if (!(rah & IGC_RAH_AV))
3038                         return i;
3039         }
3040
3041         return -1;
3042 }
3043
3044 /**
3045  * igc_add_mac_filter() - Add MAC address filter
3046  * @adapter: Pointer to adapter where the filter should be added
3047  * @type: MAC address filter type (source or destination)
3048  * @addr: MAC address
3049  * @queue: If non-negative, queue assignment feature is enabled and frames
3050  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
3051  *         assignment is disabled.
3052  *
3053  * Return: 0 in case of success, negative errno code otherwise.
3054  */
3055 static int igc_add_mac_filter(struct igc_adapter *adapter,
3056                               enum igc_mac_filter_type type, const u8 *addr,
3057                               int queue)
3058 {
3059         struct net_device *dev = adapter->netdev;
3060         int index;
3061
3062         index = igc_find_mac_filter(adapter, type, addr);
3063         if (index >= 0)
3064                 goto update_filter;
3065
3066         index = igc_get_avail_mac_filter_slot(adapter);
3067         if (index < 0)
3068                 return -ENOSPC;
3069
3070         netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n",
3071                    index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
3072                    addr, queue);
3073
3074 update_filter:
3075         igc_set_mac_filter_hw(adapter, index, type, addr, queue);
3076         return 0;
3077 }
3078
3079 /**
3080  * igc_del_mac_filter() - Delete MAC address filter
3081  * @adapter: Pointer to adapter where the filter should be deleted from
3082  * @type: MAC address filter type (source or destination)
3083  * @addr: MAC address
3084  */
3085 static void igc_del_mac_filter(struct igc_adapter *adapter,
3086                                enum igc_mac_filter_type type, const u8 *addr)
3087 {
3088         struct net_device *dev = adapter->netdev;
3089         int index;
3090
3091         index = igc_find_mac_filter(adapter, type, addr);
3092         if (index < 0)
3093                 return;
3094
3095         if (index == 0) {
3096                 /* If this is the default filter, we don't actually delete it.
3097                  * We just reset to its default value i.e. disable queue
3098                  * assignment.
3099                  */
3100                 netdev_dbg(dev, "Disable default MAC filter queue assignment");
3101
3102                 igc_set_mac_filter_hw(adapter, 0, type, addr, -1);
3103         } else {
3104                 netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n",
3105                            index,
3106                            type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
3107                            addr);
3108
3109                 igc_clear_mac_filter_hw(adapter, index);
3110         }
3111 }
3112
3113 /**
3114  * igc_add_vlan_prio_filter() - Add VLAN priority filter
3115  * @adapter: Pointer to adapter where the filter should be added
3116  * @prio: VLAN priority value
3117  * @queue: Queue number which matching frames are assigned to
3118  *
3119  * Return: 0 in case of success, negative errno code otherwise.
3120  */
3121 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio,
3122                                     int queue)
3123 {
3124         struct net_device *dev = adapter->netdev;
3125         struct igc_hw *hw = &adapter->hw;
3126         u32 vlanpqf;
3127
3128         vlanpqf = rd32(IGC_VLANPQF);
3129
3130         if (vlanpqf & IGC_VLANPQF_VALID(prio)) {
3131                 netdev_dbg(dev, "VLAN priority filter already in use\n");
3132                 return -EEXIST;
3133         }
3134
3135         vlanpqf |= IGC_VLANPQF_QSEL(prio, queue);
3136         vlanpqf |= IGC_VLANPQF_VALID(prio);
3137
3138         wr32(IGC_VLANPQF, vlanpqf);
3139
3140         netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n",
3141                    prio, queue);
3142         return 0;
3143 }
3144
3145 /**
3146  * igc_del_vlan_prio_filter() - Delete VLAN priority filter
3147  * @adapter: Pointer to adapter where the filter should be deleted from
3148  * @prio: VLAN priority value
3149  */
3150 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio)
3151 {
3152         struct igc_hw *hw = &adapter->hw;
3153         u32 vlanpqf;
3154
3155         vlanpqf = rd32(IGC_VLANPQF);
3156
3157         vlanpqf &= ~IGC_VLANPQF_VALID(prio);
3158         vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK);
3159
3160         wr32(IGC_VLANPQF, vlanpqf);
3161
3162         netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n",
3163                    prio);
3164 }
3165
3166 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter)
3167 {
3168         struct igc_hw *hw = &adapter->hw;
3169         int i;
3170
3171         for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3172                 u32 etqf = rd32(IGC_ETQF(i));
3173
3174                 if (!(etqf & IGC_ETQF_FILTER_ENABLE))
3175                         return i;
3176         }
3177
3178         return -1;
3179 }
3180
3181 /**
3182  * igc_add_etype_filter() - Add ethertype filter
3183  * @adapter: Pointer to adapter where the filter should be added
3184  * @etype: Ethertype value
3185  * @queue: If non-negative, queue assignment feature is enabled and frames
3186  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
3187  *         assignment is disabled.
3188  *
3189  * Return: 0 in case of success, negative errno code otherwise.
3190  */
3191 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype,
3192                                 int queue)
3193 {
3194         struct igc_hw *hw = &adapter->hw;
3195         int index;
3196         u32 etqf;
3197
3198         index = igc_get_avail_etype_filter_slot(adapter);
3199         if (index < 0)
3200                 return -ENOSPC;
3201
3202         etqf = rd32(IGC_ETQF(index));
3203
3204         etqf &= ~IGC_ETQF_ETYPE_MASK;
3205         etqf |= etype;
3206
3207         if (queue >= 0) {
3208                 etqf &= ~IGC_ETQF_QUEUE_MASK;
3209                 etqf |= (queue << IGC_ETQF_QUEUE_SHIFT);
3210                 etqf |= IGC_ETQF_QUEUE_ENABLE;
3211         }
3212
3213         etqf |= IGC_ETQF_FILTER_ENABLE;
3214
3215         wr32(IGC_ETQF(index), etqf);
3216
3217         netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n",
3218                    etype, queue);
3219         return 0;
3220 }
3221
3222 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype)
3223 {
3224         struct igc_hw *hw = &adapter->hw;
3225         int i;
3226
3227         for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3228                 u32 etqf = rd32(IGC_ETQF(i));
3229
3230                 if ((etqf & IGC_ETQF_ETYPE_MASK) == etype)
3231                         return i;
3232         }
3233
3234         return -1;
3235 }
3236
3237 /**
3238  * igc_del_etype_filter() - Delete ethertype filter
3239  * @adapter: Pointer to adapter where the filter should be deleted from
3240  * @etype: Ethertype value
3241  */
3242 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype)
3243 {
3244         struct igc_hw *hw = &adapter->hw;
3245         int index;
3246
3247         index = igc_find_etype_filter(adapter, etype);
3248         if (index < 0)
3249                 return;
3250
3251         wr32(IGC_ETQF(index), 0);
3252
3253         netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n",
3254                    etype);
3255 }
3256
3257 static int igc_flex_filter_select(struct igc_adapter *adapter,
3258                                   struct igc_flex_filter *input,
3259                                   u32 *fhft)
3260 {
3261         struct igc_hw *hw = &adapter->hw;
3262         u8 fhft_index;
3263         u32 fhftsl;
3264
3265         if (input->index >= MAX_FLEX_FILTER) {
3266                 dev_err(&adapter->pdev->dev, "Wrong Flex Filter index selected!\n");
3267                 return -EINVAL;
3268         }
3269
3270         /* Indirect table select register */
3271         fhftsl = rd32(IGC_FHFTSL);
3272         fhftsl &= ~IGC_FHFTSL_FTSL_MASK;
3273         switch (input->index) {
3274         case 0 ... 7:
3275                 fhftsl |= 0x00;
3276                 break;
3277         case 8 ... 15:
3278                 fhftsl |= 0x01;
3279                 break;
3280         case 16 ... 23:
3281                 fhftsl |= 0x02;
3282                 break;
3283         case 24 ... 31:
3284                 fhftsl |= 0x03;
3285                 break;
3286         }
3287         wr32(IGC_FHFTSL, fhftsl);
3288
3289         /* Normalize index down to host table register */
3290         fhft_index = input->index % 8;
3291
3292         *fhft = (fhft_index < 4) ? IGC_FHFT(fhft_index) :
3293                 IGC_FHFT_EXT(fhft_index - 4);
3294
3295         return 0;
3296 }
3297
3298 static int igc_write_flex_filter_ll(struct igc_adapter *adapter,
3299                                     struct igc_flex_filter *input)
3300 {
3301         struct device *dev = &adapter->pdev->dev;
3302         struct igc_hw *hw = &adapter->hw;
3303         u8 *data = input->data;
3304         u8 *mask = input->mask;
3305         u32 queuing;
3306         u32 fhft;
3307         u32 wufc;
3308         int ret;
3309         int i;
3310
3311         /* Length has to be aligned to 8. Otherwise the filter will fail. Bail
3312          * out early to avoid surprises later.
3313          */
3314         if (input->length % 8 != 0) {
3315                 dev_err(dev, "The length of a flex filter has to be 8 byte aligned!\n");
3316                 return -EINVAL;
3317         }
3318
3319         /* Select corresponding flex filter register and get base for host table. */
3320         ret = igc_flex_filter_select(adapter, input, &fhft);
3321         if (ret)
3322                 return ret;
3323
3324         /* When adding a filter globally disable flex filter feature. That is
3325          * recommended within the datasheet.
3326          */
3327         wufc = rd32(IGC_WUFC);
3328         wufc &= ~IGC_WUFC_FLEX_HQ;
3329         wr32(IGC_WUFC, wufc);
3330
3331         /* Configure filter */
3332         queuing = input->length & IGC_FHFT_LENGTH_MASK;
3333         queuing |= (input->rx_queue << IGC_FHFT_QUEUE_SHIFT) & IGC_FHFT_QUEUE_MASK;
3334         queuing |= (input->prio << IGC_FHFT_PRIO_SHIFT) & IGC_FHFT_PRIO_MASK;
3335
3336         if (input->immediate_irq)
3337                 queuing |= IGC_FHFT_IMM_INT;
3338
3339         if (input->drop)
3340                 queuing |= IGC_FHFT_DROP;
3341
3342         wr32(fhft + 0xFC, queuing);
3343
3344         /* Write data (128 byte) and mask (128 bit) */
3345         for (i = 0; i < 16; ++i) {
3346                 const size_t data_idx = i * 8;
3347                 const size_t row_idx = i * 16;
3348                 u32 dw0 =
3349                         (data[data_idx + 0] << 0) |
3350                         (data[data_idx + 1] << 8) |
3351                         (data[data_idx + 2] << 16) |
3352                         (data[data_idx + 3] << 24);
3353                 u32 dw1 =
3354                         (data[data_idx + 4] << 0) |
3355                         (data[data_idx + 5] << 8) |
3356                         (data[data_idx + 6] << 16) |
3357                         (data[data_idx + 7] << 24);
3358                 u32 tmp;
3359
3360                 /* Write row: dw0, dw1 and mask */
3361                 wr32(fhft + row_idx, dw0);
3362                 wr32(fhft + row_idx + 4, dw1);
3363
3364                 /* mask is only valid for MASK(7, 0) */
3365                 tmp = rd32(fhft + row_idx + 8);
3366                 tmp &= ~GENMASK(7, 0);
3367                 tmp |= mask[i];
3368                 wr32(fhft + row_idx + 8, tmp);
3369         }
3370
3371         /* Enable filter. */
3372         wufc |= IGC_WUFC_FLEX_HQ;
3373         if (input->index > 8) {
3374                 /* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */
3375                 u32 wufc_ext = rd32(IGC_WUFC_EXT);
3376
3377                 wufc_ext |= (IGC_WUFC_EXT_FLX8 << (input->index - 8));
3378
3379                 wr32(IGC_WUFC_EXT, wufc_ext);
3380         } else {
3381                 wufc |= (IGC_WUFC_FLX0 << input->index);
3382         }
3383         wr32(IGC_WUFC, wufc);
3384
3385         dev_dbg(&adapter->pdev->dev, "Added flex filter %u to HW.\n",
3386                 input->index);
3387
3388         return 0;
3389 }
3390
3391 static void igc_flex_filter_add_field(struct igc_flex_filter *flex,
3392                                       const void *src, unsigned int offset,
3393                                       size_t len, const void *mask)
3394 {
3395         int i;
3396
3397         /* data */
3398         memcpy(&flex->data[offset], src, len);
3399
3400         /* mask */
3401         for (i = 0; i < len; ++i) {
3402                 const unsigned int idx = i + offset;
3403                 const u8 *ptr = mask;
3404
3405                 if (mask) {
3406                         if (ptr[i] & 0xff)
3407                                 flex->mask[idx / 8] |= BIT(idx % 8);
3408
3409                         continue;
3410                 }
3411
3412                 flex->mask[idx / 8] |= BIT(idx % 8);
3413         }
3414 }
3415
3416 static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter)
3417 {
3418         struct igc_hw *hw = &adapter->hw;
3419         u32 wufc, wufc_ext;
3420         int i;
3421
3422         wufc = rd32(IGC_WUFC);
3423         wufc_ext = rd32(IGC_WUFC_EXT);
3424
3425         for (i = 0; i < MAX_FLEX_FILTER; i++) {
3426                 if (i < 8) {
3427                         if (!(wufc & (IGC_WUFC_FLX0 << i)))
3428                                 return i;
3429                 } else {
3430                         if (!(wufc_ext & (IGC_WUFC_EXT_FLX8 << (i - 8))))
3431                                 return i;
3432                 }
3433         }
3434
3435         return -ENOSPC;
3436 }
3437
3438 static bool igc_flex_filter_in_use(struct igc_adapter *adapter)
3439 {
3440         struct igc_hw *hw = &adapter->hw;
3441         u32 wufc, wufc_ext;
3442
3443         wufc = rd32(IGC_WUFC);
3444         wufc_ext = rd32(IGC_WUFC_EXT);
3445
3446         if (wufc & IGC_WUFC_FILTER_MASK)
3447                 return true;
3448
3449         if (wufc_ext & IGC_WUFC_EXT_FILTER_MASK)
3450                 return true;
3451
3452         return false;
3453 }
3454
3455 static int igc_add_flex_filter(struct igc_adapter *adapter,
3456                                struct igc_nfc_rule *rule)
3457 {
3458         struct igc_flex_filter flex = { };
3459         struct igc_nfc_filter *filter = &rule->filter;
3460         unsigned int eth_offset, user_offset;
3461         int ret, index;
3462         bool vlan;
3463
3464         index = igc_find_avail_flex_filter_slot(adapter);
3465         if (index < 0)
3466                 return -ENOSPC;
3467
3468         /* Construct the flex filter:
3469          *  -> dest_mac [6]
3470          *  -> src_mac [6]
3471          *  -> tpid [2]
3472          *  -> vlan tci [2]
3473          *  -> ether type [2]
3474          *  -> user data [8]
3475          *  -> = 26 bytes => 32 length
3476          */
3477         flex.index    = index;
3478         flex.length   = 32;
3479         flex.rx_queue = rule->action;
3480
3481         vlan = rule->filter.vlan_tci || rule->filter.vlan_etype;
3482         eth_offset = vlan ? 16 : 12;
3483         user_offset = vlan ? 18 : 14;
3484
3485         /* Add destination MAC  */
3486         if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
3487                 igc_flex_filter_add_field(&flex, &filter->dst_addr, 0,
3488                                           ETH_ALEN, NULL);
3489
3490         /* Add source MAC */
3491         if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
3492                 igc_flex_filter_add_field(&flex, &filter->src_addr, 6,
3493                                           ETH_ALEN, NULL);
3494
3495         /* Add VLAN etype */
3496         if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE)
3497                 igc_flex_filter_add_field(&flex, &filter->vlan_etype, 12,
3498                                           sizeof(filter->vlan_etype),
3499                                           NULL);
3500
3501         /* Add VLAN TCI */
3502         if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI)
3503                 igc_flex_filter_add_field(&flex, &filter->vlan_tci, 14,
3504                                           sizeof(filter->vlan_tci), NULL);
3505
3506         /* Add Ether type */
3507         if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
3508                 __be16 etype = cpu_to_be16(filter->etype);
3509
3510                 igc_flex_filter_add_field(&flex, &etype, eth_offset,
3511                                           sizeof(etype), NULL);
3512         }
3513
3514         /* Add user data */
3515         if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA)
3516                 igc_flex_filter_add_field(&flex, &filter->user_data,
3517                                           user_offset,
3518                                           sizeof(filter->user_data),
3519                                           filter->user_mask);
3520
3521         /* Add it down to the hardware and enable it. */
3522         ret = igc_write_flex_filter_ll(adapter, &flex);
3523         if (ret)
3524                 return ret;
3525
3526         filter->flex_index = index;
3527
3528         return 0;
3529 }
3530
3531 static void igc_del_flex_filter(struct igc_adapter *adapter,
3532                                 u16 reg_index)
3533 {
3534         struct igc_hw *hw = &adapter->hw;
3535         u32 wufc;
3536
3537         /* Just disable the filter. The filter table itself is kept
3538          * intact. Another flex_filter_add() should override the "old" data
3539          * then.
3540          */
3541         if (reg_index > 8) {
3542                 u32 wufc_ext = rd32(IGC_WUFC_EXT);
3543
3544                 wufc_ext &= ~(IGC_WUFC_EXT_FLX8 << (reg_index - 8));
3545                 wr32(IGC_WUFC_EXT, wufc_ext);
3546         } else {
3547                 wufc = rd32(IGC_WUFC);
3548
3549                 wufc &= ~(IGC_WUFC_FLX0 << reg_index);
3550                 wr32(IGC_WUFC, wufc);
3551         }
3552
3553         if (igc_flex_filter_in_use(adapter))
3554                 return;
3555
3556         /* No filters are in use, we may disable flex filters */
3557         wufc = rd32(IGC_WUFC);
3558         wufc &= ~IGC_WUFC_FLEX_HQ;
3559         wr32(IGC_WUFC, wufc);
3560 }
3561
3562 static int igc_enable_nfc_rule(struct igc_adapter *adapter,
3563                                struct igc_nfc_rule *rule)
3564 {
3565         int err;
3566
3567         if (rule->flex) {
3568                 return igc_add_flex_filter(adapter, rule);
3569         }
3570
3571         if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
3572                 err = igc_add_etype_filter(adapter, rule->filter.etype,
3573                                            rule->action);
3574                 if (err)
3575                         return err;
3576         }
3577
3578         if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) {
3579                 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
3580                                          rule->filter.src_addr, rule->action);
3581                 if (err)
3582                         return err;
3583         }
3584
3585         if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) {
3586                 err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
3587                                          rule->filter.dst_addr, rule->action);
3588                 if (err)
3589                         return err;
3590         }
3591
3592         if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
3593                 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
3594                            VLAN_PRIO_SHIFT;
3595
3596                 err = igc_add_vlan_prio_filter(adapter, prio, rule->action);
3597                 if (err)
3598                         return err;
3599         }
3600
3601         return 0;
3602 }
3603
3604 static void igc_disable_nfc_rule(struct igc_adapter *adapter,
3605                                  const struct igc_nfc_rule *rule)
3606 {
3607         if (rule->flex) {
3608                 igc_del_flex_filter(adapter, rule->filter.flex_index);
3609                 return;
3610         }
3611
3612         if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE)
3613                 igc_del_etype_filter(adapter, rule->filter.etype);
3614
3615         if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
3616                 int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
3617                            VLAN_PRIO_SHIFT;
3618
3619                 igc_del_vlan_prio_filter(adapter, prio);
3620         }
3621
3622         if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
3623                 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
3624                                    rule->filter.src_addr);
3625
3626         if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
3627                 igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
3628                                    rule->filter.dst_addr);
3629 }
3630
3631 /**
3632  * igc_get_nfc_rule() - Get NFC rule
3633  * @adapter: Pointer to adapter
3634  * @location: Rule location
3635  *
3636  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3637  *
3638  * Return: Pointer to NFC rule at @location. If not found, NULL.
3639  */
3640 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter,
3641                                       u32 location)
3642 {
3643         struct igc_nfc_rule *rule;
3644
3645         list_for_each_entry(rule, &adapter->nfc_rule_list, list) {
3646                 if (rule->location == location)
3647                         return rule;
3648                 if (rule->location > location)
3649                         break;
3650         }
3651
3652         return NULL;
3653 }
3654
3655 /**
3656  * igc_del_nfc_rule() - Delete NFC rule
3657  * @adapter: Pointer to adapter
3658  * @rule: Pointer to rule to be deleted
3659  *
3660  * Disable NFC rule in hardware and delete it from adapter.
3661  *
3662  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3663  */
3664 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
3665 {
3666         igc_disable_nfc_rule(adapter, rule);
3667
3668         list_del(&rule->list);
3669         adapter->nfc_rule_count--;
3670
3671         kfree(rule);
3672 }
3673
3674 static void igc_flush_nfc_rules(struct igc_adapter *adapter)
3675 {
3676         struct igc_nfc_rule *rule, *tmp;
3677
3678         mutex_lock(&adapter->nfc_rule_lock);
3679
3680         list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list)
3681                 igc_del_nfc_rule(adapter, rule);
3682
3683         mutex_unlock(&adapter->nfc_rule_lock);
3684 }
3685
3686 /**
3687  * igc_add_nfc_rule() - Add NFC rule
3688  * @adapter: Pointer to adapter
3689  * @rule: Pointer to rule to be added
3690  *
3691  * Enable NFC rule in hardware and add it to adapter.
3692  *
3693  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3694  *
3695  * Return: 0 on success, negative errno on failure.
3696  */
3697 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
3698 {
3699         struct igc_nfc_rule *pred, *cur;
3700         int err;
3701
3702         err = igc_enable_nfc_rule(adapter, rule);
3703         if (err)
3704                 return err;
3705
3706         pred = NULL;
3707         list_for_each_entry(cur, &adapter->nfc_rule_list, list) {
3708                 if (cur->location >= rule->location)
3709                         break;
3710                 pred = cur;
3711         }
3712
3713         list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list);
3714         adapter->nfc_rule_count++;
3715         return 0;
3716 }
3717
3718 static void igc_restore_nfc_rules(struct igc_adapter *adapter)
3719 {
3720         struct igc_nfc_rule *rule;
3721
3722         mutex_lock(&adapter->nfc_rule_lock);
3723
3724         list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list)
3725                 igc_enable_nfc_rule(adapter, rule);
3726
3727         mutex_unlock(&adapter->nfc_rule_lock);
3728 }
3729
3730 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr)
3731 {
3732         struct igc_adapter *adapter = netdev_priv(netdev);
3733
3734         return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1);
3735 }
3736
3737 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr)
3738 {
3739         struct igc_adapter *adapter = netdev_priv(netdev);
3740
3741         igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr);
3742         return 0;
3743 }
3744
3745 /**
3746  * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3747  * @netdev: network interface device structure
3748  *
3749  * The set_rx_mode entry point is called whenever the unicast or multicast
3750  * address lists or the network interface flags are updated.  This routine is
3751  * responsible for configuring the hardware for proper unicast, multicast,
3752  * promiscuous mode, and all-multi behavior.
3753  */
3754 static void igc_set_rx_mode(struct net_device *netdev)
3755 {
3756         struct igc_adapter *adapter = netdev_priv(netdev);
3757         struct igc_hw *hw = &adapter->hw;
3758         u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
3759         int count;
3760
3761         /* Check for Promiscuous and All Multicast modes */
3762         if (netdev->flags & IFF_PROMISC) {
3763                 rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE;
3764         } else {
3765                 if (netdev->flags & IFF_ALLMULTI) {
3766                         rctl |= IGC_RCTL_MPE;
3767                 } else {
3768                         /* Write addresses to the MTA, if the attempt fails
3769                          * then we should just turn on promiscuous mode so
3770                          * that we can at least receive multicast traffic
3771                          */
3772                         count = igc_write_mc_addr_list(netdev);
3773                         if (count < 0)
3774                                 rctl |= IGC_RCTL_MPE;
3775                 }
3776         }
3777
3778         /* Write addresses to available RAR registers, if there is not
3779          * sufficient space to store all the addresses then enable
3780          * unicast promiscuous mode
3781          */
3782         if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync))
3783                 rctl |= IGC_RCTL_UPE;
3784
3785         /* update state of unicast and multicast */
3786         rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE);
3787         wr32(IGC_RCTL, rctl);
3788
3789 #if (PAGE_SIZE < 8192)
3790         if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB)
3791                 rlpml = IGC_MAX_FRAME_BUILD_SKB;
3792 #endif
3793         wr32(IGC_RLPML, rlpml);
3794 }
3795
3796 /**
3797  * igc_configure - configure the hardware for RX and TX
3798  * @adapter: private board structure
3799  */
3800 static void igc_configure(struct igc_adapter *adapter)
3801 {
3802         struct net_device *netdev = adapter->netdev;
3803         int i = 0;
3804
3805         igc_get_hw_control(adapter);
3806         igc_set_rx_mode(netdev);
3807
3808         igc_restore_vlan(adapter);
3809
3810         igc_setup_tctl(adapter);
3811         igc_setup_mrqc(adapter);
3812         igc_setup_rctl(adapter);
3813
3814         igc_set_default_mac_filter(adapter);
3815         igc_restore_nfc_rules(adapter);
3816
3817         igc_configure_tx(adapter);
3818         igc_configure_rx(adapter);
3819
3820         igc_rx_fifo_flush_base(&adapter->hw);
3821
3822         /* call igc_desc_unused which always leaves
3823          * at least 1 descriptor unused to make sure
3824          * next_to_use != next_to_clean
3825          */
3826         for (i = 0; i < adapter->num_rx_queues; i++) {
3827                 struct igc_ring *ring = adapter->rx_ring[i];
3828
3829                 if (ring->xsk_pool)
3830                         igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
3831                 else
3832                         igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
3833         }
3834 }
3835
3836 /**
3837  * igc_write_ivar - configure ivar for given MSI-X vector
3838  * @hw: pointer to the HW structure
3839  * @msix_vector: vector number we are allocating to a given ring
3840  * @index: row index of IVAR register to write within IVAR table
3841  * @offset: column offset of in IVAR, should be multiple of 8
3842  *
3843  * The IVAR table consists of 2 columns,
3844  * each containing an cause allocation for an Rx and Tx ring, and a
3845  * variable number of rows depending on the number of queues supported.
3846  */
3847 static void igc_write_ivar(struct igc_hw *hw, int msix_vector,
3848                            int index, int offset)
3849 {
3850         u32 ivar = array_rd32(IGC_IVAR0, index);
3851
3852         /* clear any bits that are currently set */
3853         ivar &= ~((u32)0xFF << offset);
3854
3855         /* write vector and valid bit */
3856         ivar |= (msix_vector | IGC_IVAR_VALID) << offset;
3857
3858         array_wr32(IGC_IVAR0, index, ivar);
3859 }
3860
3861 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)
3862 {
3863         struct igc_adapter *adapter = q_vector->adapter;
3864         struct igc_hw *hw = &adapter->hw;
3865         int rx_queue = IGC_N0_QUEUE;
3866         int tx_queue = IGC_N0_QUEUE;
3867
3868         if (q_vector->rx.ring)
3869                 rx_queue = q_vector->rx.ring->reg_idx;
3870         if (q_vector->tx.ring)
3871                 tx_queue = q_vector->tx.ring->reg_idx;
3872
3873         switch (hw->mac.type) {
3874         case igc_i225:
3875                 if (rx_queue > IGC_N0_QUEUE)
3876                         igc_write_ivar(hw, msix_vector,
3877                                        rx_queue >> 1,
3878                                        (rx_queue & 0x1) << 4);
3879                 if (tx_queue > IGC_N0_QUEUE)
3880                         igc_write_ivar(hw, msix_vector,
3881                                        tx_queue >> 1,
3882                                        ((tx_queue & 0x1) << 4) + 8);
3883                 q_vector->eims_value = BIT(msix_vector);
3884                 break;
3885         default:
3886                 WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n");
3887                 break;
3888         }
3889
3890         /* add q_vector eims value to global eims_enable_mask */
3891         adapter->eims_enable_mask |= q_vector->eims_value;
3892
3893         /* configure q_vector to set itr on first interrupt */
3894         q_vector->set_itr = 1;
3895 }
3896
3897 /**
3898  * igc_configure_msix - Configure MSI-X hardware
3899  * @adapter: Pointer to adapter structure
3900  *
3901  * igc_configure_msix sets up the hardware to properly
3902  * generate MSI-X interrupts.
3903  */
3904 static void igc_configure_msix(struct igc_adapter *adapter)
3905 {
3906         struct igc_hw *hw = &adapter->hw;
3907         int i, vector = 0;
3908         u32 tmp;
3909
3910         adapter->eims_enable_mask = 0;
3911
3912         /* set vector for other causes, i.e. link changes */
3913         switch (hw->mac.type) {
3914         case igc_i225:
3915                 /* Turn on MSI-X capability first, or our settings
3916                  * won't stick.  And it will take days to debug.
3917                  */
3918                 wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE |
3919                      IGC_GPIE_PBA | IGC_GPIE_EIAME |
3920                      IGC_GPIE_NSICR);
3921
3922                 /* enable msix_other interrupt */
3923                 adapter->eims_other = BIT(vector);
3924                 tmp = (vector++ | IGC_IVAR_VALID) << 8;
3925
3926                 wr32(IGC_IVAR_MISC, tmp);
3927                 break;
3928         default:
3929                 /* do nothing, since nothing else supports MSI-X */
3930                 break;
3931         } /* switch (hw->mac.type) */
3932
3933         adapter->eims_enable_mask |= adapter->eims_other;
3934
3935         for (i = 0; i < adapter->num_q_vectors; i++)
3936                 igc_assign_vector(adapter->q_vector[i], vector++);
3937
3938         wrfl();
3939 }
3940
3941 /**
3942  * igc_irq_enable - Enable default interrupt generation settings
3943  * @adapter: board private structure
3944  */
3945 static void igc_irq_enable(struct igc_adapter *adapter)
3946 {
3947         struct igc_hw *hw = &adapter->hw;
3948
3949         if (adapter->msix_entries) {
3950                 u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA;
3951                 u32 regval = rd32(IGC_EIAC);
3952
3953                 wr32(IGC_EIAC, regval | adapter->eims_enable_mask);
3954                 regval = rd32(IGC_EIAM);
3955                 wr32(IGC_EIAM, regval | adapter->eims_enable_mask);
3956                 wr32(IGC_EIMS, adapter->eims_enable_mask);
3957                 wr32(IGC_IMS, ims);
3958         } else {
3959                 wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3960                 wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3961         }
3962 }
3963
3964 /**
3965  * igc_irq_disable - Mask off interrupt generation on the NIC
3966  * @adapter: board private structure
3967  */
3968 static void igc_irq_disable(struct igc_adapter *adapter)
3969 {
3970         struct igc_hw *hw = &adapter->hw;
3971
3972         if (adapter->msix_entries) {
3973                 u32 regval = rd32(IGC_EIAM);
3974
3975                 wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask);
3976                 wr32(IGC_EIMC, adapter->eims_enable_mask);
3977                 regval = rd32(IGC_EIAC);
3978                 wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask);
3979         }
3980
3981         wr32(IGC_IAM, 0);
3982         wr32(IGC_IMC, ~0);
3983         wrfl();
3984
3985         if (adapter->msix_entries) {
3986                 int vector = 0, i;
3987
3988                 synchronize_irq(adapter->msix_entries[vector++].vector);
3989
3990                 for (i = 0; i < adapter->num_q_vectors; i++)
3991                         synchronize_irq(adapter->msix_entries[vector++].vector);
3992         } else {
3993                 synchronize_irq(adapter->pdev->irq);
3994         }
3995 }
3996
3997 void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
3998                               const u32 max_rss_queues)
3999 {
4000         /* Determine if we need to pair queues. */
4001         /* If rss_queues > half of max_rss_queues, pair the queues in
4002          * order to conserve interrupts due to limited supply.
4003          */
4004         if (adapter->rss_queues > (max_rss_queues / 2))
4005                 adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4006         else
4007                 adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
4008 }
4009
4010 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
4011 {
4012         return IGC_MAX_RX_QUEUES;
4013 }
4014
4015 static void igc_init_queue_configuration(struct igc_adapter *adapter)
4016 {
4017         u32 max_rss_queues;
4018
4019         max_rss_queues = igc_get_max_rss_queues(adapter);
4020         adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
4021
4022         igc_set_flag_queue_pairs(adapter, max_rss_queues);
4023 }
4024
4025 /**
4026  * igc_reset_q_vector - Reset config for interrupt vector
4027  * @adapter: board private structure to initialize
4028  * @v_idx: Index of vector to be reset
4029  *
4030  * If NAPI is enabled it will delete any references to the
4031  * NAPI struct. This is preparation for igc_free_q_vector.
4032  */
4033 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx)
4034 {
4035         struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
4036
4037         /* if we're coming from igc_set_interrupt_capability, the vectors are
4038          * not yet allocated
4039          */
4040         if (!q_vector)
4041                 return;
4042
4043         if (q_vector->tx.ring)
4044                 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
4045
4046         if (q_vector->rx.ring)
4047                 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
4048
4049         netif_napi_del(&q_vector->napi);
4050 }
4051
4052 /**
4053  * igc_free_q_vector - Free memory allocated for specific interrupt vector
4054  * @adapter: board private structure to initialize
4055  * @v_idx: Index of vector to be freed
4056  *
4057  * This function frees the memory allocated to the q_vector.
4058  */
4059 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
4060 {
4061         struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
4062
4063         adapter->q_vector[v_idx] = NULL;
4064
4065         /* igc_get_stats64() might access the rings on this vector,
4066          * we must wait a grace period before freeing it.
4067          */
4068         if (q_vector)
4069                 kfree_rcu(q_vector, rcu);
4070 }
4071
4072 /**
4073  * igc_free_q_vectors - Free memory allocated for interrupt vectors
4074  * @adapter: board private structure to initialize
4075  *
4076  * This function frees the memory allocated to the q_vectors.  In addition if
4077  * NAPI is enabled it will delete any references to the NAPI struct prior
4078  * to freeing the q_vector.
4079  */
4080 static void igc_free_q_vectors(struct igc_adapter *adapter)
4081 {
4082         int v_idx = adapter->num_q_vectors;
4083
4084         adapter->num_tx_queues = 0;
4085         adapter->num_rx_queues = 0;
4086         adapter->num_q_vectors = 0;
4087
4088         while (v_idx--) {
4089                 igc_reset_q_vector(adapter, v_idx);
4090                 igc_free_q_vector(adapter, v_idx);
4091         }
4092 }
4093
4094 /**
4095  * igc_update_itr - update the dynamic ITR value based on statistics
4096  * @q_vector: pointer to q_vector
4097  * @ring_container: ring info to update the itr for
4098  *
4099  * Stores a new ITR value based on packets and byte
4100  * counts during the last interrupt.  The advantage of per interrupt
4101  * computation is faster updates and more accurate ITR for the current
4102  * traffic pattern.  Constants in this function were computed
4103  * based on theoretical maximum wire speed and thresholds were set based
4104  * on testing data as well as attempting to minimize response time
4105  * while increasing bulk throughput.
4106  * NOTE: These calculations are only valid when operating in a single-
4107  * queue environment.
4108  */
4109 static void igc_update_itr(struct igc_q_vector *q_vector,
4110                            struct igc_ring_container *ring_container)
4111 {
4112         unsigned int packets = ring_container->total_packets;
4113         unsigned int bytes = ring_container->total_bytes;
4114         u8 itrval = ring_container->itr;
4115
4116         /* no packets, exit with status unchanged */
4117         if (packets == 0)
4118                 return;
4119
4120         switch (itrval) {
4121         case lowest_latency:
4122                 /* handle TSO and jumbo frames */
4123                 if (bytes / packets > 8000)
4124                         itrval = bulk_latency;
4125                 else if ((packets < 5) && (bytes > 512))
4126                         itrval = low_latency;
4127                 break;
4128         case low_latency:  /* 50 usec aka 20000 ints/s */
4129                 if (bytes > 10000) {
4130                         /* this if handles the TSO accounting */
4131                         if (bytes / packets > 8000)
4132                                 itrval = bulk_latency;
4133                         else if ((packets < 10) || ((bytes / packets) > 1200))
4134                                 itrval = bulk_latency;
4135                         else if ((packets > 35))
4136                                 itrval = lowest_latency;
4137                 } else if (bytes / packets > 2000) {
4138                         itrval = bulk_latency;
4139                 } else if (packets <= 2 && bytes < 512) {
4140                         itrval = lowest_latency;
4141                 }
4142                 break;
4143         case bulk_latency: /* 250 usec aka 4000 ints/s */
4144                 if (bytes > 25000) {
4145                         if (packets > 35)
4146                                 itrval = low_latency;
4147                 } else if (bytes < 1500) {
4148                         itrval = low_latency;
4149                 }
4150                 break;
4151         }
4152
4153         /* clear work counters since we have the values we need */
4154         ring_container->total_bytes = 0;
4155         ring_container->total_packets = 0;
4156
4157         /* write updated itr to ring container */
4158         ring_container->itr = itrval;
4159 }
4160
4161 static void igc_set_itr(struct igc_q_vector *q_vector)
4162 {
4163         struct igc_adapter *adapter = q_vector->adapter;
4164         u32 new_itr = q_vector->itr_val;
4165         u8 current_itr = 0;
4166
4167         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4168         switch (adapter->link_speed) {
4169         case SPEED_10:
4170         case SPEED_100:
4171                 current_itr = 0;
4172                 new_itr = IGC_4K_ITR;
4173                 goto set_itr_now;
4174         default:
4175                 break;
4176         }
4177
4178         igc_update_itr(q_vector, &q_vector->tx);
4179         igc_update_itr(q_vector, &q_vector->rx);
4180
4181         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4182
4183         /* conservative mode (itr 3) eliminates the lowest_latency setting */
4184         if (current_itr == lowest_latency &&
4185             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4186             (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4187                 current_itr = low_latency;
4188
4189         switch (current_itr) {
4190         /* counts and packets in update_itr are dependent on these numbers */
4191         case lowest_latency:
4192                 new_itr = IGC_70K_ITR; /* 70,000 ints/sec */
4193                 break;
4194         case low_latency:
4195                 new_itr = IGC_20K_ITR; /* 20,000 ints/sec */
4196                 break;
4197         case bulk_latency:
4198                 new_itr = IGC_4K_ITR;  /* 4,000 ints/sec */
4199                 break;
4200         default:
4201                 break;
4202         }
4203
4204 set_itr_now:
4205         if (new_itr != q_vector->itr_val) {
4206                 /* this attempts to bias the interrupt rate towards Bulk
4207                  * by adding intermediate steps when interrupt rate is
4208                  * increasing
4209                  */
4210                 new_itr = new_itr > q_vector->itr_val ?
4211                           max((new_itr * q_vector->itr_val) /
4212                           (new_itr + (q_vector->itr_val >> 2)),
4213                           new_itr) : new_itr;
4214                 /* Don't write the value here; it resets the adapter's
4215                  * internal timer, and causes us to delay far longer than
4216                  * we should between interrupts.  Instead, we write the ITR
4217                  * value at the beginning of the next interrupt so the timing
4218                  * ends up being correct.
4219                  */
4220                 q_vector->itr_val = new_itr;
4221                 q_vector->set_itr = 1;
4222         }
4223 }
4224
4225 static void igc_reset_interrupt_capability(struct igc_adapter *adapter)
4226 {
4227         int v_idx = adapter->num_q_vectors;
4228
4229         if (adapter->msix_entries) {
4230                 pci_disable_msix(adapter->pdev);
4231                 kfree(adapter->msix_entries);
4232                 adapter->msix_entries = NULL;
4233         } else if (adapter->flags & IGC_FLAG_HAS_MSI) {
4234                 pci_disable_msi(adapter->pdev);
4235         }
4236
4237         while (v_idx--)
4238                 igc_reset_q_vector(adapter, v_idx);
4239 }
4240
4241 /**
4242  * igc_set_interrupt_capability - set MSI or MSI-X if supported
4243  * @adapter: Pointer to adapter structure
4244  * @msix: boolean value for MSI-X capability
4245  *
4246  * Attempt to configure interrupts using the best available
4247  * capabilities of the hardware and kernel.
4248  */
4249 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
4250                                          bool msix)
4251 {
4252         int numvecs, i;
4253         int err;
4254
4255         if (!msix)
4256                 goto msi_only;
4257         adapter->flags |= IGC_FLAG_HAS_MSIX;
4258
4259         /* Number of supported queues. */
4260         adapter->num_rx_queues = adapter->rss_queues;
4261
4262         adapter->num_tx_queues = adapter->rss_queues;
4263
4264         /* start with one vector for every Rx queue */
4265         numvecs = adapter->num_rx_queues;
4266
4267         /* if Tx handler is separate add 1 for every Tx queue */
4268         if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS))
4269                 numvecs += adapter->num_tx_queues;
4270
4271         /* store the number of vectors reserved for queues */
4272         adapter->num_q_vectors = numvecs;
4273
4274         /* add 1 vector for link status interrupts */
4275         numvecs++;
4276
4277         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
4278                                         GFP_KERNEL);
4279
4280         if (!adapter->msix_entries)
4281                 return;
4282
4283         /* populate entry values */
4284         for (i = 0; i < numvecs; i++)
4285                 adapter->msix_entries[i].entry = i;
4286
4287         err = pci_enable_msix_range(adapter->pdev,
4288                                     adapter->msix_entries,
4289                                     numvecs,
4290                                     numvecs);
4291         if (err > 0)
4292                 return;
4293
4294         kfree(adapter->msix_entries);
4295         adapter->msix_entries = NULL;
4296
4297         igc_reset_interrupt_capability(adapter);
4298
4299 msi_only:
4300         adapter->flags &= ~IGC_FLAG_HAS_MSIX;
4301
4302         adapter->rss_queues = 1;
4303         adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4304         adapter->num_rx_queues = 1;
4305         adapter->num_tx_queues = 1;
4306         adapter->num_q_vectors = 1;
4307         if (!pci_enable_msi(adapter->pdev))
4308                 adapter->flags |= IGC_FLAG_HAS_MSI;
4309 }
4310
4311 /**
4312  * igc_update_ring_itr - update the dynamic ITR value based on packet size
4313  * @q_vector: pointer to q_vector
4314  *
4315  * Stores a new ITR value based on strictly on packet size.  This
4316  * algorithm is less sophisticated than that used in igc_update_itr,
4317  * due to the difficulty of synchronizing statistics across multiple
4318  * receive rings.  The divisors and thresholds used by this function
4319  * were determined based on theoretical maximum wire speed and testing
4320  * data, in order to minimize response time while increasing bulk
4321  * throughput.
4322  * NOTE: This function is called only when operating in a multiqueue
4323  * receive environment.
4324  */
4325 static void igc_update_ring_itr(struct igc_q_vector *q_vector)
4326 {
4327         struct igc_adapter *adapter = q_vector->adapter;
4328         int new_val = q_vector->itr_val;
4329         int avg_wire_size = 0;
4330         unsigned int packets;
4331
4332         /* For non-gigabit speeds, just fix the interrupt rate at 4000
4333          * ints/sec - ITR timer value of 120 ticks.
4334          */
4335         switch (adapter->link_speed) {
4336         case SPEED_10:
4337         case SPEED_100:
4338                 new_val = IGC_4K_ITR;
4339                 goto set_itr_val;
4340         default:
4341                 break;
4342         }
4343
4344         packets = q_vector->rx.total_packets;
4345         if (packets)
4346                 avg_wire_size = q_vector->rx.total_bytes / packets;
4347
4348         packets = q_vector->tx.total_packets;
4349         if (packets)
4350                 avg_wire_size = max_t(u32, avg_wire_size,
4351                                       q_vector->tx.total_bytes / packets);
4352
4353         /* if avg_wire_size isn't set no work was done */
4354         if (!avg_wire_size)
4355                 goto clear_counts;
4356
4357         /* Add 24 bytes to size to account for CRC, preamble, and gap */
4358         avg_wire_size += 24;
4359
4360         /* Don't starve jumbo frames */
4361         avg_wire_size = min(avg_wire_size, 3000);
4362
4363         /* Give a little boost to mid-size frames */
4364         if (avg_wire_size > 300 && avg_wire_size < 1200)
4365                 new_val = avg_wire_size / 3;
4366         else
4367                 new_val = avg_wire_size / 2;
4368
4369         /* conservative mode (itr 3) eliminates the lowest_latency setting */
4370         if (new_val < IGC_20K_ITR &&
4371             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4372             (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4373                 new_val = IGC_20K_ITR;
4374
4375 set_itr_val:
4376         if (new_val != q_vector->itr_val) {
4377                 q_vector->itr_val = new_val;
4378                 q_vector->set_itr = 1;
4379         }
4380 clear_counts:
4381         q_vector->rx.total_bytes = 0;
4382         q_vector->rx.total_packets = 0;
4383         q_vector->tx.total_bytes = 0;
4384         q_vector->tx.total_packets = 0;
4385 }
4386
4387 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)
4388 {
4389         struct igc_adapter *adapter = q_vector->adapter;
4390         struct igc_hw *hw = &adapter->hw;
4391
4392         if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
4393             (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
4394                 if (adapter->num_q_vectors == 1)
4395                         igc_set_itr(q_vector);
4396                 else
4397                         igc_update_ring_itr(q_vector);
4398         }
4399
4400         if (!test_bit(__IGC_DOWN, &adapter->state)) {
4401                 if (adapter->msix_entries)
4402                         wr32(IGC_EIMS, q_vector->eims_value);
4403                 else
4404                         igc_irq_enable(adapter);
4405         }
4406 }
4407
4408 static void igc_add_ring(struct igc_ring *ring,
4409                          struct igc_ring_container *head)
4410 {
4411         head->ring = ring;
4412         head->count++;
4413 }
4414
4415 /**
4416  * igc_cache_ring_register - Descriptor ring to register mapping
4417  * @adapter: board private structure to initialize
4418  *
4419  * Once we know the feature-set enabled for the device, we'll cache
4420  * the register offset the descriptor ring is assigned to.
4421  */
4422 static void igc_cache_ring_register(struct igc_adapter *adapter)
4423 {
4424         int i = 0, j = 0;
4425
4426         switch (adapter->hw.mac.type) {
4427         case igc_i225:
4428         default:
4429                 for (; i < adapter->num_rx_queues; i++)
4430                         adapter->rx_ring[i]->reg_idx = i;
4431                 for (; j < adapter->num_tx_queues; j++)
4432                         adapter->tx_ring[j]->reg_idx = j;
4433                 break;
4434         }
4435 }
4436
4437 /**
4438  * igc_poll - NAPI Rx polling callback
4439  * @napi: napi polling structure
4440  * @budget: count of how many packets we should handle
4441  */
4442 static int igc_poll(struct napi_struct *napi, int budget)
4443 {
4444         struct igc_q_vector *q_vector = container_of(napi,
4445                                                      struct igc_q_vector,
4446                                                      napi);
4447         struct igc_ring *rx_ring = q_vector->rx.ring;
4448         bool clean_complete = true;
4449         int work_done = 0;
4450
4451         if (q_vector->tx.ring)
4452                 clean_complete = igc_clean_tx_irq(q_vector, budget);
4453
4454         if (rx_ring) {
4455                 int cleaned = rx_ring->xsk_pool ?
4456                               igc_clean_rx_irq_zc(q_vector, budget) :
4457                               igc_clean_rx_irq(q_vector, budget);
4458
4459                 work_done += cleaned;
4460                 if (cleaned >= budget)
4461                         clean_complete = false;
4462         }
4463
4464         /* If all work not completed, return budget and keep polling */
4465         if (!clean_complete)
4466                 return budget;
4467
4468         /* Exit the polling mode, but don't re-enable interrupts if stack might
4469          * poll us due to busy-polling
4470          */
4471         if (likely(napi_complete_done(napi, work_done)))
4472                 igc_ring_irq_enable(q_vector);
4473
4474         return min(work_done, budget - 1);
4475 }
4476
4477 /**
4478  * igc_alloc_q_vector - Allocate memory for a single interrupt vector
4479  * @adapter: board private structure to initialize
4480  * @v_count: q_vectors allocated on adapter, used for ring interleaving
4481  * @v_idx: index of vector in adapter struct
4482  * @txr_count: total number of Tx rings to allocate
4483  * @txr_idx: index of first Tx ring to allocate
4484  * @rxr_count: total number of Rx rings to allocate
4485  * @rxr_idx: index of first Rx ring to allocate
4486  *
4487  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
4488  */
4489 static int igc_alloc_q_vector(struct igc_adapter *adapter,
4490                               unsigned int v_count, unsigned int v_idx,
4491                               unsigned int txr_count, unsigned int txr_idx,
4492                               unsigned int rxr_count, unsigned int rxr_idx)
4493 {
4494         struct igc_q_vector *q_vector;
4495         struct igc_ring *ring;
4496         int ring_count;
4497
4498         /* igc only supports 1 Tx and/or 1 Rx queue per vector */
4499         if (txr_count > 1 || rxr_count > 1)
4500                 return -ENOMEM;
4501
4502         ring_count = txr_count + rxr_count;
4503
4504         /* allocate q_vector and rings */
4505         q_vector = adapter->q_vector[v_idx];
4506         if (!q_vector)
4507                 q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
4508                                    GFP_KERNEL);
4509         else
4510                 memset(q_vector, 0, struct_size(q_vector, ring, ring_count));
4511         if (!q_vector)
4512                 return -ENOMEM;
4513
4514         /* initialize NAPI */
4515         netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll);
4516
4517         /* tie q_vector and adapter together */
4518         adapter->q_vector[v_idx] = q_vector;
4519         q_vector->adapter = adapter;
4520
4521         /* initialize work limits */
4522         q_vector->tx.work_limit = adapter->tx_work_limit;
4523
4524         /* initialize ITR configuration */
4525         q_vector->itr_register = adapter->io_addr + IGC_EITR(0);
4526         q_vector->itr_val = IGC_START_ITR;
4527
4528         /* initialize pointer to rings */
4529         ring = q_vector->ring;
4530
4531         /* initialize ITR */
4532         if (rxr_count) {
4533                 /* rx or rx/tx vector */
4534                 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
4535                         q_vector->itr_val = adapter->rx_itr_setting;
4536         } else {
4537                 /* tx only vector */
4538                 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
4539                         q_vector->itr_val = adapter->tx_itr_setting;
4540         }
4541
4542         if (txr_count) {
4543                 /* assign generic ring traits */
4544                 ring->dev = &adapter->pdev->dev;
4545                 ring->netdev = adapter->netdev;
4546
4547                 /* configure backlink on ring */
4548                 ring->q_vector = q_vector;
4549
4550                 /* update q_vector Tx values */
4551                 igc_add_ring(ring, &q_vector->tx);
4552
4553                 /* apply Tx specific ring traits */
4554                 ring->count = adapter->tx_ring_count;
4555                 ring->queue_index = txr_idx;
4556
4557                 /* assign ring to adapter */
4558                 adapter->tx_ring[txr_idx] = ring;
4559
4560                 /* push pointer to next ring */
4561                 ring++;
4562         }
4563
4564         if (rxr_count) {
4565                 /* assign generic ring traits */
4566                 ring->dev = &adapter->pdev->dev;
4567                 ring->netdev = adapter->netdev;
4568
4569                 /* configure backlink on ring */
4570                 ring->q_vector = q_vector;
4571
4572                 /* update q_vector Rx values */
4573                 igc_add_ring(ring, &q_vector->rx);
4574
4575                 /* apply Rx specific ring traits */
4576                 ring->count = adapter->rx_ring_count;
4577                 ring->queue_index = rxr_idx;
4578
4579                 /* assign ring to adapter */
4580                 adapter->rx_ring[rxr_idx] = ring;
4581         }
4582
4583         return 0;
4584 }
4585
4586 /**
4587  * igc_alloc_q_vectors - Allocate memory for interrupt vectors
4588  * @adapter: board private structure to initialize
4589  *
4590  * We allocate one q_vector per queue interrupt.  If allocation fails we
4591  * return -ENOMEM.
4592  */
4593 static int igc_alloc_q_vectors(struct igc_adapter *adapter)
4594 {
4595         int rxr_remaining = adapter->num_rx_queues;
4596         int txr_remaining = adapter->num_tx_queues;
4597         int rxr_idx = 0, txr_idx = 0, v_idx = 0;
4598         int q_vectors = adapter->num_q_vectors;
4599         int err;
4600
4601         if (q_vectors >= (rxr_remaining + txr_remaining)) {
4602                 for (; rxr_remaining; v_idx++) {
4603                         err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
4604                                                  0, 0, 1, rxr_idx);
4605
4606                         if (err)
4607                                 goto err_out;
4608
4609                         /* update counts and index */
4610                         rxr_remaining--;
4611                         rxr_idx++;
4612                 }
4613         }
4614
4615         for (; v_idx < q_vectors; v_idx++) {
4616                 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
4617                 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
4618
4619                 err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
4620                                          tqpv, txr_idx, rqpv, rxr_idx);
4621
4622                 if (err)
4623                         goto err_out;
4624
4625                 /* update counts and index */
4626                 rxr_remaining -= rqpv;
4627                 txr_remaining -= tqpv;
4628                 rxr_idx++;
4629                 txr_idx++;
4630         }
4631
4632         return 0;
4633
4634 err_out:
4635         adapter->num_tx_queues = 0;
4636         adapter->num_rx_queues = 0;
4637         adapter->num_q_vectors = 0;
4638
4639         while (v_idx--)
4640                 igc_free_q_vector(adapter, v_idx);
4641
4642         return -ENOMEM;
4643 }
4644
4645 /**
4646  * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
4647  * @adapter: Pointer to adapter structure
4648  * @msix: boolean for MSI-X capability
4649  *
4650  * This function initializes the interrupts and allocates all of the queues.
4651  */
4652 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix)
4653 {
4654         struct net_device *dev = adapter->netdev;
4655         int err = 0;
4656
4657         igc_set_interrupt_capability(adapter, msix);
4658
4659         err = igc_alloc_q_vectors(adapter);
4660         if (err) {
4661                 netdev_err(dev, "Unable to allocate memory for vectors\n");
4662                 goto err_alloc_q_vectors;
4663         }
4664
4665         igc_cache_ring_register(adapter);
4666
4667         return 0;
4668
4669 err_alloc_q_vectors:
4670         igc_reset_interrupt_capability(adapter);
4671         return err;
4672 }
4673
4674 /**
4675  * igc_sw_init - Initialize general software structures (struct igc_adapter)
4676  * @adapter: board private structure to initialize
4677  *
4678  * igc_sw_init initializes the Adapter private data structure.
4679  * Fields are initialized based on PCI device information and
4680  * OS network device settings (MTU size).
4681  */
4682 static int igc_sw_init(struct igc_adapter *adapter)
4683 {
4684         struct net_device *netdev = adapter->netdev;
4685         struct pci_dev *pdev = adapter->pdev;
4686         struct igc_hw *hw = &adapter->hw;
4687
4688         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4689
4690         /* set default ring sizes */
4691         adapter->tx_ring_count = IGC_DEFAULT_TXD;
4692         adapter->rx_ring_count = IGC_DEFAULT_RXD;
4693
4694         /* set default ITR values */
4695         adapter->rx_itr_setting = IGC_DEFAULT_ITR;
4696         adapter->tx_itr_setting = IGC_DEFAULT_ITR;
4697
4698         /* set default work limits */
4699         adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
4700
4701         /* adjust max frame to be at least the size of a standard frame */
4702         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
4703                                 VLAN_HLEN;
4704         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4705
4706         mutex_init(&adapter->nfc_rule_lock);
4707         INIT_LIST_HEAD(&adapter->nfc_rule_list);
4708         adapter->nfc_rule_count = 0;
4709
4710         spin_lock_init(&adapter->stats64_lock);
4711         /* Assume MSI-X interrupts, will be checked during IRQ allocation */
4712         adapter->flags |= IGC_FLAG_HAS_MSIX;
4713
4714         igc_init_queue_configuration(adapter);
4715
4716         /* This call may decrease the number of queues */
4717         if (igc_init_interrupt_scheme(adapter, true)) {
4718                 netdev_err(netdev, "Unable to allocate memory for queues\n");
4719                 return -ENOMEM;
4720         }
4721
4722         /* Explicitly disable IRQ since the NIC can be in any state. */
4723         igc_irq_disable(adapter);
4724
4725         set_bit(__IGC_DOWN, &adapter->state);
4726
4727         return 0;
4728 }
4729
4730 /**
4731  * igc_up - Open the interface and prepare it to handle traffic
4732  * @adapter: board private structure
4733  */
4734 void igc_up(struct igc_adapter *adapter)
4735 {
4736         struct igc_hw *hw = &adapter->hw;
4737         int i = 0;
4738
4739         /* hardware has been reset, we need to reload some things */
4740         igc_configure(adapter);
4741
4742         clear_bit(__IGC_DOWN, &adapter->state);
4743
4744         for (i = 0; i < adapter->num_q_vectors; i++)
4745                 napi_enable(&adapter->q_vector[i]->napi);
4746
4747         if (adapter->msix_entries)
4748                 igc_configure_msix(adapter);
4749         else
4750                 igc_assign_vector(adapter->q_vector[0], 0);
4751
4752         /* Clear any pending interrupts. */
4753         rd32(IGC_ICR);
4754         igc_irq_enable(adapter);
4755
4756         netif_tx_start_all_queues(adapter->netdev);
4757
4758         /* start the watchdog. */
4759         hw->mac.get_link_status = true;
4760         schedule_work(&adapter->watchdog_task);
4761 }
4762
4763 /**
4764  * igc_update_stats - Update the board statistics counters
4765  * @adapter: board private structure
4766  */
4767 void igc_update_stats(struct igc_adapter *adapter)
4768 {
4769         struct rtnl_link_stats64 *net_stats = &adapter->stats64;
4770         struct pci_dev *pdev = adapter->pdev;
4771         struct igc_hw *hw = &adapter->hw;
4772         u64 _bytes, _packets;
4773         u64 bytes, packets;
4774         unsigned int start;
4775         u32 mpc;
4776         int i;
4777
4778         /* Prevent stats update while adapter is being reset, or if the pci
4779          * connection is down.
4780          */
4781         if (adapter->link_speed == 0)
4782                 return;
4783         if (pci_channel_offline(pdev))
4784                 return;
4785
4786         packets = 0;
4787         bytes = 0;
4788
4789         rcu_read_lock();
4790         for (i = 0; i < adapter->num_rx_queues; i++) {
4791                 struct igc_ring *ring = adapter->rx_ring[i];
4792                 u32 rqdpc = rd32(IGC_RQDPC(i));
4793
4794                 if (hw->mac.type >= igc_i225)
4795                         wr32(IGC_RQDPC(i), 0);
4796
4797                 if (rqdpc) {
4798                         ring->rx_stats.drops += rqdpc;
4799                         net_stats->rx_fifo_errors += rqdpc;
4800                 }
4801
4802                 do {
4803                         start = u64_stats_fetch_begin(&ring->rx_syncp);
4804                         _bytes = ring->rx_stats.bytes;
4805                         _packets = ring->rx_stats.packets;
4806                 } while (u64_stats_fetch_retry(&ring->rx_syncp, start));
4807                 bytes += _bytes;
4808                 packets += _packets;
4809         }
4810
4811         net_stats->rx_bytes = bytes;
4812         net_stats->rx_packets = packets;
4813
4814         packets = 0;
4815         bytes = 0;
4816         for (i = 0; i < adapter->num_tx_queues; i++) {
4817                 struct igc_ring *ring = adapter->tx_ring[i];
4818
4819                 do {
4820                         start = u64_stats_fetch_begin(&ring->tx_syncp);
4821                         _bytes = ring->tx_stats.bytes;
4822                         _packets = ring->tx_stats.packets;
4823                 } while (u64_stats_fetch_retry(&ring->tx_syncp, start));
4824                 bytes += _bytes;
4825                 packets += _packets;
4826         }
4827         net_stats->tx_bytes = bytes;
4828         net_stats->tx_packets = packets;
4829         rcu_read_unlock();
4830
4831         /* read stats registers */
4832         adapter->stats.crcerrs += rd32(IGC_CRCERRS);
4833         adapter->stats.gprc += rd32(IGC_GPRC);
4834         adapter->stats.gorc += rd32(IGC_GORCL);
4835         rd32(IGC_GORCH); /* clear GORCL */
4836         adapter->stats.bprc += rd32(IGC_BPRC);
4837         adapter->stats.mprc += rd32(IGC_MPRC);
4838         adapter->stats.roc += rd32(IGC_ROC);
4839
4840         adapter->stats.prc64 += rd32(IGC_PRC64);
4841         adapter->stats.prc127 += rd32(IGC_PRC127);
4842         adapter->stats.prc255 += rd32(IGC_PRC255);
4843         adapter->stats.prc511 += rd32(IGC_PRC511);
4844         adapter->stats.prc1023 += rd32(IGC_PRC1023);
4845         adapter->stats.prc1522 += rd32(IGC_PRC1522);
4846         adapter->stats.tlpic += rd32(IGC_TLPIC);
4847         adapter->stats.rlpic += rd32(IGC_RLPIC);
4848         adapter->stats.hgptc += rd32(IGC_HGPTC);
4849
4850         mpc = rd32(IGC_MPC);
4851         adapter->stats.mpc += mpc;
4852         net_stats->rx_fifo_errors += mpc;
4853         adapter->stats.scc += rd32(IGC_SCC);
4854         adapter->stats.ecol += rd32(IGC_ECOL);
4855         adapter->stats.mcc += rd32(IGC_MCC);
4856         adapter->stats.latecol += rd32(IGC_LATECOL);
4857         adapter->stats.dc += rd32(IGC_DC);
4858         adapter->stats.rlec += rd32(IGC_RLEC);
4859         adapter->stats.xonrxc += rd32(IGC_XONRXC);
4860         adapter->stats.xontxc += rd32(IGC_XONTXC);
4861         adapter->stats.xoffrxc += rd32(IGC_XOFFRXC);
4862         adapter->stats.xofftxc += rd32(IGC_XOFFTXC);
4863         adapter->stats.fcruc += rd32(IGC_FCRUC);
4864         adapter->stats.gptc += rd32(IGC_GPTC);
4865         adapter->stats.gotc += rd32(IGC_GOTCL);
4866         rd32(IGC_GOTCH); /* clear GOTCL */
4867         adapter->stats.rnbc += rd32(IGC_RNBC);
4868         adapter->stats.ruc += rd32(IGC_RUC);
4869         adapter->stats.rfc += rd32(IGC_RFC);
4870         adapter->stats.rjc += rd32(IGC_RJC);
4871         adapter->stats.tor += rd32(IGC_TORH);
4872         adapter->stats.tot += rd32(IGC_TOTH);
4873         adapter->stats.tpr += rd32(IGC_TPR);
4874
4875         adapter->stats.ptc64 += rd32(IGC_PTC64);
4876         adapter->stats.ptc127 += rd32(IGC_PTC127);
4877         adapter->stats.ptc255 += rd32(IGC_PTC255);
4878         adapter->stats.ptc511 += rd32(IGC_PTC511);
4879         adapter->stats.ptc1023 += rd32(IGC_PTC1023);
4880         adapter->stats.ptc1522 += rd32(IGC_PTC1522);
4881
4882         adapter->stats.mptc += rd32(IGC_MPTC);
4883         adapter->stats.bptc += rd32(IGC_BPTC);
4884
4885         adapter->stats.tpt += rd32(IGC_TPT);
4886         adapter->stats.colc += rd32(IGC_COLC);
4887         adapter->stats.colc += rd32(IGC_RERC);
4888
4889         adapter->stats.algnerrc += rd32(IGC_ALGNERRC);
4890
4891         adapter->stats.tsctc += rd32(IGC_TSCTC);
4892
4893         adapter->stats.iac += rd32(IGC_IAC);
4894
4895         /* Fill out the OS statistics structure */
4896         net_stats->multicast = adapter->stats.mprc;
4897         net_stats->collisions = adapter->stats.colc;
4898
4899         /* Rx Errors */
4900
4901         /* RLEC on some newer hardware can be incorrect so build
4902          * our own version based on RUC and ROC
4903          */
4904         net_stats->rx_errors = adapter->stats.rxerrc +
4905                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4906                 adapter->stats.ruc + adapter->stats.roc +
4907                 adapter->stats.cexterr;
4908         net_stats->rx_length_errors = adapter->stats.ruc +
4909                                       adapter->stats.roc;
4910         net_stats->rx_crc_errors = adapter->stats.crcerrs;
4911         net_stats->rx_frame_errors = adapter->stats.algnerrc;
4912         net_stats->rx_missed_errors = adapter->stats.mpc;
4913
4914         /* Tx Errors */
4915         net_stats->tx_errors = adapter->stats.ecol +
4916                                adapter->stats.latecol;
4917         net_stats->tx_aborted_errors = adapter->stats.ecol;
4918         net_stats->tx_window_errors = adapter->stats.latecol;
4919         net_stats->tx_carrier_errors = adapter->stats.tncrs;
4920
4921         /* Tx Dropped needs to be maintained elsewhere */
4922
4923         /* Management Stats */
4924         adapter->stats.mgptc += rd32(IGC_MGTPTC);
4925         adapter->stats.mgprc += rd32(IGC_MGTPRC);
4926         adapter->stats.mgpdc += rd32(IGC_MGTPDC);
4927 }
4928
4929 /**
4930  * igc_down - Close the interface
4931  * @adapter: board private structure
4932  */
4933 void igc_down(struct igc_adapter *adapter)
4934 {
4935         struct net_device *netdev = adapter->netdev;
4936         struct igc_hw *hw = &adapter->hw;
4937         u32 tctl, rctl;
4938         int i = 0;
4939
4940         set_bit(__IGC_DOWN, &adapter->state);
4941
4942         igc_ptp_suspend(adapter);
4943
4944         if (pci_device_is_present(adapter->pdev)) {
4945                 /* disable receives in the hardware */
4946                 rctl = rd32(IGC_RCTL);
4947                 wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN);
4948                 /* flush and sleep below */
4949         }
4950         /* set trans_start so we don't get spurious watchdogs during reset */
4951         netif_trans_update(netdev);
4952
4953         netif_carrier_off(netdev);
4954         netif_tx_stop_all_queues(netdev);
4955
4956         if (pci_device_is_present(adapter->pdev)) {
4957                 /* disable transmits in the hardware */
4958                 tctl = rd32(IGC_TCTL);
4959                 tctl &= ~IGC_TCTL_EN;
4960                 wr32(IGC_TCTL, tctl);
4961                 /* flush both disables and wait for them to finish */
4962                 wrfl();
4963                 usleep_range(10000, 20000);
4964
4965                 igc_irq_disable(adapter);
4966         }
4967
4968         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
4969
4970         for (i = 0; i < adapter->num_q_vectors; i++) {
4971                 if (adapter->q_vector[i]) {
4972                         napi_synchronize(&adapter->q_vector[i]->napi);
4973                         napi_disable(&adapter->q_vector[i]->napi);
4974                 }
4975         }
4976
4977         del_timer_sync(&adapter->watchdog_timer);
4978         del_timer_sync(&adapter->phy_info_timer);
4979
4980         /* record the stats before reset*/
4981         spin_lock(&adapter->stats64_lock);
4982         igc_update_stats(adapter);
4983         spin_unlock(&adapter->stats64_lock);
4984
4985         adapter->link_speed = 0;
4986         adapter->link_duplex = 0;
4987
4988         if (!pci_channel_offline(adapter->pdev))
4989                 igc_reset(adapter);
4990
4991         /* clear VLAN promisc flag so VFTA will be updated if necessary */
4992         adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
4993
4994         igc_clean_all_tx_rings(adapter);
4995         igc_clean_all_rx_rings(adapter);
4996 }
4997
4998 void igc_reinit_locked(struct igc_adapter *adapter)
4999 {
5000         while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
5001                 usleep_range(1000, 2000);
5002         igc_down(adapter);
5003         igc_up(adapter);
5004         clear_bit(__IGC_RESETTING, &adapter->state);
5005 }
5006
5007 static void igc_reset_task(struct work_struct *work)
5008 {
5009         struct igc_adapter *adapter;
5010
5011         adapter = container_of(work, struct igc_adapter, reset_task);
5012
5013         rtnl_lock();
5014         /* If we're already down or resetting, just bail */
5015         if (test_bit(__IGC_DOWN, &adapter->state) ||
5016             test_bit(__IGC_RESETTING, &adapter->state)) {
5017                 rtnl_unlock();
5018                 return;
5019         }
5020
5021         igc_rings_dump(adapter);
5022         igc_regs_dump(adapter);
5023         netdev_err(adapter->netdev, "Reset adapter\n");
5024         igc_reinit_locked(adapter);
5025         rtnl_unlock();
5026 }
5027
5028 /**
5029  * igc_change_mtu - Change the Maximum Transfer Unit
5030  * @netdev: network interface device structure
5031  * @new_mtu: new value for maximum frame size
5032  *
5033  * Returns 0 on success, negative on failure
5034  */
5035 static int igc_change_mtu(struct net_device *netdev, int new_mtu)
5036 {
5037         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5038         struct igc_adapter *adapter = netdev_priv(netdev);
5039
5040         if (igc_xdp_is_enabled(adapter) && new_mtu > ETH_DATA_LEN) {
5041                 netdev_dbg(netdev, "Jumbo frames not supported with XDP");
5042                 return -EINVAL;
5043         }
5044
5045         /* adjust max frame to be at least the size of a standard frame */
5046         if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
5047                 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
5048
5049         while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
5050                 usleep_range(1000, 2000);
5051
5052         /* igc_down has a dependency on max_frame_size */
5053         adapter->max_frame_size = max_frame;
5054
5055         if (netif_running(netdev))
5056                 igc_down(adapter);
5057
5058         netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5059         netdev->mtu = new_mtu;
5060
5061         if (netif_running(netdev))
5062                 igc_up(adapter);
5063         else
5064                 igc_reset(adapter);
5065
5066         clear_bit(__IGC_RESETTING, &adapter->state);
5067
5068         return 0;
5069 }
5070
5071 /**
5072  * igc_get_stats64 - Get System Network Statistics
5073  * @netdev: network interface device structure
5074  * @stats: rtnl_link_stats64 pointer
5075  *
5076  * Returns the address of the device statistics structure.
5077  * The statistics are updated here and also from the timer callback.
5078  */
5079 static void igc_get_stats64(struct net_device *netdev,
5080                             struct rtnl_link_stats64 *stats)
5081 {
5082         struct igc_adapter *adapter = netdev_priv(netdev);
5083
5084         spin_lock(&adapter->stats64_lock);
5085         if (!test_bit(__IGC_RESETTING, &adapter->state))
5086                 igc_update_stats(adapter);
5087         memcpy(stats, &adapter->stats64, sizeof(*stats));
5088         spin_unlock(&adapter->stats64_lock);
5089 }
5090
5091 static netdev_features_t igc_fix_features(struct net_device *netdev,
5092                                           netdev_features_t features)
5093 {
5094         /* Since there is no support for separate Rx/Tx vlan accel
5095          * enable/disable make sure Tx flag is always in same state as Rx.
5096          */
5097         if (features & NETIF_F_HW_VLAN_CTAG_RX)
5098                 features |= NETIF_F_HW_VLAN_CTAG_TX;
5099         else
5100                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
5101
5102         return features;
5103 }
5104
5105 static int igc_set_features(struct net_device *netdev,
5106                             netdev_features_t features)
5107 {
5108         netdev_features_t changed = netdev->features ^ features;
5109         struct igc_adapter *adapter = netdev_priv(netdev);
5110
5111         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
5112                 igc_vlan_mode(netdev, features);
5113
5114         /* Add VLAN support */
5115         if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
5116                 return 0;
5117
5118         if (!(features & NETIF_F_NTUPLE))
5119                 igc_flush_nfc_rules(adapter);
5120
5121         netdev->features = features;
5122
5123         if (netif_running(netdev))
5124                 igc_reinit_locked(adapter);
5125         else
5126                 igc_reset(adapter);
5127
5128         return 1;
5129 }
5130
5131 static netdev_features_t
5132 igc_features_check(struct sk_buff *skb, struct net_device *dev,
5133                    netdev_features_t features)
5134 {
5135         unsigned int network_hdr_len, mac_hdr_len;
5136
5137         /* Make certain the headers can be described by a context descriptor */
5138         mac_hdr_len = skb_network_header(skb) - skb->data;
5139         if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN))
5140                 return features & ~(NETIF_F_HW_CSUM |
5141                                     NETIF_F_SCTP_CRC |
5142                                     NETIF_F_HW_VLAN_CTAG_TX |
5143                                     NETIF_F_TSO |
5144                                     NETIF_F_TSO6);
5145
5146         network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
5147         if (unlikely(network_hdr_len >  IGC_MAX_NETWORK_HDR_LEN))
5148                 return features & ~(NETIF_F_HW_CSUM |
5149                                     NETIF_F_SCTP_CRC |
5150                                     NETIF_F_TSO |
5151                                     NETIF_F_TSO6);
5152
5153         /* We can only support IPv4 TSO in tunnels if we can mangle the
5154          * inner IP ID field, so strip TSO if MANGLEID is not supported.
5155          */
5156         if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
5157                 features &= ~NETIF_F_TSO;
5158
5159         return features;
5160 }
5161
5162 static void igc_tsync_interrupt(struct igc_adapter *adapter)
5163 {
5164         u32 ack, tsauxc, sec, nsec, tsicr;
5165         struct igc_hw *hw = &adapter->hw;
5166         struct ptp_clock_event event;
5167         struct timespec64 ts;
5168
5169         tsicr = rd32(IGC_TSICR);
5170         ack = 0;
5171
5172         if (tsicr & IGC_TSICR_SYS_WRAP) {
5173                 event.type = PTP_CLOCK_PPS;
5174                 if (adapter->ptp_caps.pps)
5175                         ptp_clock_event(adapter->ptp_clock, &event);
5176                 ack |= IGC_TSICR_SYS_WRAP;
5177         }
5178
5179         if (tsicr & IGC_TSICR_TXTS) {
5180                 /* retrieve hardware timestamp */
5181                 schedule_work(&adapter->ptp_tx_work);
5182                 ack |= IGC_TSICR_TXTS;
5183         }
5184
5185         if (tsicr & IGC_TSICR_TT0) {
5186                 spin_lock(&adapter->tmreg_lock);
5187                 ts = timespec64_add(adapter->perout[0].start,
5188                                     adapter->perout[0].period);
5189                 wr32(IGC_TRGTTIML0, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
5190                 wr32(IGC_TRGTTIMH0, (u32)ts.tv_sec);
5191                 tsauxc = rd32(IGC_TSAUXC);
5192                 tsauxc |= IGC_TSAUXC_EN_TT0;
5193                 wr32(IGC_TSAUXC, tsauxc);
5194                 adapter->perout[0].start = ts;
5195                 spin_unlock(&adapter->tmreg_lock);
5196                 ack |= IGC_TSICR_TT0;
5197         }
5198
5199         if (tsicr & IGC_TSICR_TT1) {
5200                 spin_lock(&adapter->tmreg_lock);
5201                 ts = timespec64_add(adapter->perout[1].start,
5202                                     adapter->perout[1].period);
5203                 wr32(IGC_TRGTTIML1, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
5204                 wr32(IGC_TRGTTIMH1, (u32)ts.tv_sec);
5205                 tsauxc = rd32(IGC_TSAUXC);
5206                 tsauxc |= IGC_TSAUXC_EN_TT1;
5207                 wr32(IGC_TSAUXC, tsauxc);
5208                 adapter->perout[1].start = ts;
5209                 spin_unlock(&adapter->tmreg_lock);
5210                 ack |= IGC_TSICR_TT1;
5211         }
5212
5213         if (tsicr & IGC_TSICR_AUTT0) {
5214                 nsec = rd32(IGC_AUXSTMPL0);
5215                 sec  = rd32(IGC_AUXSTMPH0);
5216                 event.type = PTP_CLOCK_EXTTS;
5217                 event.index = 0;
5218                 event.timestamp = sec * NSEC_PER_SEC + nsec;
5219                 ptp_clock_event(adapter->ptp_clock, &event);
5220                 ack |= IGC_TSICR_AUTT0;
5221         }
5222
5223         if (tsicr & IGC_TSICR_AUTT1) {
5224                 nsec = rd32(IGC_AUXSTMPL1);
5225                 sec  = rd32(IGC_AUXSTMPH1);
5226                 event.type = PTP_CLOCK_EXTTS;
5227                 event.index = 1;
5228                 event.timestamp = sec * NSEC_PER_SEC + nsec;
5229                 ptp_clock_event(adapter->ptp_clock, &event);
5230                 ack |= IGC_TSICR_AUTT1;
5231         }
5232
5233         /* acknowledge the interrupts */
5234         wr32(IGC_TSICR, ack);
5235 }
5236
5237 /**
5238  * igc_msix_other - msix other interrupt handler
5239  * @irq: interrupt number
5240  * @data: pointer to a q_vector
5241  */
5242 static irqreturn_t igc_msix_other(int irq, void *data)
5243 {
5244         struct igc_adapter *adapter = data;
5245         struct igc_hw *hw = &adapter->hw;
5246         u32 icr = rd32(IGC_ICR);
5247
5248         /* reading ICR causes bit 31 of EICR to be cleared */
5249         if (icr & IGC_ICR_DRSTA)
5250                 schedule_work(&adapter->reset_task);
5251
5252         if (icr & IGC_ICR_DOUTSYNC) {
5253                 /* HW is reporting DMA is out of sync */
5254                 adapter->stats.doosync++;
5255         }
5256
5257         if (icr & IGC_ICR_LSC) {
5258                 hw->mac.get_link_status = true;
5259                 /* guard against interrupt when we're going down */
5260                 if (!test_bit(__IGC_DOWN, &adapter->state))
5261                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5262         }
5263
5264         if (icr & IGC_ICR_TS)
5265                 igc_tsync_interrupt(adapter);
5266
5267         wr32(IGC_EIMS, adapter->eims_other);
5268
5269         return IRQ_HANDLED;
5270 }
5271
5272 static void igc_write_itr(struct igc_q_vector *q_vector)
5273 {
5274         u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;
5275
5276         if (!q_vector->set_itr)
5277                 return;
5278
5279         if (!itr_val)
5280                 itr_val = IGC_ITR_VAL_MASK;
5281
5282         itr_val |= IGC_EITR_CNT_IGNR;
5283
5284         writel(itr_val, q_vector->itr_register);
5285         q_vector->set_itr = 0;
5286 }
5287
5288 static irqreturn_t igc_msix_ring(int irq, void *data)
5289 {
5290         struct igc_q_vector *q_vector = data;
5291
5292         /* Write the ITR value calculated from the previous interrupt. */
5293         igc_write_itr(q_vector);
5294
5295         napi_schedule(&q_vector->napi);
5296
5297         return IRQ_HANDLED;
5298 }
5299
5300 /**
5301  * igc_request_msix - Initialize MSI-X interrupts
5302  * @adapter: Pointer to adapter structure
5303  *
5304  * igc_request_msix allocates MSI-X vectors and requests interrupts from the
5305  * kernel.
5306  */
5307 static int igc_request_msix(struct igc_adapter *adapter)
5308 {
5309         unsigned int num_q_vectors = adapter->num_q_vectors;
5310         int i = 0, err = 0, vector = 0, free_vector = 0;
5311         struct net_device *netdev = adapter->netdev;
5312
5313         err = request_irq(adapter->msix_entries[vector].vector,
5314                           &igc_msix_other, 0, netdev->name, adapter);
5315         if (err)
5316                 goto err_out;
5317
5318         if (num_q_vectors > MAX_Q_VECTORS) {
5319                 num_q_vectors = MAX_Q_VECTORS;
5320                 dev_warn(&adapter->pdev->dev,
5321                          "The number of queue vectors (%d) is higher than max allowed (%d)\n",
5322                          adapter->num_q_vectors, MAX_Q_VECTORS);
5323         }
5324         for (i = 0; i < num_q_vectors; i++) {
5325                 struct igc_q_vector *q_vector = adapter->q_vector[i];
5326
5327                 vector++;
5328
5329                 q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);
5330
5331                 if (q_vector->rx.ring && q_vector->tx.ring)
5332                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
5333                                 q_vector->rx.ring->queue_index);
5334                 else if (q_vector->tx.ring)
5335                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
5336                                 q_vector->tx.ring->queue_index);
5337                 else if (q_vector->rx.ring)
5338                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
5339                                 q_vector->rx.ring->queue_index);
5340                 else
5341                         sprintf(q_vector->name, "%s-unused", netdev->name);
5342
5343                 err = request_irq(adapter->msix_entries[vector].vector,
5344                                   igc_msix_ring, 0, q_vector->name,
5345                                   q_vector);
5346                 if (err)
5347                         goto err_free;
5348         }
5349
5350         igc_configure_msix(adapter);
5351         return 0;
5352
5353 err_free:
5354         /* free already assigned IRQs */
5355         free_irq(adapter->msix_entries[free_vector++].vector, adapter);
5356
5357         vector--;
5358         for (i = 0; i < vector; i++) {
5359                 free_irq(adapter->msix_entries[free_vector++].vector,
5360                          adapter->q_vector[i]);
5361         }
5362 err_out:
5363         return err;
5364 }
5365
5366 /**
5367  * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
5368  * @adapter: Pointer to adapter structure
5369  *
5370  * This function resets the device so that it has 0 rx queues, tx queues, and
5371  * MSI-X interrupts allocated.
5372  */
5373 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter)
5374 {
5375         igc_free_q_vectors(adapter);
5376         igc_reset_interrupt_capability(adapter);
5377 }
5378
5379 /* Need to wait a few seconds after link up to get diagnostic information from
5380  * the phy
5381  */
5382 static void igc_update_phy_info(struct timer_list *t)
5383 {
5384         struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5385
5386         igc_get_phy_info(&adapter->hw);
5387 }
5388
5389 /**
5390  * igc_has_link - check shared code for link and determine up/down
5391  * @adapter: pointer to driver private info
5392  */
5393 bool igc_has_link(struct igc_adapter *adapter)
5394 {
5395         struct igc_hw *hw = &adapter->hw;
5396         bool link_active = false;
5397
5398         /* get_link_status is set on LSC (link status) interrupt or
5399          * rx sequence error interrupt.  get_link_status will stay
5400          * false until the igc_check_for_link establishes link
5401          * for copper adapters ONLY
5402          */
5403         if (!hw->mac.get_link_status)
5404                 return true;
5405         hw->mac.ops.check_for_link(hw);
5406         link_active = !hw->mac.get_link_status;
5407
5408         if (hw->mac.type == igc_i225) {
5409                 if (!netif_carrier_ok(adapter->netdev)) {
5410                         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5411                 } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) {
5412                         adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE;
5413                         adapter->link_check_timeout = jiffies;
5414                 }
5415         }
5416
5417         return link_active;
5418 }
5419
5420 /**
5421  * igc_watchdog - Timer Call-back
5422  * @t: timer for the watchdog
5423  */
5424 static void igc_watchdog(struct timer_list *t)
5425 {
5426         struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5427         /* Do the rest outside of interrupt context */
5428         schedule_work(&adapter->watchdog_task);
5429 }
5430
5431 static void igc_watchdog_task(struct work_struct *work)
5432 {
5433         struct igc_adapter *adapter = container_of(work,
5434                                                    struct igc_adapter,
5435                                                    watchdog_task);
5436         struct net_device *netdev = adapter->netdev;
5437         struct igc_hw *hw = &adapter->hw;
5438         struct igc_phy_info *phy = &hw->phy;
5439         u16 phy_data, retry_count = 20;
5440         u32 link;
5441         int i;
5442
5443         link = igc_has_link(adapter);
5444
5445         if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) {
5446                 if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5447                         adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5448                 else
5449                         link = false;
5450         }
5451
5452         if (link) {
5453                 /* Cancel scheduled suspend requests. */
5454                 pm_runtime_resume(netdev->dev.parent);
5455
5456                 if (!netif_carrier_ok(netdev)) {
5457                         u32 ctrl;
5458
5459                         hw->mac.ops.get_speed_and_duplex(hw,
5460                                                          &adapter->link_speed,
5461                                                          &adapter->link_duplex);
5462
5463                         ctrl = rd32(IGC_CTRL);
5464                         /* Link status message must follow this format */
5465                         netdev_info(netdev,
5466                                     "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5467                                     adapter->link_speed,
5468                                     adapter->link_duplex == FULL_DUPLEX ?
5469                                     "Full" : "Half",
5470                                     (ctrl & IGC_CTRL_TFCE) &&
5471                                     (ctrl & IGC_CTRL_RFCE) ? "RX/TX" :
5472                                     (ctrl & IGC_CTRL_RFCE) ?  "RX" :
5473                                     (ctrl & IGC_CTRL_TFCE) ?  "TX" : "None");
5474
5475                         /* disable EEE if enabled */
5476                         if ((adapter->flags & IGC_FLAG_EEE) &&
5477                             adapter->link_duplex == HALF_DUPLEX) {
5478                                 netdev_info(netdev,
5479                                             "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n");
5480                                 adapter->hw.dev_spec._base.eee_enable = false;
5481                                 adapter->flags &= ~IGC_FLAG_EEE;
5482                         }
5483
5484                         /* check if SmartSpeed worked */
5485                         igc_check_downshift(hw);
5486                         if (phy->speed_downgraded)
5487                                 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5488
5489                         /* adjust timeout factor according to speed/duplex */
5490                         adapter->tx_timeout_factor = 1;
5491                         switch (adapter->link_speed) {
5492                         case SPEED_10:
5493                                 adapter->tx_timeout_factor = 14;
5494                                 break;
5495                         case SPEED_100:
5496                         case SPEED_1000:
5497                         case SPEED_2500:
5498                                 adapter->tx_timeout_factor = 7;
5499                                 break;
5500                         }
5501
5502                         /* Once the launch time has been set on the wire, there
5503                          * is a delay before the link speed can be determined
5504                          * based on link-up activity. Write into the register
5505                          * as soon as we know the correct link speed.
5506                          */
5507                         igc_tsn_adjust_txtime_offset(adapter);
5508
5509                         if (adapter->link_speed != SPEED_1000)
5510                                 goto no_wait;
5511
5512                         /* wait for Remote receiver status OK */
5513 retry_read_status:
5514                         if (!igc_read_phy_reg(hw, PHY_1000T_STATUS,
5515                                               &phy_data)) {
5516                                 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5517                                     retry_count) {
5518                                         msleep(100);
5519                                         retry_count--;
5520                                         goto retry_read_status;
5521                                 } else if (!retry_count) {
5522                                         netdev_err(netdev, "exceed max 2 second\n");
5523                                 }
5524                         } else {
5525                                 netdev_err(netdev, "read 1000Base-T Status Reg\n");
5526                         }
5527 no_wait:
5528                         netif_carrier_on(netdev);
5529
5530                         /* link state has changed, schedule phy info update */
5531                         if (!test_bit(__IGC_DOWN, &adapter->state))
5532                                 mod_timer(&adapter->phy_info_timer,
5533                                           round_jiffies(jiffies + 2 * HZ));
5534                 }
5535         } else {
5536                 if (netif_carrier_ok(netdev)) {
5537                         adapter->link_speed = 0;
5538                         adapter->link_duplex = 0;
5539
5540                         /* Links status message must follow this format */
5541                         netdev_info(netdev, "NIC Link is Down\n");
5542                         netif_carrier_off(netdev);
5543
5544                         /* link state has changed, schedule phy info update */
5545                         if (!test_bit(__IGC_DOWN, &adapter->state))
5546                                 mod_timer(&adapter->phy_info_timer,
5547                                           round_jiffies(jiffies + 2 * HZ));
5548
5549                         /* link is down, time to check for alternate media */
5550                         if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
5551                                 if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
5552                                         schedule_work(&adapter->reset_task);
5553                                         /* return immediately */
5554                                         return;
5555                                 }
5556                         }
5557                         pm_schedule_suspend(netdev->dev.parent,
5558                                             MSEC_PER_SEC * 5);
5559
5560                 /* also check for alternate media here */
5561                 } else if (!netif_carrier_ok(netdev) &&
5562                            (adapter->flags & IGC_FLAG_MAS_ENABLE)) {
5563                         if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
5564                                 schedule_work(&adapter->reset_task);
5565                                 /* return immediately */
5566                                 return;
5567                         }
5568                 }
5569         }
5570
5571         spin_lock(&adapter->stats64_lock);
5572         igc_update_stats(adapter);
5573         spin_unlock(&adapter->stats64_lock);
5574
5575         for (i = 0; i < adapter->num_tx_queues; i++) {
5576                 struct igc_ring *tx_ring = adapter->tx_ring[i];
5577
5578                 if (!netif_carrier_ok(netdev)) {
5579                         /* We've lost link, so the controller stops DMA,
5580                          * but we've got queued Tx work that's never going
5581                          * to get done, so reset controller to flush Tx.
5582                          * (Do the reset outside of interrupt context).
5583                          */
5584                         if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) {
5585                                 adapter->tx_timeout_count++;
5586                                 schedule_work(&adapter->reset_task);
5587                                 /* return immediately since reset is imminent */
5588                                 return;
5589                         }
5590                 }
5591
5592                 /* Force detection of hung controller every watchdog period */
5593                 set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5594         }
5595
5596         /* Cause software interrupt to ensure Rx ring is cleaned */
5597         if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5598                 u32 eics = 0;
5599
5600                 for (i = 0; i < adapter->num_q_vectors; i++)
5601                         eics |= adapter->q_vector[i]->eims_value;
5602                 wr32(IGC_EICS, eics);
5603         } else {
5604                 wr32(IGC_ICS, IGC_ICS_RXDMT0);
5605         }
5606
5607         igc_ptp_tx_hang(adapter);
5608
5609         /* Reset the timer */
5610         if (!test_bit(__IGC_DOWN, &adapter->state)) {
5611                 if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)
5612                         mod_timer(&adapter->watchdog_timer,
5613                                   round_jiffies(jiffies +  HZ));
5614                 else
5615                         mod_timer(&adapter->watchdog_timer,
5616                                   round_jiffies(jiffies + 2 * HZ));
5617         }
5618 }
5619
5620 /**
5621  * igc_intr_msi - Interrupt Handler
5622  * @irq: interrupt number
5623  * @data: pointer to a network interface device structure
5624  */
5625 static irqreturn_t igc_intr_msi(int irq, void *data)
5626 {
5627         struct igc_adapter *adapter = data;
5628         struct igc_q_vector *q_vector = adapter->q_vector[0];
5629         struct igc_hw *hw = &adapter->hw;
5630         /* read ICR disables interrupts using IAM */
5631         u32 icr = rd32(IGC_ICR);
5632
5633         igc_write_itr(q_vector);
5634
5635         if (icr & IGC_ICR_DRSTA)
5636                 schedule_work(&adapter->reset_task);
5637
5638         if (icr & IGC_ICR_DOUTSYNC) {
5639                 /* HW is reporting DMA is out of sync */
5640                 adapter->stats.doosync++;
5641         }
5642
5643         if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5644                 hw->mac.get_link_status = true;
5645                 if (!test_bit(__IGC_DOWN, &adapter->state))
5646                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5647         }
5648
5649         if (icr & IGC_ICR_TS)
5650                 igc_tsync_interrupt(adapter);
5651
5652         napi_schedule(&q_vector->napi);
5653
5654         return IRQ_HANDLED;
5655 }
5656
5657 /**
5658  * igc_intr - Legacy Interrupt Handler
5659  * @irq: interrupt number
5660  * @data: pointer to a network interface device structure
5661  */
5662 static irqreturn_t igc_intr(int irq, void *data)
5663 {
5664         struct igc_adapter *adapter = data;
5665         struct igc_q_vector *q_vector = adapter->q_vector[0];
5666         struct igc_hw *hw = &adapter->hw;
5667         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
5668          * need for the IMC write
5669          */
5670         u32 icr = rd32(IGC_ICR);
5671
5672         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5673          * not set, then the adapter didn't send an interrupt
5674          */
5675         if (!(icr & IGC_ICR_INT_ASSERTED))
5676                 return IRQ_NONE;
5677
5678         igc_write_itr(q_vector);
5679
5680         if (icr & IGC_ICR_DRSTA)
5681                 schedule_work(&adapter->reset_task);
5682
5683         if (icr & IGC_ICR_DOUTSYNC) {
5684                 /* HW is reporting DMA is out of sync */
5685                 adapter->stats.doosync++;
5686         }
5687
5688         if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5689                 hw->mac.get_link_status = true;
5690                 /* guard against interrupt when we're going down */
5691                 if (!test_bit(__IGC_DOWN, &adapter->state))
5692                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5693         }
5694
5695         if (icr & IGC_ICR_TS)
5696                 igc_tsync_interrupt(adapter);
5697
5698         napi_schedule(&q_vector->napi);
5699
5700         return IRQ_HANDLED;
5701 }
5702
5703 static void igc_free_irq(struct igc_adapter *adapter)
5704 {
5705         if (adapter->msix_entries) {
5706                 int vector = 0, i;
5707
5708                 free_irq(adapter->msix_entries[vector++].vector, adapter);
5709
5710                 for (i = 0; i < adapter->num_q_vectors; i++)
5711                         free_irq(adapter->msix_entries[vector++].vector,
5712                                  adapter->q_vector[i]);
5713         } else {
5714                 free_irq(adapter->pdev->irq, adapter);
5715         }
5716 }
5717
5718 /**
5719  * igc_request_irq - initialize interrupts
5720  * @adapter: Pointer to adapter structure
5721  *
5722  * Attempts to configure interrupts using the best available
5723  * capabilities of the hardware and kernel.
5724  */
5725 static int igc_request_irq(struct igc_adapter *adapter)
5726 {
5727         struct net_device *netdev = adapter->netdev;
5728         struct pci_dev *pdev = adapter->pdev;
5729         int err = 0;
5730
5731         if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5732                 err = igc_request_msix(adapter);
5733                 if (!err)
5734                         goto request_done;
5735                 /* fall back to MSI */
5736                 igc_free_all_tx_resources(adapter);
5737                 igc_free_all_rx_resources(adapter);
5738
5739                 igc_clear_interrupt_scheme(adapter);
5740                 err = igc_init_interrupt_scheme(adapter, false);
5741                 if (err)
5742                         goto request_done;
5743                 igc_setup_all_tx_resources(adapter);
5744                 igc_setup_all_rx_resources(adapter);
5745                 igc_configure(adapter);
5746         }
5747
5748         igc_assign_vector(adapter->q_vector[0], 0);
5749
5750         if (adapter->flags & IGC_FLAG_HAS_MSI) {
5751                 err = request_irq(pdev->irq, &igc_intr_msi, 0,
5752                                   netdev->name, adapter);
5753                 if (!err)
5754                         goto request_done;
5755
5756                 /* fall back to legacy interrupts */
5757                 igc_reset_interrupt_capability(adapter);
5758                 adapter->flags &= ~IGC_FLAG_HAS_MSI;
5759         }
5760
5761         err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED,
5762                           netdev->name, adapter);
5763
5764         if (err)
5765                 netdev_err(netdev, "Error %d getting interrupt\n", err);
5766
5767 request_done:
5768         return err;
5769 }
5770
5771 /**
5772  * __igc_open - Called when a network interface is made active
5773  * @netdev: network interface device structure
5774  * @resuming: boolean indicating if the device is resuming
5775  *
5776  * Returns 0 on success, negative value on failure
5777  *
5778  * The open entry point is called when a network interface is made
5779  * active by the system (IFF_UP).  At this point all resources needed
5780  * for transmit and receive operations are allocated, the interrupt
5781  * handler is registered with the OS, the watchdog timer is started,
5782  * and the stack is notified that the interface is ready.
5783  */
5784 static int __igc_open(struct net_device *netdev, bool resuming)
5785 {
5786         struct igc_adapter *adapter = netdev_priv(netdev);
5787         struct pci_dev *pdev = adapter->pdev;
5788         struct igc_hw *hw = &adapter->hw;
5789         int err = 0;
5790         int i = 0;
5791
5792         /* disallow open during test */
5793
5794         if (test_bit(__IGC_TESTING, &adapter->state)) {
5795                 WARN_ON(resuming);
5796                 return -EBUSY;
5797         }
5798
5799         if (!resuming)
5800                 pm_runtime_get_sync(&pdev->dev);
5801
5802         netif_carrier_off(netdev);
5803
5804         /* allocate transmit descriptors */
5805         err = igc_setup_all_tx_resources(adapter);
5806         if (err)
5807                 goto err_setup_tx;
5808
5809         /* allocate receive descriptors */
5810         err = igc_setup_all_rx_resources(adapter);
5811         if (err)
5812                 goto err_setup_rx;
5813
5814         igc_power_up_link(adapter);
5815
5816         igc_configure(adapter);
5817
5818         err = igc_request_irq(adapter);
5819         if (err)
5820                 goto err_req_irq;
5821
5822         /* Notify the stack of the actual queue counts. */
5823         err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
5824         if (err)
5825                 goto err_set_queues;
5826
5827         err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
5828         if (err)
5829                 goto err_set_queues;
5830
5831         clear_bit(__IGC_DOWN, &adapter->state);
5832
5833         for (i = 0; i < adapter->num_q_vectors; i++)
5834                 napi_enable(&adapter->q_vector[i]->napi);
5835
5836         /* Clear any pending interrupts. */
5837         rd32(IGC_ICR);
5838         igc_irq_enable(adapter);
5839
5840         if (!resuming)
5841                 pm_runtime_put(&pdev->dev);
5842
5843         netif_tx_start_all_queues(netdev);
5844
5845         /* start the watchdog. */
5846         hw->mac.get_link_status = true;
5847         schedule_work(&adapter->watchdog_task);
5848
5849         return IGC_SUCCESS;
5850
5851 err_set_queues:
5852         igc_free_irq(adapter);
5853 err_req_irq:
5854         igc_release_hw_control(adapter);
5855         igc_power_down_phy_copper_base(&adapter->hw);
5856         igc_free_all_rx_resources(adapter);
5857 err_setup_rx:
5858         igc_free_all_tx_resources(adapter);
5859 err_setup_tx:
5860         igc_reset(adapter);
5861         if (!resuming)
5862                 pm_runtime_put(&pdev->dev);
5863
5864         return err;
5865 }
5866
5867 int igc_open(struct net_device *netdev)
5868 {
5869         return __igc_open(netdev, false);
5870 }
5871
5872 /**
5873  * __igc_close - Disables a network interface
5874  * @netdev: network interface device structure
5875  * @suspending: boolean indicating the device is suspending
5876  *
5877  * Returns 0, this is not allowed to fail
5878  *
5879  * The close entry point is called when an interface is de-activated
5880  * by the OS.  The hardware is still under the driver's control, but
5881  * needs to be disabled.  A global MAC reset is issued to stop the
5882  * hardware, and all transmit and receive resources are freed.
5883  */
5884 static int __igc_close(struct net_device *netdev, bool suspending)
5885 {
5886         struct igc_adapter *adapter = netdev_priv(netdev);
5887         struct pci_dev *pdev = adapter->pdev;
5888
5889         WARN_ON(test_bit(__IGC_RESETTING, &adapter->state));
5890
5891         if (!suspending)
5892                 pm_runtime_get_sync(&pdev->dev);
5893
5894         igc_down(adapter);
5895
5896         igc_release_hw_control(adapter);
5897
5898         igc_free_irq(adapter);
5899
5900         igc_free_all_tx_resources(adapter);
5901         igc_free_all_rx_resources(adapter);
5902
5903         if (!suspending)
5904                 pm_runtime_put_sync(&pdev->dev);
5905
5906         return 0;
5907 }
5908
5909 int igc_close(struct net_device *netdev)
5910 {
5911         if (netif_device_present(netdev) || netdev->dismantle)
5912                 return __igc_close(netdev, false);
5913         return 0;
5914 }
5915
5916 /**
5917  * igc_ioctl - Access the hwtstamp interface
5918  * @netdev: network interface device structure
5919  * @ifr: interface request data
5920  * @cmd: ioctl command
5921  **/
5922 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5923 {
5924         switch (cmd) {
5925         case SIOCGHWTSTAMP:
5926                 return igc_ptp_get_ts_config(netdev, ifr);
5927         case SIOCSHWTSTAMP:
5928                 return igc_ptp_set_ts_config(netdev, ifr);
5929         default:
5930                 return -EOPNOTSUPP;
5931         }
5932 }
5933
5934 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue,
5935                                       bool enable)
5936 {
5937         struct igc_ring *ring;
5938
5939         if (queue < 0 || queue >= adapter->num_tx_queues)
5940                 return -EINVAL;
5941
5942         ring = adapter->tx_ring[queue];
5943         ring->launchtime_enable = enable;
5944
5945         return 0;
5946 }
5947
5948 static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now)
5949 {
5950         struct timespec64 b;
5951
5952         b = ktime_to_timespec64(base_time);
5953
5954         return timespec64_compare(now, &b) > 0;
5955 }
5956
5957 static bool validate_schedule(struct igc_adapter *adapter,
5958                               const struct tc_taprio_qopt_offload *qopt)
5959 {
5960         int queue_uses[IGC_MAX_TX_QUEUES] = { };
5961         struct igc_hw *hw = &adapter->hw;
5962         struct timespec64 now;
5963         size_t n;
5964
5965         if (qopt->cycle_time_extension)
5966                 return false;
5967
5968         igc_ptp_read(adapter, &now);
5969
5970         /* If we program the controller's BASET registers with a time
5971          * in the future, it will hold all the packets until that
5972          * time, causing a lot of TX Hangs, so to avoid that, we
5973          * reject schedules that would start in the future.
5974          * Note: Limitation above is no longer in i226.
5975          */
5976         if (!is_base_time_past(qopt->base_time, &now) &&
5977             igc_is_device_id_i225(hw))
5978                 return false;
5979
5980         for (n = 0; n < qopt->num_entries; n++) {
5981                 const struct tc_taprio_sched_entry *e, *prev;
5982                 int i;
5983
5984                 prev = n ? &qopt->entries[n - 1] : NULL;
5985                 e = &qopt->entries[n];
5986
5987                 /* i225 only supports "global" frame preemption
5988                  * settings.
5989                  */
5990                 if (e->command != TC_TAPRIO_CMD_SET_GATES)
5991                         return false;
5992
5993                 for (i = 0; i < adapter->num_tx_queues; i++) {
5994                         if (e->gate_mask & BIT(i))
5995                                 queue_uses[i]++;
5996
5997                         /* There are limitations: A single queue cannot be
5998                          * opened and closed multiple times per cycle unless the
5999                          * gate stays open. Check for it.
6000                          */
6001                         if (queue_uses[i] > 1 &&
6002                             !(prev->gate_mask & BIT(i)))
6003                                 return false;
6004                 }
6005         }
6006
6007         return true;
6008 }
6009
6010 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter,
6011                                      struct tc_etf_qopt_offload *qopt)
6012 {
6013         struct igc_hw *hw = &adapter->hw;
6014         int err;
6015
6016         if (hw->mac.type != igc_i225)
6017                 return -EOPNOTSUPP;
6018
6019         err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable);
6020         if (err)
6021                 return err;
6022
6023         return igc_tsn_offload_apply(adapter);
6024 }
6025
6026 static int igc_tsn_clear_schedule(struct igc_adapter *adapter)
6027 {
6028         int i;
6029
6030         adapter->base_time = 0;
6031         adapter->cycle_time = NSEC_PER_SEC;
6032
6033         for (i = 0; i < adapter->num_tx_queues; i++) {
6034                 struct igc_ring *ring = adapter->tx_ring[i];
6035
6036                 ring->start_time = 0;
6037                 ring->end_time = NSEC_PER_SEC;
6038         }
6039
6040         return 0;
6041 }
6042
6043 static int igc_save_qbv_schedule(struct igc_adapter *adapter,
6044                                  struct tc_taprio_qopt_offload *qopt)
6045 {
6046         bool queue_configured[IGC_MAX_TX_QUEUES] = { };
6047         struct igc_hw *hw = &adapter->hw;
6048         u32 start_time = 0, end_time = 0;
6049         size_t n;
6050         int i;
6051
6052         adapter->qbv_enable = qopt->enable;
6053
6054         if (!qopt->enable)
6055                 return igc_tsn_clear_schedule(adapter);
6056
6057         if (qopt->base_time < 0)
6058                 return -ERANGE;
6059
6060         if (igc_is_device_id_i225(hw) && adapter->base_time)
6061                 return -EALREADY;
6062
6063         if (!validate_schedule(adapter, qopt))
6064                 return -EINVAL;
6065
6066         adapter->cycle_time = qopt->cycle_time;
6067         adapter->base_time = qopt->base_time;
6068
6069         for (n = 0; n < qopt->num_entries; n++) {
6070                 struct tc_taprio_sched_entry *e = &qopt->entries[n];
6071
6072                 end_time += e->interval;
6073
6074                 /* If any of the conditions below are true, we need to manually
6075                  * control the end time of the cycle.
6076                  * 1. Qbv users can specify a cycle time that is not equal
6077                  * to the total GCL intervals. Hence, recalculation is
6078                  * necessary here to exclude the time interval that
6079                  * exceeds the cycle time.
6080                  * 2. According to IEEE Std. 802.1Q-2018 section 8.6.9.2,
6081                  * once the end of the list is reached, it will switch
6082                  * to the END_OF_CYCLE state and leave the gates in the
6083                  * same state until the next cycle is started.
6084                  */
6085                 if (end_time > adapter->cycle_time ||
6086                     n + 1 == qopt->num_entries)
6087                         end_time = adapter->cycle_time;
6088
6089                 for (i = 0; i < adapter->num_tx_queues; i++) {
6090                         struct igc_ring *ring = adapter->tx_ring[i];
6091
6092                         if (!(e->gate_mask & BIT(i)))
6093                                 continue;
6094
6095                         /* Check whether a queue stays open for more than one
6096                          * entry. If so, keep the start and advance the end
6097                          * time.
6098                          */
6099                         if (!queue_configured[i])
6100                                 ring->start_time = start_time;
6101                         ring->end_time = end_time;
6102
6103                         queue_configured[i] = true;
6104                 }
6105
6106                 start_time += e->interval;
6107         }
6108
6109         /* Check whether a queue gets configured.
6110          * If not, set the start and end time to be end time.
6111          */
6112         for (i = 0; i < adapter->num_tx_queues; i++) {
6113                 if (!queue_configured[i]) {
6114                         struct igc_ring *ring = adapter->tx_ring[i];
6115
6116                         ring->start_time = end_time;
6117                         ring->end_time = end_time;
6118                 }
6119         }
6120
6121         return 0;
6122 }
6123
6124 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter,
6125                                          struct tc_taprio_qopt_offload *qopt)
6126 {
6127         struct igc_hw *hw = &adapter->hw;
6128         int err;
6129
6130         if (hw->mac.type != igc_i225)
6131                 return -EOPNOTSUPP;
6132
6133         err = igc_save_qbv_schedule(adapter, qopt);
6134         if (err)
6135                 return err;
6136
6137         return igc_tsn_offload_apply(adapter);
6138 }
6139
6140 static int igc_save_cbs_params(struct igc_adapter *adapter, int queue,
6141                                bool enable, int idleslope, int sendslope,
6142                                int hicredit, int locredit)
6143 {
6144         bool cbs_status[IGC_MAX_SR_QUEUES] = { false };
6145         struct net_device *netdev = adapter->netdev;
6146         struct igc_ring *ring;
6147         int i;
6148
6149         /* i225 has two sets of credit-based shaper logic.
6150          * Supporting it only on the top two priority queues
6151          */
6152         if (queue < 0 || queue > 1)
6153                 return -EINVAL;
6154
6155         ring = adapter->tx_ring[queue];
6156
6157         for (i = 0; i < IGC_MAX_SR_QUEUES; i++)
6158                 if (adapter->tx_ring[i])
6159                         cbs_status[i] = adapter->tx_ring[i]->cbs_enable;
6160
6161         /* CBS should be enabled on the highest priority queue first in order
6162          * for the CBS algorithm to operate as intended.
6163          */
6164         if (enable) {
6165                 if (queue == 1 && !cbs_status[0]) {
6166                         netdev_err(netdev,
6167                                    "Enabling CBS on queue1 before queue0\n");
6168                         return -EINVAL;
6169                 }
6170         } else {
6171                 if (queue == 0 && cbs_status[1]) {
6172                         netdev_err(netdev,
6173                                    "Disabling CBS on queue0 before queue1\n");
6174                         return -EINVAL;
6175                 }
6176         }
6177
6178         ring->cbs_enable = enable;
6179         ring->idleslope = idleslope;
6180         ring->sendslope = sendslope;
6181         ring->hicredit = hicredit;
6182         ring->locredit = locredit;
6183
6184         return 0;
6185 }
6186
6187 static int igc_tsn_enable_cbs(struct igc_adapter *adapter,
6188                               struct tc_cbs_qopt_offload *qopt)
6189 {
6190         struct igc_hw *hw = &adapter->hw;
6191         int err;
6192
6193         if (hw->mac.type != igc_i225)
6194                 return -EOPNOTSUPP;
6195
6196         if (qopt->queue < 0 || qopt->queue > 1)
6197                 return -EINVAL;
6198
6199         err = igc_save_cbs_params(adapter, qopt->queue, qopt->enable,
6200                                   qopt->idleslope, qopt->sendslope,
6201                                   qopt->hicredit, qopt->locredit);
6202         if (err)
6203                 return err;
6204
6205         return igc_tsn_offload_apply(adapter);
6206 }
6207
6208 static int igc_tc_query_caps(struct igc_adapter *adapter,
6209                              struct tc_query_caps_base *base)
6210 {
6211         struct igc_hw *hw = &adapter->hw;
6212
6213         switch (base->type) {
6214         case TC_SETUP_QDISC_TAPRIO: {
6215                 struct tc_taprio_caps *caps = base->caps;
6216
6217                 if (hw->mac.type != igc_i225)
6218                         return -EOPNOTSUPP;
6219
6220                 caps->gate_mask_per_txq = true;
6221
6222                 return 0;
6223         }
6224         default:
6225                 return -EOPNOTSUPP;
6226         }
6227 }
6228
6229 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type,
6230                         void *type_data)
6231 {
6232         struct igc_adapter *adapter = netdev_priv(dev);
6233
6234         switch (type) {
6235         case TC_QUERY_CAPS:
6236                 return igc_tc_query_caps(adapter, type_data);
6237         case TC_SETUP_QDISC_TAPRIO:
6238                 return igc_tsn_enable_qbv_scheduling(adapter, type_data);
6239
6240         case TC_SETUP_QDISC_ETF:
6241                 return igc_tsn_enable_launchtime(adapter, type_data);
6242
6243         case TC_SETUP_QDISC_CBS:
6244                 return igc_tsn_enable_cbs(adapter, type_data);
6245
6246         default:
6247                 return -EOPNOTSUPP;
6248         }
6249 }
6250
6251 static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf)
6252 {
6253         struct igc_adapter *adapter = netdev_priv(dev);
6254
6255         switch (bpf->command) {
6256         case XDP_SETUP_PROG:
6257                 return igc_xdp_set_prog(adapter, bpf->prog, bpf->extack);
6258         case XDP_SETUP_XSK_POOL:
6259                 return igc_xdp_setup_pool(adapter, bpf->xsk.pool,
6260                                           bpf->xsk.queue_id);
6261         default:
6262                 return -EOPNOTSUPP;
6263         }
6264 }
6265
6266 static int igc_xdp_xmit(struct net_device *dev, int num_frames,
6267                         struct xdp_frame **frames, u32 flags)
6268 {
6269         struct igc_adapter *adapter = netdev_priv(dev);
6270         int cpu = smp_processor_id();
6271         struct netdev_queue *nq;
6272         struct igc_ring *ring;
6273         int i, drops;
6274
6275         if (unlikely(test_bit(__IGC_DOWN, &adapter->state)))
6276                 return -ENETDOWN;
6277
6278         if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
6279                 return -EINVAL;
6280
6281         ring = igc_xdp_get_tx_ring(adapter, cpu);
6282         nq = txring_txq(ring);
6283
6284         __netif_tx_lock(nq, cpu);
6285
6286         drops = 0;
6287         for (i = 0; i < num_frames; i++) {
6288                 int err;
6289                 struct xdp_frame *xdpf = frames[i];
6290
6291                 err = igc_xdp_init_tx_descriptor(ring, xdpf);
6292                 if (err) {
6293                         xdp_return_frame_rx_napi(xdpf);
6294                         drops++;
6295                 }
6296         }
6297
6298         if (flags & XDP_XMIT_FLUSH)
6299                 igc_flush_tx_descriptors(ring);
6300
6301         __netif_tx_unlock(nq);
6302
6303         return num_frames - drops;
6304 }
6305
6306 static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter,
6307                                         struct igc_q_vector *q_vector)
6308 {
6309         struct igc_hw *hw = &adapter->hw;
6310         u32 eics = 0;
6311
6312         eics |= q_vector->eims_value;
6313         wr32(IGC_EICS, eics);
6314 }
6315
6316 int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
6317 {
6318         struct igc_adapter *adapter = netdev_priv(dev);
6319         struct igc_q_vector *q_vector;
6320         struct igc_ring *ring;
6321
6322         if (test_bit(__IGC_DOWN, &adapter->state))
6323                 return -ENETDOWN;
6324
6325         if (!igc_xdp_is_enabled(adapter))
6326                 return -ENXIO;
6327
6328         if (queue_id >= adapter->num_rx_queues)
6329                 return -EINVAL;
6330
6331         ring = adapter->rx_ring[queue_id];
6332
6333         if (!ring->xsk_pool)
6334                 return -ENXIO;
6335
6336         q_vector = adapter->q_vector[queue_id];
6337         if (!napi_if_scheduled_mark_missed(&q_vector->napi))
6338                 igc_trigger_rxtxq_interrupt(adapter, q_vector);
6339
6340         return 0;
6341 }
6342
6343 static const struct net_device_ops igc_netdev_ops = {
6344         .ndo_open               = igc_open,
6345         .ndo_stop               = igc_close,
6346         .ndo_start_xmit         = igc_xmit_frame,
6347         .ndo_set_rx_mode        = igc_set_rx_mode,
6348         .ndo_set_mac_address    = igc_set_mac,
6349         .ndo_change_mtu         = igc_change_mtu,
6350         .ndo_get_stats64        = igc_get_stats64,
6351         .ndo_fix_features       = igc_fix_features,
6352         .ndo_set_features       = igc_set_features,
6353         .ndo_features_check     = igc_features_check,
6354         .ndo_eth_ioctl          = igc_ioctl,
6355         .ndo_setup_tc           = igc_setup_tc,
6356         .ndo_bpf                = igc_bpf,
6357         .ndo_xdp_xmit           = igc_xdp_xmit,
6358         .ndo_xsk_wakeup         = igc_xsk_wakeup,
6359 };
6360
6361 /* PCIe configuration access */
6362 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6363 {
6364         struct igc_adapter *adapter = hw->back;
6365
6366         pci_read_config_word(adapter->pdev, reg, value);
6367 }
6368
6369 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6370 {
6371         struct igc_adapter *adapter = hw->back;
6372
6373         pci_write_config_word(adapter->pdev, reg, *value);
6374 }
6375
6376 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6377 {
6378         struct igc_adapter *adapter = hw->back;
6379
6380         if (!pci_is_pcie(adapter->pdev))
6381                 return -IGC_ERR_CONFIG;
6382
6383         pcie_capability_read_word(adapter->pdev, reg, value);
6384
6385         return IGC_SUCCESS;
6386 }
6387
6388 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6389 {
6390         struct igc_adapter *adapter = hw->back;
6391
6392         if (!pci_is_pcie(adapter->pdev))
6393                 return -IGC_ERR_CONFIG;
6394
6395         pcie_capability_write_word(adapter->pdev, reg, *value);
6396
6397         return IGC_SUCCESS;
6398 }
6399
6400 u32 igc_rd32(struct igc_hw *hw, u32 reg)
6401 {
6402         struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw);
6403         u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
6404         u32 value = 0;
6405
6406         if (IGC_REMOVED(hw_addr))
6407                 return ~value;
6408
6409         value = readl(&hw_addr[reg]);
6410
6411         /* reads should not return all F's */
6412         if (!(~value) && (!reg || !(~readl(hw_addr)))) {
6413                 struct net_device *netdev = igc->netdev;
6414
6415                 hw->hw_addr = NULL;
6416                 netif_device_detach(netdev);
6417                 netdev_err(netdev, "PCIe link lost, device now detached\n");
6418                 WARN(pci_device_is_present(igc->pdev),
6419                      "igc: Failed to read reg 0x%x!\n", reg);
6420         }
6421
6422         return value;
6423 }
6424
6425 /**
6426  * igc_probe - Device Initialization Routine
6427  * @pdev: PCI device information struct
6428  * @ent: entry in igc_pci_tbl
6429  *
6430  * Returns 0 on success, negative on failure
6431  *
6432  * igc_probe initializes an adapter identified by a pci_dev structure.
6433  * The OS initialization, configuring the adapter private structure,
6434  * and a hardware reset occur.
6435  */
6436 static int igc_probe(struct pci_dev *pdev,
6437                      const struct pci_device_id *ent)
6438 {
6439         struct igc_adapter *adapter;
6440         struct net_device *netdev;
6441         struct igc_hw *hw;
6442         const struct igc_info *ei = igc_info_tbl[ent->driver_data];
6443         int err;
6444
6445         err = pci_enable_device_mem(pdev);
6446         if (err)
6447                 return err;
6448
6449         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
6450         if (err) {
6451                 dev_err(&pdev->dev,
6452                         "No usable DMA configuration, aborting\n");
6453                 goto err_dma;
6454         }
6455
6456         err = pci_request_mem_regions(pdev, igc_driver_name);
6457         if (err)
6458                 goto err_pci_reg;
6459
6460         err = pci_enable_ptm(pdev, NULL);
6461         if (err < 0)
6462                 dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n");
6463
6464         pci_set_master(pdev);
6465
6466         err = -ENOMEM;
6467         netdev = alloc_etherdev_mq(sizeof(struct igc_adapter),
6468                                    IGC_MAX_TX_QUEUES);
6469
6470         if (!netdev)
6471                 goto err_alloc_etherdev;
6472
6473         SET_NETDEV_DEV(netdev, &pdev->dev);
6474
6475         pci_set_drvdata(pdev, netdev);
6476         adapter = netdev_priv(netdev);
6477         adapter->netdev = netdev;
6478         adapter->pdev = pdev;
6479         hw = &adapter->hw;
6480         hw->back = adapter;
6481         adapter->port_num = hw->bus.func;
6482         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
6483
6484         err = pci_save_state(pdev);
6485         if (err)
6486                 goto err_ioremap;
6487
6488         err = -EIO;
6489         adapter->io_addr = ioremap(pci_resource_start(pdev, 0),
6490                                    pci_resource_len(pdev, 0));
6491         if (!adapter->io_addr)
6492                 goto err_ioremap;
6493
6494         /* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
6495         hw->hw_addr = adapter->io_addr;
6496
6497         netdev->netdev_ops = &igc_netdev_ops;
6498         igc_ethtool_set_ops(netdev);
6499         netdev->watchdog_timeo = 5 * HZ;
6500
6501         netdev->mem_start = pci_resource_start(pdev, 0);
6502         netdev->mem_end = pci_resource_end(pdev, 0);
6503
6504         /* PCI config space info */
6505         hw->vendor_id = pdev->vendor;
6506         hw->device_id = pdev->device;
6507         hw->revision_id = pdev->revision;
6508         hw->subsystem_vendor_id = pdev->subsystem_vendor;
6509         hw->subsystem_device_id = pdev->subsystem_device;
6510
6511         /* Copy the default MAC and PHY function pointers */
6512         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
6513         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
6514
6515         /* Initialize skew-specific constants */
6516         err = ei->get_invariants(hw);
6517         if (err)
6518                 goto err_sw_init;
6519
6520         /* Add supported features to the features list*/
6521         netdev->features |= NETIF_F_SG;
6522         netdev->features |= NETIF_F_TSO;
6523         netdev->features |= NETIF_F_TSO6;
6524         netdev->features |= NETIF_F_TSO_ECN;
6525         netdev->features |= NETIF_F_RXCSUM;
6526         netdev->features |= NETIF_F_HW_CSUM;
6527         netdev->features |= NETIF_F_SCTP_CRC;
6528         netdev->features |= NETIF_F_HW_TC;
6529
6530 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
6531                                   NETIF_F_GSO_GRE_CSUM | \
6532                                   NETIF_F_GSO_IPXIP4 | \
6533                                   NETIF_F_GSO_IPXIP6 | \
6534                                   NETIF_F_GSO_UDP_TUNNEL | \
6535                                   NETIF_F_GSO_UDP_TUNNEL_CSUM)
6536
6537         netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES;
6538         netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES;
6539
6540         /* setup the private structure */
6541         err = igc_sw_init(adapter);
6542         if (err)
6543                 goto err_sw_init;
6544
6545         /* copy netdev features into list of user selectable features */
6546         netdev->hw_features |= NETIF_F_NTUPLE;
6547         netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
6548         netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
6549         netdev->hw_features |= netdev->features;
6550
6551         netdev->features |= NETIF_F_HIGHDMA;
6552
6553         netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
6554         netdev->mpls_features |= NETIF_F_HW_CSUM;
6555         netdev->hw_enc_features |= netdev->vlan_features;
6556
6557         /* MTU range: 68 - 9216 */
6558         netdev->min_mtu = ETH_MIN_MTU;
6559         netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
6560
6561         /* before reading the NVM, reset the controller to put the device in a
6562          * known good starting state
6563          */
6564         hw->mac.ops.reset_hw(hw);
6565
6566         if (igc_get_flash_presence_i225(hw)) {
6567                 if (hw->nvm.ops.validate(hw) < 0) {
6568                         dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
6569                         err = -EIO;
6570                         goto err_eeprom;
6571                 }
6572         }
6573
6574         if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
6575                 /* copy the MAC address out of the NVM */
6576                 if (hw->mac.ops.read_mac_addr(hw))
6577                         dev_err(&pdev->dev, "NVM Read Error\n");
6578         }
6579
6580         eth_hw_addr_set(netdev, hw->mac.addr);
6581
6582         if (!is_valid_ether_addr(netdev->dev_addr)) {
6583                 dev_err(&pdev->dev, "Invalid MAC Address\n");
6584                 err = -EIO;
6585                 goto err_eeprom;
6586         }
6587
6588         /* configure RXPBSIZE and TXPBSIZE */
6589         wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT);
6590         wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
6591
6592         timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
6593         timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0);
6594
6595         INIT_WORK(&adapter->reset_task, igc_reset_task);
6596         INIT_WORK(&adapter->watchdog_task, igc_watchdog_task);
6597
6598         /* Initialize link properties that are user-changeable */
6599         adapter->fc_autoneg = true;
6600         hw->mac.autoneg = true;
6601         hw->phy.autoneg_advertised = 0xaf;
6602
6603         hw->fc.requested_mode = igc_fc_default;
6604         hw->fc.current_mode = igc_fc_default;
6605
6606         /* By default, support wake on port A */
6607         adapter->flags |= IGC_FLAG_WOL_SUPPORTED;
6608
6609         /* initialize the wol settings based on the eeprom settings */
6610         if (adapter->flags & IGC_FLAG_WOL_SUPPORTED)
6611                 adapter->wol |= IGC_WUFC_MAG;
6612
6613         device_set_wakeup_enable(&adapter->pdev->dev,
6614                                  adapter->flags & IGC_FLAG_WOL_SUPPORTED);
6615
6616         igc_ptp_init(adapter);
6617
6618         igc_tsn_clear_schedule(adapter);
6619
6620         /* reset the hardware with the new settings */
6621         igc_reset(adapter);
6622
6623         /* let the f/w know that the h/w is now under the control of the
6624          * driver.
6625          */
6626         igc_get_hw_control(adapter);
6627
6628         strncpy(netdev->name, "eth%d", IFNAMSIZ);
6629         err = register_netdev(netdev);
6630         if (err)
6631                 goto err_register;
6632
6633          /* carrier off reporting is important to ethtool even BEFORE open */
6634         netif_carrier_off(netdev);
6635
6636         /* Check if Media Autosense is enabled */
6637         adapter->ei = *ei;
6638
6639         /* print pcie link status and MAC address */
6640         pcie_print_link_status(pdev);
6641         netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr);
6642
6643         dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
6644         /* Disable EEE for internal PHY devices */
6645         hw->dev_spec._base.eee_enable = false;
6646         adapter->flags &= ~IGC_FLAG_EEE;
6647         igc_set_eee_i225(hw, false, false, false);
6648
6649         pm_runtime_put_noidle(&pdev->dev);
6650
6651         return 0;
6652
6653 err_register:
6654         igc_release_hw_control(adapter);
6655 err_eeprom:
6656         if (!igc_check_reset_block(hw))
6657                 igc_reset_phy(hw);
6658 err_sw_init:
6659         igc_clear_interrupt_scheme(adapter);
6660         iounmap(adapter->io_addr);
6661 err_ioremap:
6662         free_netdev(netdev);
6663 err_alloc_etherdev:
6664         pci_release_mem_regions(pdev);
6665 err_pci_reg:
6666 err_dma:
6667         pci_disable_device(pdev);
6668         return err;
6669 }
6670
6671 /**
6672  * igc_remove - Device Removal Routine
6673  * @pdev: PCI device information struct
6674  *
6675  * igc_remove is called by the PCI subsystem to alert the driver
6676  * that it should release a PCI device.  This could be caused by a
6677  * Hot-Plug event, or because the driver is going to be removed from
6678  * memory.
6679  */
6680 static void igc_remove(struct pci_dev *pdev)
6681 {
6682         struct net_device *netdev = pci_get_drvdata(pdev);
6683         struct igc_adapter *adapter = netdev_priv(netdev);
6684
6685         pm_runtime_get_noresume(&pdev->dev);
6686
6687         igc_flush_nfc_rules(adapter);
6688
6689         igc_ptp_stop(adapter);
6690
6691         set_bit(__IGC_DOWN, &adapter->state);
6692
6693         del_timer_sync(&adapter->watchdog_timer);
6694         del_timer_sync(&adapter->phy_info_timer);
6695
6696         cancel_work_sync(&adapter->reset_task);
6697         cancel_work_sync(&adapter->watchdog_task);
6698
6699         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
6700          * would have already happened in close and is redundant.
6701          */
6702         igc_release_hw_control(adapter);
6703         unregister_netdev(netdev);
6704
6705         igc_clear_interrupt_scheme(adapter);
6706         pci_iounmap(pdev, adapter->io_addr);
6707         pci_release_mem_regions(pdev);
6708
6709         free_netdev(netdev);
6710
6711         pci_disable_device(pdev);
6712 }
6713
6714 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake,
6715                           bool runtime)
6716 {
6717         struct net_device *netdev = pci_get_drvdata(pdev);
6718         struct igc_adapter *adapter = netdev_priv(netdev);
6719         u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol;
6720         struct igc_hw *hw = &adapter->hw;
6721         u32 ctrl, rctl, status;
6722         bool wake;
6723
6724         rtnl_lock();
6725         netif_device_detach(netdev);
6726
6727         if (netif_running(netdev))
6728                 __igc_close(netdev, true);
6729
6730         igc_ptp_suspend(adapter);
6731
6732         igc_clear_interrupt_scheme(adapter);
6733         rtnl_unlock();
6734
6735         status = rd32(IGC_STATUS);
6736         if (status & IGC_STATUS_LU)
6737                 wufc &= ~IGC_WUFC_LNKC;
6738
6739         if (wufc) {
6740                 igc_setup_rctl(adapter);
6741                 igc_set_rx_mode(netdev);
6742
6743                 /* turn on all-multi mode if wake on multicast is enabled */
6744                 if (wufc & IGC_WUFC_MC) {
6745                         rctl = rd32(IGC_RCTL);
6746                         rctl |= IGC_RCTL_MPE;
6747                         wr32(IGC_RCTL, rctl);
6748                 }
6749
6750                 ctrl = rd32(IGC_CTRL);
6751                 ctrl |= IGC_CTRL_ADVD3WUC;
6752                 wr32(IGC_CTRL, ctrl);
6753
6754                 /* Allow time for pending master requests to run */
6755                 igc_disable_pcie_master(hw);
6756
6757                 wr32(IGC_WUC, IGC_WUC_PME_EN);
6758                 wr32(IGC_WUFC, wufc);
6759         } else {
6760                 wr32(IGC_WUC, 0);
6761                 wr32(IGC_WUFC, 0);
6762         }
6763
6764         wake = wufc || adapter->en_mng_pt;
6765         if (!wake)
6766                 igc_power_down_phy_copper_base(&adapter->hw);
6767         else
6768                 igc_power_up_link(adapter);
6769
6770         if (enable_wake)
6771                 *enable_wake = wake;
6772
6773         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
6774          * would have already happened in close and is redundant.
6775          */
6776         igc_release_hw_control(adapter);
6777
6778         pci_disable_device(pdev);
6779
6780         return 0;
6781 }
6782
6783 #ifdef CONFIG_PM
6784 static int __maybe_unused igc_runtime_suspend(struct device *dev)
6785 {
6786         return __igc_shutdown(to_pci_dev(dev), NULL, 1);
6787 }
6788
6789 static void igc_deliver_wake_packet(struct net_device *netdev)
6790 {
6791         struct igc_adapter *adapter = netdev_priv(netdev);
6792         struct igc_hw *hw = &adapter->hw;
6793         struct sk_buff *skb;
6794         u32 wupl;
6795
6796         wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK;
6797
6798         /* WUPM stores only the first 128 bytes of the wake packet.
6799          * Read the packet only if we have the whole thing.
6800          */
6801         if (wupl == 0 || wupl > IGC_WUPM_BYTES)
6802                 return;
6803
6804         skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES);
6805         if (!skb)
6806                 return;
6807
6808         skb_put(skb, wupl);
6809
6810         /* Ensure reads are 32-bit aligned */
6811         wupl = roundup(wupl, 4);
6812
6813         memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl);
6814
6815         skb->protocol = eth_type_trans(skb, netdev);
6816         netif_rx(skb);
6817 }
6818
6819 static int __maybe_unused igc_resume(struct device *dev)
6820 {
6821         struct pci_dev *pdev = to_pci_dev(dev);
6822         struct net_device *netdev = pci_get_drvdata(pdev);
6823         struct igc_adapter *adapter = netdev_priv(netdev);
6824         struct igc_hw *hw = &adapter->hw;
6825         u32 err, val;
6826
6827         pci_set_power_state(pdev, PCI_D0);
6828         pci_restore_state(pdev);
6829         pci_save_state(pdev);
6830
6831         if (!pci_device_is_present(pdev))
6832                 return -ENODEV;
6833         err = pci_enable_device_mem(pdev);
6834         if (err) {
6835                 netdev_err(netdev, "Cannot enable PCI device from suspend\n");
6836                 return err;
6837         }
6838         pci_set_master(pdev);
6839
6840         pci_enable_wake(pdev, PCI_D3hot, 0);
6841         pci_enable_wake(pdev, PCI_D3cold, 0);
6842
6843         if (igc_init_interrupt_scheme(adapter, true)) {
6844                 netdev_err(netdev, "Unable to allocate memory for queues\n");
6845                 return -ENOMEM;
6846         }
6847
6848         igc_reset(adapter);
6849
6850         /* let the f/w know that the h/w is now under the control of the
6851          * driver.
6852          */
6853         igc_get_hw_control(adapter);
6854
6855         val = rd32(IGC_WUS);
6856         if (val & WAKE_PKT_WUS)
6857                 igc_deliver_wake_packet(netdev);
6858
6859         wr32(IGC_WUS, ~0);
6860
6861         rtnl_lock();
6862         if (!err && netif_running(netdev))
6863                 err = __igc_open(netdev, true);
6864
6865         if (!err)
6866                 netif_device_attach(netdev);
6867         rtnl_unlock();
6868
6869         return err;
6870 }
6871
6872 static int __maybe_unused igc_runtime_resume(struct device *dev)
6873 {
6874         return igc_resume(dev);
6875 }
6876
6877 static int __maybe_unused igc_suspend(struct device *dev)
6878 {
6879         return __igc_shutdown(to_pci_dev(dev), NULL, 0);
6880 }
6881
6882 static int __maybe_unused igc_runtime_idle(struct device *dev)
6883 {
6884         struct net_device *netdev = dev_get_drvdata(dev);
6885         struct igc_adapter *adapter = netdev_priv(netdev);
6886
6887         if (!igc_has_link(adapter))
6888                 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6889
6890         return -EBUSY;
6891 }
6892 #endif /* CONFIG_PM */
6893
6894 static void igc_shutdown(struct pci_dev *pdev)
6895 {
6896         bool wake;
6897
6898         __igc_shutdown(pdev, &wake, 0);
6899
6900         if (system_state == SYSTEM_POWER_OFF) {
6901                 pci_wake_from_d3(pdev, wake);
6902                 pci_set_power_state(pdev, PCI_D3hot);
6903         }
6904 }
6905
6906 /**
6907  *  igc_io_error_detected - called when PCI error is detected
6908  *  @pdev: Pointer to PCI device
6909  *  @state: The current PCI connection state
6910  *
6911  *  This function is called after a PCI bus error affecting
6912  *  this device has been detected.
6913  **/
6914 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev,
6915                                               pci_channel_state_t state)
6916 {
6917         struct net_device *netdev = pci_get_drvdata(pdev);
6918         struct igc_adapter *adapter = netdev_priv(netdev);
6919
6920         netif_device_detach(netdev);
6921
6922         if (state == pci_channel_io_perm_failure)
6923                 return PCI_ERS_RESULT_DISCONNECT;
6924
6925         if (netif_running(netdev))
6926                 igc_down(adapter);
6927         pci_disable_device(pdev);
6928
6929         /* Request a slot reset. */
6930         return PCI_ERS_RESULT_NEED_RESET;
6931 }
6932
6933 /**
6934  *  igc_io_slot_reset - called after the PCI bus has been reset.
6935  *  @pdev: Pointer to PCI device
6936  *
6937  *  Restart the card from scratch, as if from a cold-boot. Implementation
6938  *  resembles the first-half of the igc_resume routine.
6939  **/
6940 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev)
6941 {
6942         struct net_device *netdev = pci_get_drvdata(pdev);
6943         struct igc_adapter *adapter = netdev_priv(netdev);
6944         struct igc_hw *hw = &adapter->hw;
6945         pci_ers_result_t result;
6946
6947         if (pci_enable_device_mem(pdev)) {
6948                 netdev_err(netdev, "Could not re-enable PCI device after reset\n");
6949                 result = PCI_ERS_RESULT_DISCONNECT;
6950         } else {
6951                 pci_set_master(pdev);
6952                 pci_restore_state(pdev);
6953                 pci_save_state(pdev);
6954
6955                 pci_enable_wake(pdev, PCI_D3hot, 0);
6956                 pci_enable_wake(pdev, PCI_D3cold, 0);
6957
6958                 /* In case of PCI error, adapter loses its HW address
6959                  * so we should re-assign it here.
6960                  */
6961                 hw->hw_addr = adapter->io_addr;
6962
6963                 igc_reset(adapter);
6964                 wr32(IGC_WUS, ~0);
6965                 result = PCI_ERS_RESULT_RECOVERED;
6966         }
6967
6968         return result;
6969 }
6970
6971 /**
6972  *  igc_io_resume - called when traffic can start to flow again.
6973  *  @pdev: Pointer to PCI device
6974  *
6975  *  This callback is called when the error recovery driver tells us that
6976  *  its OK to resume normal operation. Implementation resembles the
6977  *  second-half of the igc_resume routine.
6978  */
6979 static void igc_io_resume(struct pci_dev *pdev)
6980 {
6981         struct net_device *netdev = pci_get_drvdata(pdev);
6982         struct igc_adapter *adapter = netdev_priv(netdev);
6983
6984         rtnl_lock();
6985         if (netif_running(netdev)) {
6986                 if (igc_open(netdev)) {
6987                         netdev_err(netdev, "igc_open failed after reset\n");
6988                         return;
6989                 }
6990         }
6991
6992         netif_device_attach(netdev);
6993
6994         /* let the f/w know that the h/w is now under the control of the
6995          * driver.
6996          */
6997         igc_get_hw_control(adapter);
6998         rtnl_unlock();
6999 }
7000
7001 static const struct pci_error_handlers igc_err_handler = {
7002         .error_detected = igc_io_error_detected,
7003         .slot_reset = igc_io_slot_reset,
7004         .resume = igc_io_resume,
7005 };
7006
7007 #ifdef CONFIG_PM
7008 static const struct dev_pm_ops igc_pm_ops = {
7009         SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume)
7010         SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume,
7011                            igc_runtime_idle)
7012 };
7013 #endif
7014
7015 static struct pci_driver igc_driver = {
7016         .name     = igc_driver_name,
7017         .id_table = igc_pci_tbl,
7018         .probe    = igc_probe,
7019         .remove   = igc_remove,
7020 #ifdef CONFIG_PM
7021         .driver.pm = &igc_pm_ops,
7022 #endif
7023         .shutdown = igc_shutdown,
7024         .err_handler = &igc_err_handler,
7025 };
7026
7027 /**
7028  * igc_reinit_queues - return error
7029  * @adapter: pointer to adapter structure
7030  */
7031 int igc_reinit_queues(struct igc_adapter *adapter)
7032 {
7033         struct net_device *netdev = adapter->netdev;
7034         int err = 0;
7035
7036         if (netif_running(netdev))
7037                 igc_close(netdev);
7038
7039         igc_reset_interrupt_capability(adapter);
7040
7041         if (igc_init_interrupt_scheme(adapter, true)) {
7042                 netdev_err(netdev, "Unable to allocate memory for queues\n");
7043                 return -ENOMEM;
7044         }
7045
7046         if (netif_running(netdev))
7047                 err = igc_open(netdev);
7048
7049         return err;
7050 }
7051
7052 /**
7053  * igc_get_hw_dev - return device
7054  * @hw: pointer to hardware structure
7055  *
7056  * used by hardware layer to print debugging information
7057  */
7058 struct net_device *igc_get_hw_dev(struct igc_hw *hw)
7059 {
7060         struct igc_adapter *adapter = hw->back;
7061
7062         return adapter->netdev;
7063 }
7064
7065 static void igc_disable_rx_ring_hw(struct igc_ring *ring)
7066 {
7067         struct igc_hw *hw = &ring->q_vector->adapter->hw;
7068         u8 idx = ring->reg_idx;
7069         u32 rxdctl;
7070
7071         rxdctl = rd32(IGC_RXDCTL(idx));
7072         rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE;
7073         rxdctl |= IGC_RXDCTL_SWFLUSH;
7074         wr32(IGC_RXDCTL(idx), rxdctl);
7075 }
7076
7077 void igc_disable_rx_ring(struct igc_ring *ring)
7078 {
7079         igc_disable_rx_ring_hw(ring);
7080         igc_clean_rx_ring(ring);
7081 }
7082
7083 void igc_enable_rx_ring(struct igc_ring *ring)
7084 {
7085         struct igc_adapter *adapter = ring->q_vector->adapter;
7086
7087         igc_configure_rx_ring(adapter, ring);
7088
7089         if (ring->xsk_pool)
7090                 igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
7091         else
7092                 igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
7093 }
7094
7095 static void igc_disable_tx_ring_hw(struct igc_ring *ring)
7096 {
7097         struct igc_hw *hw = &ring->q_vector->adapter->hw;
7098         u8 idx = ring->reg_idx;
7099         u32 txdctl;
7100
7101         txdctl = rd32(IGC_TXDCTL(idx));
7102         txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE;
7103         txdctl |= IGC_TXDCTL_SWFLUSH;
7104         wr32(IGC_TXDCTL(idx), txdctl);
7105 }
7106
7107 void igc_disable_tx_ring(struct igc_ring *ring)
7108 {
7109         igc_disable_tx_ring_hw(ring);
7110         igc_clean_tx_ring(ring);
7111 }
7112
7113 void igc_enable_tx_ring(struct igc_ring *ring)
7114 {
7115         struct igc_adapter *adapter = ring->q_vector->adapter;
7116
7117         igc_configure_tx_ring(adapter, ring);
7118 }
7119
7120 /**
7121  * igc_init_module - Driver Registration Routine
7122  *
7123  * igc_init_module is the first routine called when the driver is
7124  * loaded. All it does is register with the PCI subsystem.
7125  */
7126 static int __init igc_init_module(void)
7127 {
7128         int ret;
7129
7130         pr_info("%s\n", igc_driver_string);
7131         pr_info("%s\n", igc_copyright);
7132
7133         ret = pci_register_driver(&igc_driver);
7134         return ret;
7135 }
7136
7137 module_init(igc_init_module);
7138
7139 /**
7140  * igc_exit_module - Driver Exit Cleanup Routine
7141  *
7142  * igc_exit_module is called just before the driver is removed
7143  * from memory.
7144  */
7145 static void __exit igc_exit_module(void)
7146 {
7147         pci_unregister_driver(&igc_driver);
7148 }
7149
7150 module_exit(igc_exit_module);
7151 /* igc_main.c */