drivers/net/ethernet/intel/e1000/e1000_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 1999 - 2006 Intel Corporation. */
   3
   4 #include "e1000.h"
   5 #include <net/ip6_checksum.h>
   6 #include <linux/io.h>
   7 #include <linux/prefetch.h>
   8 #include <linux/bitops.h>
   9 #include <linux/if_vlan.h>
  10
  11 char e1000_driver_name[] = "e1000";
  12 static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver";
  13 #define DRV_VERSION "7.3.21-k8-NAPI"
  14 const char e1000_driver_version[] = DRV_VERSION;
  15 static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation.";
  16
  17 /* e1000_pci_tbl - PCI Device ID Table
  18  *
  19  * Last entry must be all 0s
  20  *
  21  * Macro expands to...
  22  *   {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
  23  */
  24 static const struct pci_device_id e1000_pci_tbl[] = {
  25         INTEL_E1000_ETHERNET_DEVICE(0x1000),
  26         INTEL_E1000_ETHERNET_DEVICE(0x1001),
  27         INTEL_E1000_ETHERNET_DEVICE(0x1004),
  28         INTEL_E1000_ETHERNET_DEVICE(0x1008),
  29         INTEL_E1000_ETHERNET_DEVICE(0x1009),
  30         INTEL_E1000_ETHERNET_DEVICE(0x100C),
  31         INTEL_E1000_ETHERNET_DEVICE(0x100D),
  32         INTEL_E1000_ETHERNET_DEVICE(0x100E),
  33         INTEL_E1000_ETHERNET_DEVICE(0x100F),
  34         INTEL_E1000_ETHERNET_DEVICE(0x1010),
  35         INTEL_E1000_ETHERNET_DEVICE(0x1011),
  36         INTEL_E1000_ETHERNET_DEVICE(0x1012),
  37         INTEL_E1000_ETHERNET_DEVICE(0x1013),
  38         INTEL_E1000_ETHERNET_DEVICE(0x1014),
  39         INTEL_E1000_ETHERNET_DEVICE(0x1015),
  40         INTEL_E1000_ETHERNET_DEVICE(0x1016),
  41         INTEL_E1000_ETHERNET_DEVICE(0x1017),
  42         INTEL_E1000_ETHERNET_DEVICE(0x1018),
  43         INTEL_E1000_ETHERNET_DEVICE(0x1019),
  44         INTEL_E1000_ETHERNET_DEVICE(0x101A),
  45         INTEL_E1000_ETHERNET_DEVICE(0x101D),
  46         INTEL_E1000_ETHERNET_DEVICE(0x101E),
  47         INTEL_E1000_ETHERNET_DEVICE(0x1026),
  48         INTEL_E1000_ETHERNET_DEVICE(0x1027),
  49         INTEL_E1000_ETHERNET_DEVICE(0x1028),
  50         INTEL_E1000_ETHERNET_DEVICE(0x1075),
  51         INTEL_E1000_ETHERNET_DEVICE(0x1076),
  52         INTEL_E1000_ETHERNET_DEVICE(0x1077),
  53         INTEL_E1000_ETHERNET_DEVICE(0x1078),
  54         INTEL_E1000_ETHERNET_DEVICE(0x1079),
  55         INTEL_E1000_ETHERNET_DEVICE(0x107A),
  56         INTEL_E1000_ETHERNET_DEVICE(0x107B),
  57         INTEL_E1000_ETHERNET_DEVICE(0x107C),
  58         INTEL_E1000_ETHERNET_DEVICE(0x108A),
  59         INTEL_E1000_ETHERNET_DEVICE(0x1099),
  60         INTEL_E1000_ETHERNET_DEVICE(0x10B5),
  61         INTEL_E1000_ETHERNET_DEVICE(0x2E6E),
  62         /* required last entry */
  63         {0,}
  64 };
  65
  66 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
  67
  68 int e1000_up(struct e1000_adapter *adapter);
  69 void e1000_down(struct e1000_adapter *adapter);
  70 void e1000_reinit_locked(struct e1000_adapter *adapter);
  71 void e1000_reset(struct e1000_adapter *adapter);
  72 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter);
  73 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter);
  74 void e1000_free_all_tx_resources(struct e1000_adapter *adapter);
  75 void e1000_free_all_rx_resources(struct e1000_adapter *adapter);
  76 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
  77                                     struct e1000_tx_ring *txdr);
  78 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
  79                                     struct e1000_rx_ring *rxdr);
  80 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
  81                                     struct e1000_tx_ring *tx_ring);
  82 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
  83                                     struct e1000_rx_ring *rx_ring);
  84 void e1000_update_stats(struct e1000_adapter *adapter);
  85
  86 static int e1000_init_module(void);
  87 static void e1000_exit_module(void);
  88 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
  89 static void e1000_remove(struct pci_dev *pdev);
  90 static int e1000_alloc_queues(struct e1000_adapter *adapter);
  91 static int e1000_sw_init(struct e1000_adapter *adapter);
  92 int e1000_open(struct net_device *netdev);
  93 int e1000_close(struct net_device *netdev);
  94 static void e1000_configure_tx(struct e1000_adapter *adapter);
  95 static void e1000_configure_rx(struct e1000_adapter *adapter);
  96 static void e1000_setup_rctl(struct e1000_adapter *adapter);
  97 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter);
  98 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter);
  99 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
 100                                 struct e1000_tx_ring *tx_ring);
 101 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
 102                                 struct e1000_rx_ring *rx_ring);
 103 static void e1000_set_rx_mode(struct net_device *netdev);
 104 static void e1000_update_phy_info_task(struct work_struct *work);
 105 static void e1000_watchdog(struct work_struct *work);
 106 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work);
 107 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
 108                                     struct net_device *netdev);
 109 static int e1000_change_mtu(struct net_device *netdev, int new_mtu);
 110 static int e1000_set_mac(struct net_device *netdev, void *p);
 111 static irqreturn_t e1000_intr(int irq, void *data);
 112 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
 113                                struct e1000_tx_ring *tx_ring);
 114 static int e1000_clean(struct napi_struct *napi, int budget);
 115 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
 116                                struct e1000_rx_ring *rx_ring,
 117                                int *work_done, int work_to_do);
 118 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
 119                                      struct e1000_rx_ring *rx_ring,
 120                                      int *work_done, int work_to_do);
 121 static void e1000_alloc_dummy_rx_buffers(struct e1000_adapter *adapter,
 122                                          struct e1000_rx_ring *rx_ring,
 123                                          int cleaned_count)
 124 {
 125 }
 126 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
 127                                    struct e1000_rx_ring *rx_ring,
 128                                    int cleaned_count);
 129 static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
 130                                          struct e1000_rx_ring *rx_ring,
 131                                          int cleaned_count);
 132 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
 133 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
 134                            int cmd);
 135 static void e1000_enter_82542_rst(struct e1000_adapter *adapter);
 136 static void e1000_leave_82542_rst(struct e1000_adapter *adapter);
 137 static void e1000_tx_timeout(struct net_device *dev);
 138 static void e1000_reset_task(struct work_struct *work);
 139 static void e1000_smartspeed(struct e1000_adapter *adapter);
 140 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
 141                                        struct sk_buff *skb);
 142
 143 static bool e1000_vlan_used(struct e1000_adapter *adapter);
 144 static void e1000_vlan_mode(struct net_device *netdev,
 145                             netdev_features_t features);
 146 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
 147                                      bool filter_on);
 148 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
 149                                  __be16 proto, u16 vid);
 150 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
 151                                   __be16 proto, u16 vid);
 152 static void e1000_restore_vlan(struct e1000_adapter *adapter);
 153
 154 #ifdef CONFIG_PM
 155 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state);
 156 static int e1000_resume(struct pci_dev *pdev);
 157 #endif
 158 static void e1000_shutdown(struct pci_dev *pdev);
 159
 160 #ifdef CONFIG_NET_POLL_CONTROLLER
 161 /* for netdump / net console */
 162 static void e1000_netpoll (struct net_device *netdev);
 163 #endif
 164
 165 #define COPYBREAK_DEFAULT 256
 166 static unsigned int copybreak __read_mostly = COPYBREAK_DEFAULT;
 167 module_param(copybreak, uint, 0644);
 168 MODULE_PARM_DESC(copybreak,
 169         "Maximum size of packet that is copied to a new buffer on receive");
 170
 171 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
 172                                                 pci_channel_state_t state);
 173 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev);
 174 static void e1000_io_resume(struct pci_dev *pdev);
 175
 176 static const struct pci_error_handlers e1000_err_handler = {
 177         .error_detected = e1000_io_error_detected,
 178         .slot_reset = e1000_io_slot_reset,
 179         .resume = e1000_io_resume,
 180 };
 181
 182 static struct pci_driver e1000_driver = {
 183         .name     = e1000_driver_name,
 184         .id_table = e1000_pci_tbl,
 185         .probe    = e1000_probe,
 186         .remove   = e1000_remove,
 187 #ifdef CONFIG_PM
 188         /* Power Management Hooks */
 189         .suspend  = e1000_suspend,
 190         .resume   = e1000_resume,
 191 #endif
 192         .shutdown = e1000_shutdown,
 193         .err_handler = &e1000_err_handler
 194 };
 195
 196 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 197 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
 198 MODULE_LICENSE("GPL v2");
 199 MODULE_VERSION(DRV_VERSION);
 200
 201 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
 202 static int debug = -1;
 203 module_param(debug, int, 0);
 204 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 205
 206 /**
 207  * e1000_get_hw_dev - return device
 208  * used by hardware layer to print debugging information
 209  *
 210  **/
 211 struct net_device *e1000_get_hw_dev(struct e1000_hw *hw)
 212 {
 213         struct e1000_adapter *adapter = hw->back;
 214         return adapter->netdev;
 215 }
 216
 217 /**
 218  * e1000_init_module - Driver Registration Routine
 219  *
 220  * e1000_init_module is the first routine called when the driver is
 221  * loaded. All it does is register with the PCI subsystem.
 222  **/
 223 static int __init e1000_init_module(void)
 224 {
 225         int ret;
 226         pr_info("%s - version %s\n", e1000_driver_string, e1000_driver_version);
 227
 228         pr_info("%s\n", e1000_copyright);
 229
 230         ret = pci_register_driver(&e1000_driver);
 231         if (copybreak != COPYBREAK_DEFAULT) {
 232                 if (copybreak == 0)
 233                         pr_info("copybreak disabled\n");
 234                 else
 235                         pr_info("copybreak enabled for "
 236                                    "packets <= %u bytes\n", copybreak);
 237         }
 238         return ret;
 239 }
 240
 241 module_init(e1000_init_module);
 242
 243 /**
 244  * e1000_exit_module - Driver Exit Cleanup Routine
 245  *
 246  * e1000_exit_module is called just before the driver is removed
 247  * from memory.
 248  **/
 249 static void __exit e1000_exit_module(void)
 250 {
 251         pci_unregister_driver(&e1000_driver);
 252 }
 253
 254 module_exit(e1000_exit_module);
 255
 256 static int e1000_request_irq(struct e1000_adapter *adapter)
 257 {
 258         struct net_device *netdev = adapter->netdev;
 259         irq_handler_t handler = e1000_intr;
 260         int irq_flags = IRQF_SHARED;
 261         int err;
 262
 263         err = request_irq(adapter->pdev->irq, handler, irq_flags, netdev->name,
 264                           netdev);
 265         if (err) {
 266                 e_err(probe, "Unable to allocate interrupt Error: %d\n", err);
 267         }
 268
 269         return err;
 270 }
 271
 272 static void e1000_free_irq(struct e1000_adapter *adapter)
 273 {
 274         struct net_device *netdev = adapter->netdev;
 275
 276         free_irq(adapter->pdev->irq, netdev);
 277 }
 278
 279 /**
 280  * e1000_irq_disable - Mask off interrupt generation on the NIC
 281  * @adapter: board private structure
 282  **/
 283 static void e1000_irq_disable(struct e1000_adapter *adapter)
 284 {
 285         struct e1000_hw *hw = &adapter->hw;
 286
 287         ew32(IMC, ~0);
 288         E1000_WRITE_FLUSH();
 289         synchronize_irq(adapter->pdev->irq);
 290 }
 291
 292 /**
 293  * e1000_irq_enable - Enable default interrupt generation settings
 294  * @adapter: board private structure
 295  **/
 296 static void e1000_irq_enable(struct e1000_adapter *adapter)
 297 {
 298         struct e1000_hw *hw = &adapter->hw;
 299
 300         ew32(IMS, IMS_ENABLE_MASK);
 301         E1000_WRITE_FLUSH();
 302 }
 303
 304 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
 305 {
 306         struct e1000_hw *hw = &adapter->hw;
 307         struct net_device *netdev = adapter->netdev;
 308         u16 vid = hw->mng_cookie.vlan_id;
 309         u16 old_vid = adapter->mng_vlan_id;
 310
 311         if (!e1000_vlan_used(adapter))
 312                 return;
 313
 314         if (!test_bit(vid, adapter->active_vlans)) {
 315                 if (hw->mng_cookie.status &
 316                     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) {
 317                         e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
 318                         adapter->mng_vlan_id = vid;
 319                 } else {
 320                         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
 321                 }
 322                 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) &&
 323                     (vid != old_vid) &&
 324                     !test_bit(old_vid, adapter->active_vlans))
 325                         e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
 326                                                old_vid);
 327         } else {
 328                 adapter->mng_vlan_id = vid;
 329         }
 330 }
 331
 332 static void e1000_init_manageability(struct e1000_adapter *adapter)
 333 {
 334         struct e1000_hw *hw = &adapter->hw;
 335
 336         if (adapter->en_mng_pt) {
 337                 u32 manc = er32(MANC);
 338
 339                 /* disable hardware interception of ARP */
 340                 manc &= ~(E1000_MANC_ARP_EN);
 341
 342                 ew32(MANC, manc);
 343         }
 344 }
 345
 346 static void e1000_release_manageability(struct e1000_adapter *adapter)
 347 {
 348         struct e1000_hw *hw = &adapter->hw;
 349
 350         if (adapter->en_mng_pt) {
 351                 u32 manc = er32(MANC);
 352
 353                 /* re-enable hardware interception of ARP */
 354                 manc |= E1000_MANC_ARP_EN;
 355
 356                 ew32(MANC, manc);
 357         }
 358 }
 359
 360 /**
 361  * e1000_configure - configure the hardware for RX and TX
 362  * @adapter = private board structure
 363  **/
 364 static void e1000_configure(struct e1000_adapter *adapter)
 365 {
 366         struct net_device *netdev = adapter->netdev;
 367         int i;
 368
 369         e1000_set_rx_mode(netdev);
 370
 371         e1000_restore_vlan(adapter);
 372         e1000_init_manageability(adapter);
 373
 374         e1000_configure_tx(adapter);
 375         e1000_setup_rctl(adapter);
 376         e1000_configure_rx(adapter);
 377         /* call E1000_DESC_UNUSED which always leaves
 378          * at least 1 descriptor unused to make sure
 379          * next_to_use != next_to_clean
 380          */
 381         for (i = 0; i < adapter->num_rx_queues; i++) {
 382                 struct e1000_rx_ring *ring = &adapter->rx_ring[i];
 383                 adapter->alloc_rx_buf(adapter, ring,
 384                                       E1000_DESC_UNUSED(ring));
 385         }
 386 }
 387
 388 int e1000_up(struct e1000_adapter *adapter)
 389 {
 390         struct e1000_hw *hw = &adapter->hw;
 391
 392         /* hardware has been reset, we need to reload some things */
 393         e1000_configure(adapter);
 394
 395         clear_bit(__E1000_DOWN, &adapter->flags);
 396
 397         napi_enable(&adapter->napi);
 398
 399         e1000_irq_enable(adapter);
 400
 401         netif_wake_queue(adapter->netdev);
 402
 403         /* fire a link change interrupt to start the watchdog */
 404         ew32(ICS, E1000_ICS_LSC);
 405         return 0;
 406 }
 407
 408 /**
 409  * e1000_power_up_phy - restore link in case the phy was powered down
 410  * @adapter: address of board private structure
 411  *
 412  * The phy may be powered down to save power and turn off link when the
 413  * driver is unloaded and wake on lan is not enabled (among others)
 414  * *** this routine MUST be followed by a call to e1000_reset ***
 415  **/
 416 void e1000_power_up_phy(struct e1000_adapter *adapter)
 417 {
 418         struct e1000_hw *hw = &adapter->hw;
 419         u16 mii_reg = 0;
 420
 421         /* Just clear the power down bit to wake the phy back up */
 422         if (hw->media_type == e1000_media_type_copper) {
 423                 /* according to the manual, the phy will retain its
 424                  * settings across a power-down/up cycle
 425                  */
 426                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 427                 mii_reg &= ~MII_CR_POWER_DOWN;
 428                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 429         }
 430 }
 431
 432 static void e1000_power_down_phy(struct e1000_adapter *adapter)
 433 {
 434         struct e1000_hw *hw = &adapter->hw;
 435
 436         /* Power down the PHY so no link is implied when interface is down *
 437          * The PHY cannot be powered down if any of the following is true *
 438          * (a) WoL is enabled
 439          * (b) AMT is active
 440          * (c) SoL/IDER session is active
 441          */
 442         if (!adapter->wol && hw->mac_type >= e1000_82540 &&
 443            hw->media_type == e1000_media_type_copper) {
 444                 u16 mii_reg = 0;
 445
 446                 switch (hw->mac_type) {
 447                 case e1000_82540:
 448                 case e1000_82545:
 449                 case e1000_82545_rev_3:
 450                 case e1000_82546:
 451                 case e1000_ce4100:
 452                 case e1000_82546_rev_3:
 453                 case e1000_82541:
 454                 case e1000_82541_rev_2:
 455                 case e1000_82547:
 456                 case e1000_82547_rev_2:
 457                         if (er32(MANC) & E1000_MANC_SMBUS_EN)
 458                                 goto out;
 459                         break;
 460                 default:
 461                         goto out;
 462                 }
 463                 e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
 464                 mii_reg |= MII_CR_POWER_DOWN;
 465                 e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
 466                 msleep(1);
 467         }
 468 out:
 469         return;
 470 }
 471
 472 static void e1000_down_and_stop(struct e1000_adapter *adapter)
 473 {
 474         set_bit(__E1000_DOWN, &adapter->flags);
 475
 476         cancel_delayed_work_sync(&adapter->watchdog_task);
 477
 478         /*
 479          * Since the watchdog task can reschedule other tasks, we should cancel
 480          * it first, otherwise we can run into the situation when a work is
 481          * still running after the adapter has been turned down.
 482          */
 483
 484         cancel_delayed_work_sync(&adapter->phy_info_task);
 485         cancel_delayed_work_sync(&adapter->fifo_stall_task);
 486
 487         /* Only kill reset task if adapter is not resetting */
 488         if (!test_bit(__E1000_RESETTING, &adapter->flags))
 489                 cancel_work_sync(&adapter->reset_task);
 490 }
 491
 492 void e1000_down(struct e1000_adapter *adapter)
 493 {
 494         struct e1000_hw *hw = &adapter->hw;
 495         struct net_device *netdev = adapter->netdev;
 496         u32 rctl, tctl;
 497
 498         /* disable receives in the hardware */
 499         rctl = er32(RCTL);
 500         ew32(RCTL, rctl & ~E1000_RCTL_EN);
 501         /* flush and sleep below */
 502
 503         netif_tx_disable(netdev);
 504
 505         /* disable transmits in the hardware */
 506         tctl = er32(TCTL);
 507         tctl &= ~E1000_TCTL_EN;
 508         ew32(TCTL, tctl);
 509         /* flush both disables and wait for them to finish */
 510         E1000_WRITE_FLUSH();
 511         msleep(10);
 512
 513         /* Set the carrier off after transmits have been disabled in the
 514          * hardware, to avoid race conditions with e1000_watchdog() (which
 515          * may be running concurrently to us, checking for the carrier
 516          * bit to decide whether it should enable transmits again). Such
 517          * a race condition would result into transmission being disabled
 518          * in the hardware until the next IFF_DOWN+IFF_UP cycle.
 519          */
 520         netif_carrier_off(netdev);
 521
 522         napi_disable(&adapter->napi);
 523
 524         e1000_irq_disable(adapter);
 525
 526         /* Setting DOWN must be after irq_disable to prevent
 527          * a screaming interrupt.  Setting DOWN also prevents
 528          * tasks from rescheduling.
 529          */
 530         e1000_down_and_stop(adapter);
 531
 532         adapter->link_speed = 0;
 533         adapter->link_duplex = 0;
 534
 535         e1000_reset(adapter);
 536         e1000_clean_all_tx_rings(adapter);
 537         e1000_clean_all_rx_rings(adapter);
 538 }
 539
 540 void e1000_reinit_locked(struct e1000_adapter *adapter)
 541 {
 542         WARN_ON(in_interrupt());
 543         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
 544                 msleep(1);
 545         e1000_down(adapter);
 546         e1000_up(adapter);
 547         clear_bit(__E1000_RESETTING, &adapter->flags);
 548 }
 549
 550 void e1000_reset(struct e1000_adapter *adapter)
 551 {
 552         struct e1000_hw *hw = &adapter->hw;
 553         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
 554         bool legacy_pba_adjust = false;
 555         u16 hwm;
 556
 557         /* Repartition Pba for greater than 9k mtu
 558          * To take effect CTRL.RST is required.
 559          */
 560
 561         switch (hw->mac_type) {
 562         case e1000_82542_rev2_0:
 563         case e1000_82542_rev2_1:
 564         case e1000_82543:
 565         case e1000_82544:
 566         case e1000_82540:
 567         case e1000_82541:
 568         case e1000_82541_rev_2:
 569                 legacy_pba_adjust = true;
 570                 pba = E1000_PBA_48K;
 571                 break;
 572         case e1000_82545:
 573         case e1000_82545_rev_3:
 574         case e1000_82546:
 575         case e1000_ce4100:
 576         case e1000_82546_rev_3:
 577                 pba = E1000_PBA_48K;
 578                 break;
 579         case e1000_82547:
 580         case e1000_82547_rev_2:
 581                 legacy_pba_adjust = true;
 582                 pba = E1000_PBA_30K;
 583                 break;
 584         case e1000_undefined:
 585         case e1000_num_macs:
 586                 break;
 587         }
 588
 589         if (legacy_pba_adjust) {
 590                 if (hw->max_frame_size > E1000_RXBUFFER_8192)
 591                         pba -= 8; /* allocate more FIFO for Tx */
 592
 593                 if (hw->mac_type == e1000_82547) {
 594                         adapter->tx_fifo_head = 0;
 595                         adapter->tx_head_addr = pba << E1000_TX_HEAD_ADDR_SHIFT;
 596                         adapter->tx_fifo_size =
 597                                 (E1000_PBA_40K - pba) << E1000_PBA_BYTES_SHIFT;
 598                         atomic_set(&adapter->tx_fifo_stall, 0);
 599                 }
 600         } else if (hw->max_frame_size >  ETH_FRAME_LEN + ETH_FCS_LEN) {
 601                 /* adjust PBA for jumbo frames */
 602                 ew32(PBA, pba);
 603
 604                 /* To maintain wire speed transmits, the Tx FIFO should be
 605                  * large enough to accommodate two full transmit packets,
 606                  * rounded up to the next 1KB and expressed in KB.  Likewise,
 607                  * the Rx FIFO should be large enough to accommodate at least
 608                  * one full receive packet and is similarly rounded up and
 609                  * expressed in KB.
 610                  */
 611                 pba = er32(PBA);
 612                 /* upper 16 bits has Tx packet buffer allocation size in KB */
 613                 tx_space = pba >> 16;
 614                 /* lower 16 bits has Rx packet buffer allocation size in KB */
 615                 pba &= 0xffff;
 616                 /* the Tx fifo also stores 16 bytes of information about the Tx
 617                  * but don't include ethernet FCS because hardware appends it
 618                  */
 619                 min_tx_space = (hw->max_frame_size +
 620                                 sizeof(struct e1000_tx_desc) -
 621                                 ETH_FCS_LEN) * 2;
 622                 min_tx_space = ALIGN(min_tx_space, 1024);
 623                 min_tx_space >>= 10;
 624                 /* software strips receive CRC, so leave room for it */
 625                 min_rx_space = hw->max_frame_size;
 626                 min_rx_space = ALIGN(min_rx_space, 1024);
 627                 min_rx_space >>= 10;
 628
 629                 /* If current Tx allocation is less than the min Tx FIFO size,
 630                  * and the min Tx FIFO size is less than the current Rx FIFO
 631                  * allocation, take space away from current Rx allocation
 632                  */
 633                 if (tx_space < min_tx_space &&
 634                     ((min_tx_space - tx_space) < pba)) {
 635                         pba = pba - (min_tx_space - tx_space);
 636
 637                         /* PCI/PCIx hardware has PBA alignment constraints */
 638                         switch (hw->mac_type) {
 639                         case e1000_82545 ... e1000_82546_rev_3:
 640                                 pba &= ~(E1000_PBA_8K - 1);
 641                                 break;
 642                         default:
 643                                 break;
 644                         }
 645
 646                         /* if short on Rx space, Rx wins and must trump Tx
 647                          * adjustment or use Early Receive if available
 648                          */
 649                         if (pba < min_rx_space)
 650                                 pba = min_rx_space;
 651                 }
 652         }
 653
 654         ew32(PBA, pba);
 655
 656         /* flow control settings:
 657          * The high water mark must be low enough to fit one full frame
 658          * (or the size used for early receive) above it in the Rx FIFO.
 659          * Set it to the lower of:
 660          * - 90% of the Rx FIFO size, and
 661          * - the full Rx FIFO size minus the early receive size (for parts
 662          *   with ERT support assuming ERT set to E1000_ERT_2048), or
 663          * - the full Rx FIFO size minus one full frame
 664          */
 665         hwm = min(((pba << 10) * 9 / 10),
 666                   ((pba << 10) - hw->max_frame_size));
 667
 668         hw->fc_high_water = hwm & 0xFFF8;       /* 8-byte granularity */
 669         hw->fc_low_water = hw->fc_high_water - 8;
 670         hw->fc_pause_time = E1000_FC_PAUSE_TIME;
 671         hw->fc_send_xon = 1;
 672         hw->fc = hw->original_fc;
 673
 674         /* Allow time for pending master requests to run */
 675         e1000_reset_hw(hw);
 676         if (hw->mac_type >= e1000_82544)
 677                 ew32(WUC, 0);
 678
 679         if (e1000_init_hw(hw))
 680                 e_dev_err("Hardware Error\n");
 681         e1000_update_mng_vlan(adapter);
 682
 683         /* if (adapter->hwflags & HWFLAGS_PHY_PWR_BIT) { */
 684         if (hw->mac_type >= e1000_82544 &&
 685             hw->autoneg == 1 &&
 686             hw->autoneg_advertised == ADVERTISE_1000_FULL) {
 687                 u32 ctrl = er32(CTRL);
 688                 /* clear phy power management bit if we are in gig only mode,
 689                  * which if enabled will attempt negotiation to 100Mb, which
 690                  * can cause a loss of link at power off or driver unload
 691                  */
 692                 ctrl &= ~E1000_CTRL_SWDPIN3;
 693                 ew32(CTRL, ctrl);
 694         }
 695
 696         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
 697         ew32(VET, ETHERNET_IEEE_VLAN_TYPE);
 698
 699         e1000_reset_adaptive(hw);
 700         e1000_phy_get_info(hw, &adapter->phy_info);
 701
 702         e1000_release_manageability(adapter);
 703 }
 704
 705 /* Dump the eeprom for users having checksum issues */
 706 static void e1000_dump_eeprom(struct e1000_adapter *adapter)
 707 {
 708         struct net_device *netdev = adapter->netdev;
 709         struct ethtool_eeprom eeprom;
 710         const struct ethtool_ops *ops = netdev->ethtool_ops;
 711         u8 *data;
 712         int i;
 713         u16 csum_old, csum_new = 0;
 714
 715         eeprom.len = ops->get_eeprom_len(netdev);
 716         eeprom.offset = 0;
 717
 718         data = kmalloc(eeprom.len, GFP_KERNEL);
 719         if (!data)
 720                 return;
 721
 722         ops->get_eeprom(netdev, &eeprom, data);
 723
 724         csum_old = (data[EEPROM_CHECKSUM_REG * 2]) +
 725                    (data[EEPROM_CHECKSUM_REG * 2 + 1] << 8);
 726         for (i = 0; i < EEPROM_CHECKSUM_REG * 2; i += 2)
 727                 csum_new += data[i] + (data[i + 1] << 8);
 728         csum_new = EEPROM_SUM - csum_new;
 729
 730         pr_err("/*********************/\n");
 731         pr_err("Current EEPROM Checksum : 0x%04x\n", csum_old);
 732         pr_err("Calculated              : 0x%04x\n", csum_new);
 733
 734         pr_err("Offset    Values\n");
 735         pr_err("========  ======\n");
 736         print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, data, 128, 0);
 737
 738         pr_err("Include this output when contacting your support provider.\n");
 739         pr_err("This is not a software error! Something bad happened to\n");
 740         pr_err("your hardware or EEPROM image. Ignoring this problem could\n");
 741         pr_err("result in further problems, possibly loss of data,\n");
 742         pr_err("corruption or system hangs!\n");
 743         pr_err("The MAC Address will be reset to 00:00:00:00:00:00,\n");
 744         pr_err("which is invalid and requires you to set the proper MAC\n");
 745         pr_err("address manually before continuing to enable this network\n");
 746         pr_err("device. Please inspect the EEPROM dump and report the\n");
 747         pr_err("issue to your hardware vendor or Intel Customer Support.\n");
 748         pr_err("/*********************/\n");
 749
 750         kfree(data);
 751 }
 752
 753 /**
 754  * e1000_is_need_ioport - determine if an adapter needs ioport resources or not
 755  * @pdev: PCI device information struct
 756  *
 757  * Return true if an adapter needs ioport resources
 758  **/
 759 static int e1000_is_need_ioport(struct pci_dev *pdev)
 760 {
 761         switch (pdev->device) {
 762         case E1000_DEV_ID_82540EM:
 763         case E1000_DEV_ID_82540EM_LOM:
 764         case E1000_DEV_ID_82540EP:
 765         case E1000_DEV_ID_82540EP_LOM:
 766         case E1000_DEV_ID_82540EP_LP:
 767         case E1000_DEV_ID_82541EI:
 768         case E1000_DEV_ID_82541EI_MOBILE:
 769         case E1000_DEV_ID_82541ER:
 770         case E1000_DEV_ID_82541ER_LOM:
 771         case E1000_DEV_ID_82541GI:
 772         case E1000_DEV_ID_82541GI_LF:
 773         case E1000_DEV_ID_82541GI_MOBILE:
 774         case E1000_DEV_ID_82544EI_COPPER:
 775         case E1000_DEV_ID_82544EI_FIBER:
 776         case E1000_DEV_ID_82544GC_COPPER:
 777         case E1000_DEV_ID_82544GC_LOM:
 778         case E1000_DEV_ID_82545EM_COPPER:
 779         case E1000_DEV_ID_82545EM_FIBER:
 780         case E1000_DEV_ID_82546EB_COPPER:
 781         case E1000_DEV_ID_82546EB_FIBER:
 782         case E1000_DEV_ID_82546EB_QUAD_COPPER:
 783                 return true;
 784         default:
 785                 return false;
 786         }
 787 }
 788
 789 static netdev_features_t e1000_fix_features(struct net_device *netdev,
 790         netdev_features_t features)
 791 {
 792         /* Since there is no support for separate Rx/Tx vlan accel
 793          * enable/disable make sure Tx flag is always in same state as Rx.
 794          */
 795         if (features & NETIF_F_HW_VLAN_CTAG_RX)
 796                 features |= NETIF_F_HW_VLAN_CTAG_TX;
 797         else
 798                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
 799
 800         return features;
 801 }
 802
 803 static int e1000_set_features(struct net_device *netdev,
 804         netdev_features_t features)
 805 {
 806         struct e1000_adapter *adapter = netdev_priv(netdev);
 807         netdev_features_t changed = features ^ netdev->features;
 808
 809         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
 810                 e1000_vlan_mode(netdev, features);
 811
 812         if (!(changed & (NETIF_F_RXCSUM | NETIF_F_RXALL)))
 813                 return 0;
 814
 815         netdev->features = features;
 816         adapter->rx_csum = !!(features & NETIF_F_RXCSUM);
 817
 818         if (netif_running(netdev))
 819                 e1000_reinit_locked(adapter);
 820         else
 821                 e1000_reset(adapter);
 822
 823         return 0;
 824 }
 825
 826 static const struct net_device_ops e1000_netdev_ops = {
 827         .ndo_open               = e1000_open,
 828         .ndo_stop               = e1000_close,
 829         .ndo_start_xmit         = e1000_xmit_frame,
 830         .ndo_set_rx_mode        = e1000_set_rx_mode,
 831         .ndo_set_mac_address    = e1000_set_mac,
 832         .ndo_tx_timeout         = e1000_tx_timeout,
 833         .ndo_change_mtu         = e1000_change_mtu,
 834         .ndo_do_ioctl           = e1000_ioctl,
 835         .ndo_validate_addr      = eth_validate_addr,
 836         .ndo_vlan_rx_add_vid    = e1000_vlan_rx_add_vid,
 837         .ndo_vlan_rx_kill_vid   = e1000_vlan_rx_kill_vid,
 838 #ifdef CONFIG_NET_POLL_CONTROLLER
 839         .ndo_poll_controller    = e1000_netpoll,
 840 #endif
 841         .ndo_fix_features       = e1000_fix_features,
 842         .ndo_set_features       = e1000_set_features,
 843 };
 844
 845 /**
 846  * e1000_init_hw_struct - initialize members of hw struct
 847  * @adapter: board private struct
 848  * @hw: structure used by e1000_hw.c
 849  *
 850  * Factors out initialization of the e1000_hw struct to its own function
 851  * that can be called very early at init (just after struct allocation).
 852  * Fields are initialized based on PCI device information and
 853  * OS network device settings (MTU size).
 854  * Returns negative error codes if MAC type setup fails.
 855  */
 856 static int e1000_init_hw_struct(struct e1000_adapter *adapter,
 857                                 struct e1000_hw *hw)
 858 {
 859         struct pci_dev *pdev = adapter->pdev;
 860
 861         /* PCI config space info */
 862         hw->vendor_id = pdev->vendor;
 863         hw->device_id = pdev->device;
 864         hw->subsystem_vendor_id = pdev->subsystem_vendor;
 865         hw->subsystem_id = pdev->subsystem_device;
 866         hw->revision_id = pdev->revision;
 867
 868         pci_read_config_word(pdev, PCI_COMMAND, &hw->pci_cmd_word);
 869
 870         hw->max_frame_size = adapter->netdev->mtu +
 871                              ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
 872         hw->min_frame_size = MINIMUM_ETHERNET_FRAME_SIZE;
 873
 874         /* identify the MAC */
 875         if (e1000_set_mac_type(hw)) {
 876                 e_err(probe, "Unknown MAC Type\n");
 877                 return -EIO;
 878         }
 879
 880         switch (hw->mac_type) {
 881         default:
 882                 break;
 883         case e1000_82541:
 884         case e1000_82547:
 885         case e1000_82541_rev_2:
 886         case e1000_82547_rev_2:
 887                 hw->phy_init_script = 1;
 888                 break;
 889         }
 890
 891         e1000_set_media_type(hw);
 892         e1000_get_bus_info(hw);
 893
 894         hw->wait_autoneg_complete = false;
 895         hw->tbi_compatibility_en = true;
 896         hw->adaptive_ifs = true;
 897
 898         /* Copper options */
 899
 900         if (hw->media_type == e1000_media_type_copper) {
 901                 hw->mdix = AUTO_ALL_MODES;
 902                 hw->disable_polarity_correction = false;
 903                 hw->master_slave = E1000_MASTER_SLAVE;
 904         }
 905
 906         return 0;
 907 }
 908
 909 /**
 910  * e1000_probe - Device Initialization Routine
 911  * @pdev: PCI device information struct
 912  * @ent: entry in e1000_pci_tbl
 913  *
 914  * Returns 0 on success, negative on failure
 915  *
 916  * e1000_probe initializes an adapter identified by a pci_dev structure.
 917  * The OS initialization, configuring of the adapter private structure,
 918  * and a hardware reset occur.
 919  **/
 920 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 921 {
 922         struct net_device *netdev;
 923         struct e1000_adapter *adapter = NULL;
 924         struct e1000_hw *hw;
 925
 926         static int cards_found;
 927         static int global_quad_port_a; /* global ksp3 port a indication */
 928         int i, err, pci_using_dac;
 929         u16 eeprom_data = 0;
 930         u16 tmp = 0;
 931         u16 eeprom_apme_mask = E1000_EEPROM_APME;
 932         int bars, need_ioport;
 933         bool disable_dev = false;
 934
 935         /* do not allocate ioport bars when not needed */
 936         need_ioport = e1000_is_need_ioport(pdev);
 937         if (need_ioport) {
 938                 bars = pci_select_bars(pdev, IORESOURCE_MEM | IORESOURCE_IO);
 939                 err = pci_enable_device(pdev);
 940         } else {
 941                 bars = pci_select_bars(pdev, IORESOURCE_MEM);
 942                 err = pci_enable_device_mem(pdev);
 943         }
 944         if (err)
 945                 return err;
 946
 947         err = pci_request_selected_regions(pdev, bars, e1000_driver_name);
 948         if (err)
 949                 goto err_pci_reg;
 950
 951         pci_set_master(pdev);
 952         err = pci_save_state(pdev);
 953         if (err)
 954                 goto err_alloc_etherdev;
 955
 956         err = -ENOMEM;
 957         netdev = alloc_etherdev(sizeof(struct e1000_adapter));
 958         if (!netdev)
 959                 goto err_alloc_etherdev;
 960
 961         SET_NETDEV_DEV(netdev, &pdev->dev);
 962
 963         pci_set_drvdata(pdev, netdev);
 964         adapter = netdev_priv(netdev);
 965         adapter->netdev = netdev;
 966         adapter->pdev = pdev;
 967         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
 968         adapter->bars = bars;
 969         adapter->need_ioport = need_ioport;
 970
 971         hw = &adapter->hw;
 972         hw->back = adapter;
 973
 974         err = -EIO;
 975         hw->hw_addr = pci_ioremap_bar(pdev, BAR_0);
 976         if (!hw->hw_addr)
 977                 goto err_ioremap;
 978
 979         if (adapter->need_ioport) {
 980                 for (i = BAR_1; i <= BAR_5; i++) {
 981                         if (pci_resource_len(pdev, i) == 0)
 982                                 continue;
 983                         if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
 984                                 hw->io_base = pci_resource_start(pdev, i);
 985                                 break;
 986                         }
 987                 }
 988         }
 989
 990         /* make ready for any if (hw->...) below */
 991         err = e1000_init_hw_struct(adapter, hw);
 992         if (err)
 993                 goto err_sw_init;
 994
 995         /* there is a workaround being applied below that limits
 996          * 64-bit DMA addresses to 64-bit hardware.  There are some
 997          * 32-bit adapters that Tx hang when given 64-bit DMA addresses
 998          */
 999         pci_using_dac = 0;
1000         if ((hw->bus_type == e1000_bus_type_pcix) &&
1001             !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
1002                 pci_using_dac = 1;
1003         } else {
1004                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1005                 if (err) {
1006                         pr_err("No usable DMA config, aborting\n");
1007                         goto err_dma;
1008                 }
1009         }
1010
1011         netdev->netdev_ops = &e1000_netdev_ops;
1012         e1000_set_ethtool_ops(netdev);
1013         netdev->watchdog_timeo = 5 * HZ;
1014         netif_napi_add(netdev, &adapter->napi, e1000_clean, 64);
1015
1016         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1017
1018         adapter->bd_number = cards_found;
1019
1020         /* setup the private structure */
1021
1022         err = e1000_sw_init(adapter);
1023         if (err)
1024                 goto err_sw_init;
1025
1026         err = -EIO;
1027         if (hw->mac_type == e1000_ce4100) {
1028                 hw->ce4100_gbe_mdio_base_virt =
1029                                         ioremap(pci_resource_start(pdev, BAR_1),
1030                                                 pci_resource_len(pdev, BAR_1));
1031
1032                 if (!hw->ce4100_gbe_mdio_base_virt)
1033                         goto err_mdio_ioremap;
1034         }
1035
1036         if (hw->mac_type >= e1000_82543) {
1037                 netdev->hw_features = NETIF_F_SG |
1038                                    NETIF_F_HW_CSUM |
1039                                    NETIF_F_HW_VLAN_CTAG_RX;
1040                 netdev->features = NETIF_F_HW_VLAN_CTAG_TX |
1041                                    NETIF_F_HW_VLAN_CTAG_FILTER;
1042         }
1043
1044         if ((hw->mac_type >= e1000_82544) &&
1045            (hw->mac_type != e1000_82547))
1046                 netdev->hw_features |= NETIF_F_TSO;
1047
1048         netdev->priv_flags |= IFF_SUPP_NOFCS;
1049
1050         netdev->features |= netdev->hw_features;
1051         netdev->hw_features |= (NETIF_F_RXCSUM |
1052                                 NETIF_F_RXALL |
1053                                 NETIF_F_RXFCS);
1054
1055         if (pci_using_dac) {
1056                 netdev->features |= NETIF_F_HIGHDMA;
1057                 netdev->vlan_features |= NETIF_F_HIGHDMA;
1058         }
1059
1060         netdev->vlan_features |= (NETIF_F_TSO |
1061                                   NETIF_F_HW_CSUM |
1062                                   NETIF_F_SG);
1063
1064         /* Do not set IFF_UNICAST_FLT for VMWare's 82545EM */
1065         if (hw->device_id != E1000_DEV_ID_82545EM_COPPER ||
1066             hw->subsystem_vendor_id != PCI_VENDOR_ID_VMWARE)
1067                 netdev->priv_flags |= IFF_UNICAST_FLT;
1068
1069         /* MTU range: 46 - 16110 */
1070         netdev->min_mtu = ETH_ZLEN - ETH_HLEN;
1071         netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
1072
1073         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
1074
1075         /* initialize eeprom parameters */
1076         if (e1000_init_eeprom_params(hw)) {
1077                 e_err(probe, "EEPROM initialization failed\n");
1078                 goto err_eeprom;
1079         }
1080
1081         /* before reading the EEPROM, reset the controller to
1082          * put the device in a known good starting state
1083          */
1084
1085         e1000_reset_hw(hw);
1086
1087         /* make sure the EEPROM is good */
1088         if (e1000_validate_eeprom_checksum(hw) < 0) {
1089                 e_err(probe, "The EEPROM Checksum Is Not Valid\n");
1090                 e1000_dump_eeprom(adapter);
1091                 /* set MAC address to all zeroes to invalidate and temporary
1092                  * disable this device for the user. This blocks regular
1093                  * traffic while still permitting ethtool ioctls from reaching
1094                  * the hardware as well as allowing the user to run the
1095                  * interface after manually setting a hw addr using
1096                  * `ip set address`
1097                  */
1098                 memset(hw->mac_addr, 0, netdev->addr_len);
1099         } else {
1100                 /* copy the MAC address out of the EEPROM */
1101                 if (e1000_read_mac_addr(hw))
1102                         e_err(probe, "EEPROM Read Error\n");
1103         }
1104         /* don't block initialization here due to bad MAC address */
1105         memcpy(netdev->dev_addr, hw->mac_addr, netdev->addr_len);
1106
1107         if (!is_valid_ether_addr(netdev->dev_addr))
1108                 e_err(probe, "Invalid MAC Address\n");
1109
1110
1111         INIT_DELAYED_WORK(&adapter->watchdog_task, e1000_watchdog);
1112         INIT_DELAYED_WORK(&adapter->fifo_stall_task,
1113                           e1000_82547_tx_fifo_stall_task);
1114         INIT_DELAYED_WORK(&adapter->phy_info_task, e1000_update_phy_info_task);
1115         INIT_WORK(&adapter->reset_task, e1000_reset_task);
1116
1117         e1000_check_options(adapter);
1118
1119         /* Initial Wake on LAN setting
1120          * If APM wake is enabled in the EEPROM,
1121          * enable the ACPI Magic Packet filter
1122          */
1123
1124         switch (hw->mac_type) {
1125         case e1000_82542_rev2_0:
1126         case e1000_82542_rev2_1:
1127         case e1000_82543:
1128                 break;
1129         case e1000_82544:
1130                 e1000_read_eeprom(hw,
1131                         EEPROM_INIT_CONTROL2_REG, 1, &eeprom_data);
1132                 eeprom_apme_mask = E1000_EEPROM_82544_APM;
1133                 break;
1134         case e1000_82546:
1135         case e1000_82546_rev_3:
1136                 if (er32(STATUS) & E1000_STATUS_FUNC_1) {
1137                         e1000_read_eeprom(hw,
1138                                 EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1139                         break;
1140                 }
1141                 /* Fall Through */
1142         default:
1143                 e1000_read_eeprom(hw,
1144                         EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1145                 break;
1146         }
1147         if (eeprom_data & eeprom_apme_mask)
1148                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1149
1150         /* now that we have the eeprom settings, apply the special cases
1151          * where the eeprom may be wrong or the board simply won't support
1152          * wake on lan on a particular port
1153          */
1154         switch (pdev->device) {
1155         case E1000_DEV_ID_82546GB_PCIE:
1156                 adapter->eeprom_wol = 0;
1157                 break;
1158         case E1000_DEV_ID_82546EB_FIBER:
1159         case E1000_DEV_ID_82546GB_FIBER:
1160                 /* Wake events only supported on port A for dual fiber
1161                  * regardless of eeprom setting
1162                  */
1163                 if (er32(STATUS) & E1000_STATUS_FUNC_1)
1164                         adapter->eeprom_wol = 0;
1165                 break;
1166         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
1167                 /* if quad port adapter, disable WoL on all but port A */
1168                 if (global_quad_port_a != 0)
1169                         adapter->eeprom_wol = 0;
1170                 else
1171                         adapter->quad_port_a = true;
1172                 /* Reset for multiple quad port adapters */
1173                 if (++global_quad_port_a == 4)
1174                         global_quad_port_a = 0;
1175                 break;
1176         }
1177
1178         /* initialize the wol settings based on the eeprom settings */
1179         adapter->wol = adapter->eeprom_wol;
1180         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1181
1182         /* Auto detect PHY address */
1183         if (hw->mac_type == e1000_ce4100) {
1184                 for (i = 0; i < 32; i++) {
1185                         hw->phy_addr = i;
1186                         e1000_read_phy_reg(hw, PHY_ID2, &tmp);
1187
1188                         if (tmp != 0 && tmp != 0xFF)
1189                                 break;
1190                 }
1191
1192                 if (i >= 32)
1193                         goto err_eeprom;
1194         }
1195
1196         /* reset the hardware with the new settings */
1197         e1000_reset(adapter);
1198
1199         strcpy(netdev->name, "eth%d");
1200         err = register_netdev(netdev);
1201         if (err)
1202                 goto err_register;
1203
1204         e1000_vlan_filter_on_off(adapter, false);
1205
1206         /* print bus type/speed/width info */
1207         e_info(probe, "(PCI%s:%dMHz:%d-bit) %pM\n",
1208                ((hw->bus_type == e1000_bus_type_pcix) ? "-X" : ""),
1209                ((hw->bus_speed == e1000_bus_speed_133) ? 133 :
1210                 (hw->bus_speed == e1000_bus_speed_120) ? 120 :
1211                 (hw->bus_speed == e1000_bus_speed_100) ? 100 :
1212                 (hw->bus_speed == e1000_bus_speed_66) ? 66 : 33),
1213                ((hw->bus_width == e1000_bus_width_64) ? 64 : 32),
1214                netdev->dev_addr);
1215
1216         /* carrier off reporting is important to ethtool even BEFORE open */
1217         netif_carrier_off(netdev);
1218
1219         e_info(probe, "Intel(R) PRO/1000 Network Connection\n");
1220
1221         cards_found++;
1222         return 0;
1223
1224 err_register:
1225 err_eeprom:
1226         e1000_phy_hw_reset(hw);
1227
1228         if (hw->flash_address)
1229                 iounmap(hw->flash_address);
1230         kfree(adapter->tx_ring);
1231         kfree(adapter->rx_ring);
1232 err_dma:
1233 err_sw_init:
1234 err_mdio_ioremap:
1235         iounmap(hw->ce4100_gbe_mdio_base_virt);
1236         iounmap(hw->hw_addr);
1237 err_ioremap:
1238         disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1239         free_netdev(netdev);
1240 err_alloc_etherdev:
1241         pci_release_selected_regions(pdev, bars);
1242 err_pci_reg:
1243         if (!adapter || disable_dev)
1244                 pci_disable_device(pdev);
1245         return err;
1246 }
1247
1248 /**
1249  * e1000_remove - Device Removal Routine
1250  * @pdev: PCI device information struct
1251  *
1252  * e1000_remove is called by the PCI subsystem to alert the driver
1253  * that it should release a PCI device. That could be caused by a
1254  * Hot-Plug event, or because the driver is going to be removed from
1255  * memory.
1256  **/
1257 static void e1000_remove(struct pci_dev *pdev)
1258 {
1259         struct net_device *netdev = pci_get_drvdata(pdev);
1260         struct e1000_adapter *adapter = netdev_priv(netdev);
1261         struct e1000_hw *hw = &adapter->hw;
1262         bool disable_dev;
1263
1264         e1000_down_and_stop(adapter);
1265         e1000_release_manageability(adapter);
1266
1267         unregister_netdev(netdev);
1268
1269         e1000_phy_hw_reset(hw);
1270
1271         kfree(adapter->tx_ring);
1272         kfree(adapter->rx_ring);
1273
1274         if (hw->mac_type == e1000_ce4100)
1275                 iounmap(hw->ce4100_gbe_mdio_base_virt);
1276         iounmap(hw->hw_addr);
1277         if (hw->flash_address)
1278                 iounmap(hw->flash_address);
1279         pci_release_selected_regions(pdev, adapter->bars);
1280
1281         disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1282         free_netdev(netdev);
1283
1284         if (disable_dev)
1285                 pci_disable_device(pdev);
1286 }
1287
1288 /**
1289  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
1290  * @adapter: board private structure to initialize
1291  *
1292  * e1000_sw_init initializes the Adapter private data structure.
1293  * e1000_init_hw_struct MUST be called before this function
1294  **/
1295 static int e1000_sw_init(struct e1000_adapter *adapter)
1296 {
1297         adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1298
1299         adapter->num_tx_queues = 1;
1300         adapter->num_rx_queues = 1;
1301
1302         if (e1000_alloc_queues(adapter)) {
1303                 e_err(probe, "Unable to allocate memory for queues\n");
1304                 return -ENOMEM;
1305         }
1306
1307         /* Explicitly disable IRQ since the NIC can be in any state. */
1308         e1000_irq_disable(adapter);
1309
1310         spin_lock_init(&adapter->stats_lock);
1311
1312         set_bit(__E1000_DOWN, &adapter->flags);
1313
1314         return 0;
1315 }
1316
1317 /**
1318  * e1000_alloc_queues - Allocate memory for all rings
1319  * @adapter: board private structure to initialize
1320  *
1321  * We allocate one ring per queue at run-time since we don't know the
1322  * number of queues at compile-time.
1323  **/
1324 static int e1000_alloc_queues(struct e1000_adapter *adapter)
1325 {
1326         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1327                                    sizeof(struct e1000_tx_ring), GFP_KERNEL);
1328         if (!adapter->tx_ring)
1329                 return -ENOMEM;
1330
1331         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1332                                    sizeof(struct e1000_rx_ring), GFP_KERNEL);
1333         if (!adapter->rx_ring) {
1334                 kfree(adapter->tx_ring);
1335                 return -ENOMEM;
1336         }
1337
1338         return E1000_SUCCESS;
1339 }
1340
1341 /**
1342  * e1000_open - Called when a network interface is made active
1343  * @netdev: network interface device structure
1344  *
1345  * Returns 0 on success, negative value on failure
1346  *
1347  * The open entry point is called when a network interface is made
1348  * active by the system (IFF_UP).  At this point all resources needed
1349  * for transmit and receive operations are allocated, the interrupt
1350  * handler is registered with the OS, the watchdog task is started,
1351  * and the stack is notified that the interface is ready.
1352  **/
1353 int e1000_open(struct net_device *netdev)
1354 {
1355         struct e1000_adapter *adapter = netdev_priv(netdev);
1356         struct e1000_hw *hw = &adapter->hw;
1357         int err;
1358
1359         /* disallow open during test */
1360         if (test_bit(__E1000_TESTING, &adapter->flags))
1361                 return -EBUSY;
1362
1363         netif_carrier_off(netdev);
1364
1365         /* allocate transmit descriptors */
1366         err = e1000_setup_all_tx_resources(adapter);
1367         if (err)
1368                 goto err_setup_tx;
1369
1370         /* allocate receive descriptors */
1371         err = e1000_setup_all_rx_resources(adapter);
1372         if (err)
1373                 goto err_setup_rx;
1374
1375         e1000_power_up_phy(adapter);
1376
1377         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
1378         if ((hw->mng_cookie.status &
1379                           E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
1380                 e1000_update_mng_vlan(adapter);
1381         }
1382
1383         /* before we allocate an interrupt, we must be ready to handle it.
1384          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1385          * as soon as we call pci_request_irq, so we have to setup our
1386          * clean_rx handler before we do so.
1387          */
1388         e1000_configure(adapter);
1389
1390         err = e1000_request_irq(adapter);
1391         if (err)
1392                 goto err_req_irq;
1393
1394         /* From here on the code is the same as e1000_up() */
1395         clear_bit(__E1000_DOWN, &adapter->flags);
1396
1397         napi_enable(&adapter->napi);
1398
1399         e1000_irq_enable(adapter);
1400
1401         netif_start_queue(netdev);
1402
1403         /* fire a link status change interrupt to start the watchdog */
1404         ew32(ICS, E1000_ICS_LSC);
1405
1406         return E1000_SUCCESS;
1407
1408 err_req_irq:
1409         e1000_power_down_phy(adapter);
1410         e1000_free_all_rx_resources(adapter);
1411 err_setup_rx:
1412         e1000_free_all_tx_resources(adapter);
1413 err_setup_tx:
1414         e1000_reset(adapter);
1415
1416         return err;
1417 }
1418
1419 /**
1420  * e1000_close - Disables a network interface
1421  * @netdev: network interface device structure
1422  *
1423  * Returns 0, this is not allowed to fail
1424  *
1425  * The close entry point is called when an interface is de-activated
1426  * by the OS.  The hardware is still under the drivers control, but
1427  * needs to be disabled.  A global MAC reset is issued to stop the
1428  * hardware, and all transmit and receive resources are freed.
1429  **/
1430 int e1000_close(struct net_device *netdev)
1431 {
1432         struct e1000_adapter *adapter = netdev_priv(netdev);
1433         struct e1000_hw *hw = &adapter->hw;
1434         int count = E1000_CHECK_RESET_COUNT;
1435
1436         while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
1437                 usleep_range(10000, 20000);
1438
1439         WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
1440         e1000_down(adapter);
1441         e1000_power_down_phy(adapter);
1442         e1000_free_irq(adapter);
1443
1444         e1000_free_all_tx_resources(adapter);
1445         e1000_free_all_rx_resources(adapter);
1446
1447         /* kill manageability vlan ID if supported, but not if a vlan with
1448          * the same ID is registered on the host OS (let 8021q kill it)
1449          */
1450         if ((hw->mng_cookie.status &
1451              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
1452             !test_bit(adapter->mng_vlan_id, adapter->active_vlans)) {
1453                 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
1454                                        adapter->mng_vlan_id);
1455         }
1456
1457         return 0;
1458 }
1459
1460 /**
1461  * e1000_check_64k_bound - check that memory doesn't cross 64kB boundary
1462  * @adapter: address of board private structure
1463  * @start: address of beginning of memory
1464  * @len: length of memory
1465  **/
1466 static bool e1000_check_64k_bound(struct e1000_adapter *adapter, void *start,
1467                                   unsigned long len)
1468 {
1469         struct e1000_hw *hw = &adapter->hw;
1470         unsigned long begin = (unsigned long)start;
1471         unsigned long end = begin + len;
1472
1473         /* First rev 82545 and 82546 need to not allow any memory
1474          * write location to cross 64k boundary due to errata 23
1475          */
1476         if (hw->mac_type == e1000_82545 ||
1477             hw->mac_type == e1000_ce4100 ||
1478             hw->mac_type == e1000_82546) {
1479                 return ((begin ^ (end - 1)) >> 16) != 0 ? false : true;
1480         }
1481
1482         return true;
1483 }
1484
1485 /**
1486  * e1000_setup_tx_resources - allocate Tx resources (Descriptors)
1487  * @adapter: board private structure
1488  * @txdr:    tx descriptor ring (for a specific queue) to setup
1489  *
1490  * Return 0 on success, negative on failure
1491  **/
1492 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
1493                                     struct e1000_tx_ring *txdr)
1494 {
1495         struct pci_dev *pdev = adapter->pdev;
1496         int size;
1497
1498         size = sizeof(struct e1000_tx_buffer) * txdr->count;
1499         txdr->buffer_info = vzalloc(size);
1500         if (!txdr->buffer_info)
1501                 return -ENOMEM;
1502
1503         /* round up to nearest 4K */
1504
1505         txdr->size = txdr->count * sizeof(struct e1000_tx_desc);
1506         txdr->size = ALIGN(txdr->size, 4096);
1507
1508         txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size, &txdr->dma,
1509                                         GFP_KERNEL);
1510         if (!txdr->desc) {
1511 setup_tx_desc_die:
1512                 vfree(txdr->buffer_info);
1513                 return -ENOMEM;
1514         }
1515
1516         /* Fix for errata 23, can't cross 64kB boundary */
1517         if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1518                 void *olddesc = txdr->desc;
1519                 dma_addr_t olddma = txdr->dma;
1520                 e_err(tx_err, "txdr align check failed: %u bytes at %p\n",
1521                       txdr->size, txdr->desc);
1522                 /* Try again, without freeing the previous */
1523                 txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size,
1524                                                 &txdr->dma, GFP_KERNEL);
1525                 /* Failed allocation, critical failure */
1526                 if (!txdr->desc) {
1527                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1528                                           olddma);
1529                         goto setup_tx_desc_die;
1530                 }
1531
1532                 if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1533                         /* give up */
1534                         dma_free_coherent(&pdev->dev, txdr->size, txdr->desc,
1535                                           txdr->dma);
1536                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1537                                           olddma);
1538                         e_err(probe, "Unable to allocate aligned memory "
1539                               "for the transmit descriptor ring\n");
1540                         vfree(txdr->buffer_info);
1541                         return -ENOMEM;
1542                 } else {
1543                         /* Free old allocation, new allocation was successful */
1544                         dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1545                                           olddma);
1546                 }
1547         }
1548         memset(txdr->desc, 0, txdr->size);
1549
1550         txdr->next_to_use = 0;
1551         txdr->next_to_clean = 0;
1552
1553         return 0;
1554 }
1555
1556 /**
1557  * e1000_setup_all_tx_resources - wrapper to allocate Tx resources
1558  *                                (Descriptors) for all queues
1559  * @adapter: board private structure
1560  *
1561  * Return 0 on success, negative on failure
1562  **/
1563 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter)
1564 {
1565         int i, err = 0;
1566
1567         for (i = 0; i < adapter->num_tx_queues; i++) {
1568                 err = e1000_setup_tx_resources(adapter, &adapter->tx_ring[i]);
1569                 if (err) {
1570                         e_err(probe, "Allocation for Tx Queue %u failed\n", i);
1571                         for (i-- ; i >= 0; i--)
1572                                 e1000_free_tx_resources(adapter,
1573                                                         &adapter->tx_ring[i]);
1574                         break;
1575                 }
1576         }
1577
1578         return err;
1579 }
1580
1581 /**
1582  * e1000_configure_tx - Configure 8254x Transmit Unit after Reset
1583  * @adapter: board private structure
1584  *
1585  * Configure the Tx unit of the MAC after a reset.
1586  **/
1587 static void e1000_configure_tx(struct e1000_adapter *adapter)
1588 {
1589         u64 tdba;
1590         struct e1000_hw *hw = &adapter->hw;
1591         u32 tdlen, tctl, tipg;
1592         u32 ipgr1, ipgr2;
1593
1594         /* Setup the HW Tx Head and Tail descriptor pointers */
1595
1596         switch (adapter->num_tx_queues) {
1597         case 1:
1598         default:
1599                 tdba = adapter->tx_ring[0].dma;
1600                 tdlen = adapter->tx_ring[0].count *
1601                         sizeof(struct e1000_tx_desc);
1602                 ew32(TDLEN, tdlen);
1603                 ew32(TDBAH, (tdba >> 32));
1604                 ew32(TDBAL, (tdba & 0x00000000ffffffffULL));
1605                 ew32(TDT, 0);
1606                 ew32(TDH, 0);
1607                 adapter->tx_ring[0].tdh = ((hw->mac_type >= e1000_82543) ?
1608                                            E1000_TDH : E1000_82542_TDH);
1609                 adapter->tx_ring[0].tdt = ((hw->mac_type >= e1000_82543) ?
1610                                            E1000_TDT : E1000_82542_TDT);
1611                 break;
1612         }
1613
1614         /* Set the default values for the Tx Inter Packet Gap timer */
1615         if ((hw->media_type == e1000_media_type_fiber ||
1616              hw->media_type == e1000_media_type_internal_serdes))
1617                 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1618         else
1619                 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1620
1621         switch (hw->mac_type) {
1622         case e1000_82542_rev2_0:
1623         case e1000_82542_rev2_1:
1624                 tipg = DEFAULT_82542_TIPG_IPGT;
1625                 ipgr1 = DEFAULT_82542_TIPG_IPGR1;
1626                 ipgr2 = DEFAULT_82542_TIPG_IPGR2;
1627                 break;
1628         default:
1629                 ipgr1 = DEFAULT_82543_TIPG_IPGR1;
1630                 ipgr2 = DEFAULT_82543_TIPG_IPGR2;
1631                 break;
1632         }
1633         tipg |= ipgr1 << E1000_TIPG_IPGR1_SHIFT;
1634         tipg |= ipgr2 << E1000_TIPG_IPGR2_SHIFT;
1635         ew32(TIPG, tipg);
1636
1637         /* Set the Tx Interrupt Delay register */
1638
1639         ew32(TIDV, adapter->tx_int_delay);
1640         if (hw->mac_type >= e1000_82540)
1641                 ew32(TADV, adapter->tx_abs_int_delay);
1642
1643         /* Program the Transmit Control Register */
1644
1645         tctl = er32(TCTL);
1646         tctl &= ~E1000_TCTL_CT;
1647         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
1648                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1649
1650         e1000_config_collision_dist(hw);
1651
1652         /* Setup Transmit Descriptor Settings for eop descriptor */
1653         adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
1654
1655         /* only set IDE if we are delaying interrupts using the timers */
1656         if (adapter->tx_int_delay)
1657                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1658
1659         if (hw->mac_type < e1000_82543)
1660                 adapter->txd_cmd |= E1000_TXD_CMD_RPS;
1661         else
1662                 adapter->txd_cmd |= E1000_TXD_CMD_RS;
1663
1664         /* Cache if we're 82544 running in PCI-X because we'll
1665          * need this to apply a workaround later in the send path.
1666          */
1667         if (hw->mac_type == e1000_82544 &&
1668             hw->bus_type == e1000_bus_type_pcix)
1669                 adapter->pcix_82544 = true;
1670
1671         ew32(TCTL, tctl);
1672
1673 }
1674
1675 /**
1676  * e1000_setup_rx_resources - allocate Rx resources (Descriptors)
1677  * @adapter: board private structure
1678  * @rxdr:    rx descriptor ring (for a specific queue) to setup
1679  *
1680  * Returns 0 on success, negative on failure
1681  **/
1682 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
1683                                     struct e1000_rx_ring *rxdr)
1684 {
1685         struct pci_dev *pdev = adapter->pdev;
1686         int size, desc_len;
1687
1688         size = sizeof(struct e1000_rx_buffer) * rxdr->count;
1689         rxdr->buffer_info = vzalloc(size);
1690         if (!rxdr->buffer_info)
1691                 return -ENOMEM;
1692
1693         desc_len = sizeof(struct e1000_rx_desc);
1694
1695         /* Round up to nearest 4K */
1696
1697         rxdr->size = rxdr->count * desc_len;
1698         rxdr->size = ALIGN(rxdr->size, 4096);
1699
1700         rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma,
1701                                         GFP_KERNEL);
1702         if (!rxdr->desc) {
1703 setup_rx_desc_die:
1704                 vfree(rxdr->buffer_info);
1705                 return -ENOMEM;
1706         }
1707
1708         /* Fix for errata 23, can't cross 64kB boundary */
1709         if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1710                 void *olddesc = rxdr->desc;
1711                 dma_addr_t olddma = rxdr->dma;
1712                 e_err(rx_err, "rxdr align check failed: %u bytes at %p\n",
1713                       rxdr->size, rxdr->desc);
1714                 /* Try again, without freeing the previous */
1715                 rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size,
1716                                                 &rxdr->dma, GFP_KERNEL);
1717                 /* Failed allocation, critical failure */
1718                 if (!rxdr->desc) {
1719                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1720                                           olddma);
1721                         goto setup_rx_desc_die;
1722                 }
1723
1724                 if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1725                         /* give up */
1726                         dma_free_coherent(&pdev->dev, rxdr->size, rxdr->desc,
1727                                           rxdr->dma);
1728                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1729                                           olddma);
1730                         e_err(probe, "Unable to allocate aligned memory for "
1731                               "the Rx descriptor ring\n");
1732                         goto setup_rx_desc_die;
1733                 } else {
1734                         /* Free old allocation, new allocation was successful */
1735                         dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1736                                           olddma);
1737                 }
1738         }
1739         memset(rxdr->desc, 0, rxdr->size);
1740
1741         rxdr->next_to_clean = 0;
1742         rxdr->next_to_use = 0;
1743         rxdr->rx_skb_top = NULL;
1744
1745         return 0;
1746 }
1747
1748 /**
1749  * e1000_setup_all_rx_resources - wrapper to allocate Rx resources
1750  *                                (Descriptors) for all queues
1751  * @adapter: board private structure
1752  *
1753  * Return 0 on success, negative on failure
1754  **/
1755 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter)
1756 {
1757         int i, err = 0;
1758
1759         for (i = 0; i < adapter->num_rx_queues; i++) {
1760                 err = e1000_setup_rx_resources(adapter, &adapter->rx_ring[i]);
1761                 if (err) {
1762                         e_err(probe, "Allocation for Rx Queue %u failed\n", i);
1763                         for (i-- ; i >= 0; i--)
1764                                 e1000_free_rx_resources(adapter,
1765                                                         &adapter->rx_ring[i]);
1766                         break;
1767                 }
1768         }
1769
1770         return err;
1771 }
1772
1773 /**
1774  * e1000_setup_rctl - configure the receive control registers
1775  * @adapter: Board private structure
1776  **/
1777 static void e1000_setup_rctl(struct e1000_adapter *adapter)
1778 {
1779         struct e1000_hw *hw = &adapter->hw;
1780         u32 rctl;
1781
1782         rctl = er32(RCTL);
1783
1784         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
1785
1786         rctl |= E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
1787                 E1000_RCTL_RDMTS_HALF |
1788                 (hw->mc_filter_type << E1000_RCTL_MO_SHIFT);
1789
1790         if (hw->tbi_compatibility_on == 1)
1791                 rctl |= E1000_RCTL_SBP;
1792         else
1793                 rctl &= ~E1000_RCTL_SBP;
1794
1795         if (adapter->netdev->mtu <= ETH_DATA_LEN)
1796                 rctl &= ~E1000_RCTL_LPE;
1797         else
1798                 rctl |= E1000_RCTL_LPE;
1799
1800         /* Setup buffer sizes */
1801         rctl &= ~E1000_RCTL_SZ_4096;
1802         rctl |= E1000_RCTL_BSEX;
1803         switch (adapter->rx_buffer_len) {
1804         case E1000_RXBUFFER_2048:
1805         default:
1806                 rctl |= E1000_RCTL_SZ_2048;
1807                 rctl &= ~E1000_RCTL_BSEX;
1808                 break;
1809         case E1000_RXBUFFER_4096:
1810                 rctl |= E1000_RCTL_SZ_4096;
1811                 break;
1812         case E1000_RXBUFFER_8192:
1813                 rctl |= E1000_RCTL_SZ_8192;
1814                 break;
1815         case E1000_RXBUFFER_16384:
1816                 rctl |= E1000_RCTL_SZ_16384;
1817                 break;
1818         }
1819
1820         /* This is useful for sniffing bad packets. */
1821         if (adapter->netdev->features & NETIF_F_RXALL) {
1822                 /* UPE and MPE will be handled by normal PROMISC logic
1823                  * in e1000e_set_rx_mode
1824                  */
1825                 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
1826                          E1000_RCTL_BAM | /* RX All Bcast Pkts */
1827                          E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
1828
1829                 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
1830                           E1000_RCTL_DPF | /* Allow filtered pause */
1831                           E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
1832                 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
1833                  * and that breaks VLANs.
1834                  */
1835         }
1836
1837         ew32(RCTL, rctl);
1838 }
1839
1840 /**
1841  * e1000_configure_rx - Configure 8254x Receive Unit after Reset
1842  * @adapter: board private structure
1843  *
1844  * Configure the Rx unit of the MAC after a reset.
1845  **/
1846 static void e1000_configure_rx(struct e1000_adapter *adapter)
1847 {
1848         u64 rdba;
1849         struct e1000_hw *hw = &adapter->hw;
1850         u32 rdlen, rctl, rxcsum;
1851
1852         if (adapter->netdev->mtu > ETH_DATA_LEN) {
1853                 rdlen = adapter->rx_ring[0].count *
1854                         sizeof(struct e1000_rx_desc);
1855                 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
1856                 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
1857         } else {
1858                 rdlen = adapter->rx_ring[0].count *
1859                         sizeof(struct e1000_rx_desc);
1860                 adapter->clean_rx = e1000_clean_rx_irq;
1861                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
1862         }
1863
1864         /* disable receives while setting up the descriptors */
1865         rctl = er32(RCTL);
1866         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1867
1868         /* set the Receive Delay Timer Register */
1869         ew32(RDTR, adapter->rx_int_delay);
1870
1871         if (hw->mac_type >= e1000_82540) {
1872                 ew32(RADV, adapter->rx_abs_int_delay);
1873                 if (adapter->itr_setting != 0)
1874                         ew32(ITR, 1000000000 / (adapter->itr * 256));
1875         }
1876
1877         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1878          * the Base and Length of the Rx Descriptor Ring
1879          */
1880         switch (adapter->num_rx_queues) {
1881         case 1:
1882         default:
1883                 rdba = adapter->rx_ring[0].dma;
1884                 ew32(RDLEN, rdlen);
1885                 ew32(RDBAH, (rdba >> 32));
1886                 ew32(RDBAL, (rdba & 0x00000000ffffffffULL));
1887                 ew32(RDT, 0);
1888                 ew32(RDH, 0);
1889                 adapter->rx_ring[0].rdh = ((hw->mac_type >= e1000_82543) ?
1890                                            E1000_RDH : E1000_82542_RDH);
1891                 adapter->rx_ring[0].rdt = ((hw->mac_type >= e1000_82543) ?
1892                                            E1000_RDT : E1000_82542_RDT);
1893                 break;
1894         }
1895
1896         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
1897         if (hw->mac_type >= e1000_82543) {
1898                 rxcsum = er32(RXCSUM);
1899                 if (adapter->rx_csum)
1900                         rxcsum |= E1000_RXCSUM_TUOFL;
1901                 else
1902                         /* don't need to clear IPPCSE as it defaults to 0 */
1903                         rxcsum &= ~E1000_RXCSUM_TUOFL;
1904                 ew32(RXCSUM, rxcsum);
1905         }
1906
1907         /* Enable Receives */
1908         ew32(RCTL, rctl | E1000_RCTL_EN);
1909 }
1910
1911 /**
1912  * e1000_free_tx_resources - Free Tx Resources per Queue
1913  * @adapter: board private structure
1914  * @tx_ring: Tx descriptor ring for a specific queue
1915  *
1916  * Free all transmit software resources
1917  **/
1918 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
1919                                     struct e1000_tx_ring *tx_ring)
1920 {
1921         struct pci_dev *pdev = adapter->pdev;
1922
1923         e1000_clean_tx_ring(adapter, tx_ring);
1924
1925         vfree(tx_ring->buffer_info);
1926         tx_ring->buffer_info = NULL;
1927
1928         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
1929                           tx_ring->dma);
1930
1931         tx_ring->desc = NULL;
1932 }
1933
1934 /**
1935  * e1000_free_all_tx_resources - Free Tx Resources for All Queues
1936  * @adapter: board private structure
1937  *
1938  * Free all transmit software resources
1939  **/
1940 void e1000_free_all_tx_resources(struct e1000_adapter *adapter)
1941 {
1942         int i;
1943
1944         for (i = 0; i < adapter->num_tx_queues; i++)
1945                 e1000_free_tx_resources(adapter, &adapter->tx_ring[i]);
1946 }
1947
1948 static void
1949 e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter,
1950                                  struct e1000_tx_buffer *buffer_info)
1951 {
1952         if (buffer_info->dma) {
1953                 if (buffer_info->mapped_as_page)
1954                         dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1955                                        buffer_info->length, DMA_TO_DEVICE);
1956                 else
1957                         dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1958                                          buffer_info->length,
1959                                          DMA_TO_DEVICE);
1960                 buffer_info->dma = 0;
1961         }
1962         if (buffer_info->skb) {
1963                 dev_kfree_skb_any(buffer_info->skb);
1964                 buffer_info->skb = NULL;
1965         }
1966         buffer_info->time_stamp = 0;
1967         /* buffer_info must be completely set up in the transmit path */
1968 }
1969
1970 /**
1971  * e1000_clean_tx_ring - Free Tx Buffers
1972  * @adapter: board private structure
1973  * @tx_ring: ring to be cleaned
1974  **/
1975 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
1976                                 struct e1000_tx_ring *tx_ring)
1977 {
1978         struct e1000_hw *hw = &adapter->hw;
1979         struct e1000_tx_buffer *buffer_info;
1980         unsigned long size;
1981         unsigned int i;
1982
1983         /* Free all the Tx ring sk_buffs */
1984
1985         for (i = 0; i < tx_ring->count; i++) {
1986                 buffer_info = &tx_ring->buffer_info[i];
1987                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
1988         }
1989
1990         netdev_reset_queue(adapter->netdev);
1991         size = sizeof(struct e1000_tx_buffer) * tx_ring->count;
1992         memset(tx_ring->buffer_info, 0, size);
1993
1994         /* Zero out the descriptor ring */
1995
1996         memset(tx_ring->desc, 0, tx_ring->size);
1997
1998         tx_ring->next_to_use = 0;
1999         tx_ring->next_to_clean = 0;
2000         tx_ring->last_tx_tso = false;
2001
2002         writel(0, hw->hw_addr + tx_ring->tdh);
2003         writel(0, hw->hw_addr + tx_ring->tdt);
2004 }
2005
2006 /**
2007  * e1000_clean_all_tx_rings - Free Tx Buffers for all queues
2008  * @adapter: board private structure
2009  **/
2010 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter)
2011 {
2012         int i;
2013
2014         for (i = 0; i < adapter->num_tx_queues; i++)
2015                 e1000_clean_tx_ring(adapter, &adapter->tx_ring[i]);
2016 }
2017
2018 /**
2019  * e1000_free_rx_resources - Free Rx Resources
2020  * @adapter: board private structure
2021  * @rx_ring: ring to clean the resources from
2022  *
2023  * Free all receive software resources
2024  **/
2025 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
2026                                     struct e1000_rx_ring *rx_ring)
2027 {
2028         struct pci_dev *pdev = adapter->pdev;
2029
2030         e1000_clean_rx_ring(adapter, rx_ring);
2031
2032         vfree(rx_ring->buffer_info);
2033         rx_ring->buffer_info = NULL;
2034
2035         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2036                           rx_ring->dma);
2037
2038         rx_ring->desc = NULL;
2039 }
2040
2041 /**
2042  * e1000_free_all_rx_resources - Free Rx Resources for All Queues
2043  * @adapter: board private structure
2044  *
2045  * Free all receive software resources
2046  **/
2047 void e1000_free_all_rx_resources(struct e1000_adapter *adapter)
2048 {
2049         int i;
2050
2051         for (i = 0; i < adapter->num_rx_queues; i++)
2052                 e1000_free_rx_resources(adapter, &adapter->rx_ring[i]);
2053 }
2054
2055 #define E1000_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
2056 static unsigned int e1000_frag_len(const struct e1000_adapter *a)
2057 {
2058         return SKB_DATA_ALIGN(a->rx_buffer_len + E1000_HEADROOM) +
2059                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2060 }
2061
2062 static void *e1000_alloc_frag(const struct e1000_adapter *a)
2063 {
2064         unsigned int len = e1000_frag_len(a);
2065         u8 *data = netdev_alloc_frag(len);
2066
2067         if (likely(data))
2068                 data += E1000_HEADROOM;
2069         return data;
2070 }
2071
2072 /**
2073  * e1000_clean_rx_ring - Free Rx Buffers per Queue
2074  * @adapter: board private structure
2075  * @rx_ring: ring to free buffers from
2076  **/
2077 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
2078                                 struct e1000_rx_ring *rx_ring)
2079 {
2080         struct e1000_hw *hw = &adapter->hw;
2081         struct e1000_rx_buffer *buffer_info;
2082         struct pci_dev *pdev = adapter->pdev;
2083         unsigned long size;
2084         unsigned int i;
2085
2086         /* Free all the Rx netfrags */
2087         for (i = 0; i < rx_ring->count; i++) {
2088                 buffer_info = &rx_ring->buffer_info[i];
2089                 if (adapter->clean_rx == e1000_clean_rx_irq) {
2090                         if (buffer_info->dma)
2091                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
2092                                                  adapter->rx_buffer_len,
2093                                                  DMA_FROM_DEVICE);
2094                         if (buffer_info->rxbuf.data) {
2095                                 skb_free_frag(buffer_info->rxbuf.data);
2096                                 buffer_info->rxbuf.data = NULL;
2097                         }
2098                 } else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) {
2099                         if (buffer_info->dma)
2100                                 dma_unmap_page(&pdev->dev, buffer_info->dma,
2101                                                adapter->rx_buffer_len,
2102                                                DMA_FROM_DEVICE);
2103                         if (buffer_info->rxbuf.page) {
2104                                 put_page(buffer_info->rxbuf.page);
2105                                 buffer_info->rxbuf.page = NULL;
2106                         }
2107                 }
2108
2109                 buffer_info->dma = 0;
2110         }
2111
2112         /* there also may be some cached data from a chained receive */
2113         napi_free_frags(&adapter->napi);
2114         rx_ring->rx_skb_top = NULL;
2115
2116         size = sizeof(struct e1000_rx_buffer) * rx_ring->count;
2117         memset(rx_ring->buffer_info, 0, size);
2118
2119         /* Zero out the descriptor ring */
2120         memset(rx_ring->desc, 0, rx_ring->size);
2121
2122         rx_ring->next_to_clean = 0;
2123         rx_ring->next_to_use = 0;
2124
2125         writel(0, hw->hw_addr + rx_ring->rdh);
2126         writel(0, hw->hw_addr + rx_ring->rdt);
2127 }
2128
2129 /**
2130  * e1000_clean_all_rx_rings - Free Rx Buffers for all queues
2131  * @adapter: board private structure
2132  **/
2133 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter)
2134 {
2135         int i;
2136
2137         for (i = 0; i < adapter->num_rx_queues; i++)
2138                 e1000_clean_rx_ring(adapter, &adapter->rx_ring[i]);
2139 }
2140
2141 /* The 82542 2.0 (revision 2) needs to have the receive unit in reset
2142  * and memory write and invalidate disabled for certain operations
2143  */
2144 static void e1000_enter_82542_rst(struct e1000_adapter *adapter)
2145 {
2146         struct e1000_hw *hw = &adapter->hw;
2147         struct net_device *netdev = adapter->netdev;
2148         u32 rctl;
2149
2150         e1000_pci_clear_mwi(hw);
2151
2152         rctl = er32(RCTL);
2153         rctl |= E1000_RCTL_RST;
2154         ew32(RCTL, rctl);
2155         E1000_WRITE_FLUSH();
2156         mdelay(5);
2157
2158         if (netif_running(netdev))
2159                 e1000_clean_all_rx_rings(adapter);
2160 }
2161
2162 static void e1000_leave_82542_rst(struct e1000_adapter *adapter)
2163 {
2164         struct e1000_hw *hw = &adapter->hw;
2165         struct net_device *netdev = adapter->netdev;
2166         u32 rctl;
2167
2168         rctl = er32(RCTL);
2169         rctl &= ~E1000_RCTL_RST;
2170         ew32(RCTL, rctl);
2171         E1000_WRITE_FLUSH();
2172         mdelay(5);
2173
2174         if (hw->pci_cmd_word & PCI_COMMAND_INVALIDATE)
2175                 e1000_pci_set_mwi(hw);
2176
2177         if (netif_running(netdev)) {
2178                 /* No need to loop, because 82542 supports only 1 queue */
2179                 struct e1000_rx_ring *ring = &adapter->rx_ring[0];
2180                 e1000_configure_rx(adapter);
2181                 adapter->alloc_rx_buf(adapter, ring, E1000_DESC_UNUSED(ring));
2182         }
2183 }
2184
2185 /**
2186  * e1000_set_mac - Change the Ethernet Address of the NIC
2187  * @netdev: network interface device structure
2188  * @p: pointer to an address structure
2189  *
2190  * Returns 0 on success, negative on failure
2191  **/
2192 static int e1000_set_mac(struct net_device *netdev, void *p)
2193 {
2194         struct e1000_adapter *adapter = netdev_priv(netdev);
2195         struct e1000_hw *hw = &adapter->hw;
2196         struct sockaddr *addr = p;
2197
2198         if (!is_valid_ether_addr(addr->sa_data))
2199                 return -EADDRNOTAVAIL;
2200
2201         /* 82542 2.0 needs to be in reset to write receive address registers */
2202
2203         if (hw->mac_type == e1000_82542_rev2_0)
2204                 e1000_enter_82542_rst(adapter);
2205
2206         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2207         memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len);
2208
2209         e1000_rar_set(hw, hw->mac_addr, 0);
2210
2211         if (hw->mac_type == e1000_82542_rev2_0)
2212                 e1000_leave_82542_rst(adapter);
2213
2214         return 0;
2215 }
2216
2217 /**
2218  * e1000_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2219  * @netdev: network interface device structure
2220  *
2221  * The set_rx_mode entry point is called whenever the unicast or multicast
2222  * address lists or the network interface flags are updated. This routine is
2223  * responsible for configuring the hardware for proper unicast, multicast,
2224  * promiscuous mode, and all-multi behavior.
2225  **/
2226 static void e1000_set_rx_mode(struct net_device *netdev)
2227 {
2228         struct e1000_adapter *adapter = netdev_priv(netdev);
2229         struct e1000_hw *hw = &adapter->hw;
2230         struct netdev_hw_addr *ha;
2231         bool use_uc = false;
2232         u32 rctl;
2233         u32 hash_value;
2234         int i, rar_entries = E1000_RAR_ENTRIES;
2235         int mta_reg_count = E1000_NUM_MTA_REGISTERS;
2236         u32 *mcarray = kcalloc(mta_reg_count, sizeof(u32), GFP_ATOMIC);
2237
2238         if (!mcarray)
2239                 return;
2240
2241         /* Check for Promiscuous and All Multicast modes */
2242
2243         rctl = er32(RCTL);
2244
2245         if (netdev->flags & IFF_PROMISC) {
2246                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2247                 rctl &= ~E1000_RCTL_VFE;
2248         } else {
2249                 if (netdev->flags & IFF_ALLMULTI)
2250                         rctl |= E1000_RCTL_MPE;
2251                 else
2252                         rctl &= ~E1000_RCTL_MPE;
2253                 /* Enable VLAN filter if there is a VLAN */
2254                 if (e1000_vlan_used(adapter))
2255                         rctl |= E1000_RCTL_VFE;
2256         }
2257
2258         if (netdev_uc_count(netdev) > rar_entries - 1) {
2259                 rctl |= E1000_RCTL_UPE;
2260         } else if (!(netdev->flags & IFF_PROMISC)) {
2261                 rctl &= ~E1000_RCTL_UPE;
2262                 use_uc = true;
2263         }
2264
2265         ew32(RCTL, rctl);
2266
2267         /* 82542 2.0 needs to be in reset to write receive address registers */
2268
2269         if (hw->mac_type == e1000_82542_rev2_0)
2270                 e1000_enter_82542_rst(adapter);
2271
2272         /* load the first 14 addresses into the exact filters 1-14. Unicast
2273          * addresses take precedence to avoid disabling unicast filtering
2274          * when possible.
2275          *
2276          * RAR 0 is used for the station MAC address
2277          * if there are not 14 addresses, go ahead and clear the filters
2278          */
2279         i = 1;
2280         if (use_uc)
2281                 netdev_for_each_uc_addr(ha, netdev) {
2282                         if (i == rar_entries)
2283                                 break;
2284                         e1000_rar_set(hw, ha->addr, i++);
2285                 }
2286
2287         netdev_for_each_mc_addr(ha, netdev) {
2288                 if (i == rar_entries) {
2289                         /* load any remaining addresses into the hash table */
2290                         u32 hash_reg, hash_bit, mta;
2291                         hash_value = e1000_hash_mc_addr(hw, ha->addr);
2292                         hash_reg = (hash_value >> 5) & 0x7F;
2293                         hash_bit = hash_value & 0x1F;
2294                         mta = (1 << hash_bit);
2295                         mcarray[hash_reg] |= mta;
2296                 } else {
2297                         e1000_rar_set(hw, ha->addr, i++);
2298                 }
2299         }
2300
2301         for (; i < rar_entries; i++) {
2302                 E1000_WRITE_REG_ARRAY(hw, RA, i << 1, 0);
2303                 E1000_WRITE_FLUSH();
2304                 E1000_WRITE_REG_ARRAY(hw, RA, (i << 1) + 1, 0);
2305                 E1000_WRITE_FLUSH();
2306         }
2307
2308         /* write the hash table completely, write from bottom to avoid
2309          * both stupid write combining chipsets, and flushing each write
2310          */
2311         for (i = mta_reg_count - 1; i >= 0 ; i--) {
2312                 /* If we are on an 82544 has an errata where writing odd
2313                  * offsets overwrites the previous even offset, but writing
2314                  * backwards over the range solves the issue by always
2315                  * writing the odd offset first
2316                  */
2317                 E1000_WRITE_REG_ARRAY(hw, MTA, i, mcarray[i]);
2318         }
2319         E1000_WRITE_FLUSH();
2320
2321         if (hw->mac_type == e1000_82542_rev2_0)
2322                 e1000_leave_82542_rst(adapter);
2323
2324         kfree(mcarray);
2325 }
2326
2327 /**
2328  * e1000_update_phy_info_task - get phy info
2329  * @work: work struct contained inside adapter struct
2330  *
2331  * Need to wait a few seconds after link up to get diagnostic information from
2332  * the phy
2333  */
2334 static void e1000_update_phy_info_task(struct work_struct *work)
2335 {
2336         struct e1000_adapter *adapter = container_of(work,
2337                                                      struct e1000_adapter,
2338                                                      phy_info_task.work);
2339
2340         e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
2341 }
2342
2343 /**
2344  * e1000_82547_tx_fifo_stall_task - task to complete work
2345  * @work: work struct contained inside adapter struct
2346  **/
2347 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work)
2348 {
2349         struct e1000_adapter *adapter = container_of(work,
2350                                                      struct e1000_adapter,
2351                                                      fifo_stall_task.work);
2352         struct e1000_hw *hw = &adapter->hw;
2353         struct net_device *netdev = adapter->netdev;
2354         u32 tctl;
2355
2356         if (atomic_read(&adapter->tx_fifo_stall)) {
2357                 if ((er32(TDT) == er32(TDH)) &&
2358                    (er32(TDFT) == er32(TDFH)) &&
2359                    (er32(TDFTS) == er32(TDFHS))) {
2360                         tctl = er32(TCTL);
2361                         ew32(TCTL, tctl & ~E1000_TCTL_EN);
2362                         ew32(TDFT, adapter->tx_head_addr);
2363                         ew32(TDFH, adapter->tx_head_addr);
2364                         ew32(TDFTS, adapter->tx_head_addr);
2365                         ew32(TDFHS, adapter->tx_head_addr);
2366                         ew32(TCTL, tctl);
2367                         E1000_WRITE_FLUSH();
2368
2369                         adapter->tx_fifo_head = 0;
2370                         atomic_set(&adapter->tx_fifo_stall, 0);
2371                         netif_wake_queue(netdev);
2372                 } else if (!test_bit(__E1000_DOWN, &adapter->flags)) {
2373                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
2374                 }
2375         }
2376 }
2377
2378 bool e1000_has_link(struct e1000_adapter *adapter)
2379 {
2380         struct e1000_hw *hw = &adapter->hw;
2381         bool link_active = false;
2382
2383         /* get_link_status is set on LSC (link status) interrupt or rx
2384          * sequence error interrupt (except on intel ce4100).
2385          * get_link_status will stay false until the
2386          * e1000_check_for_link establishes link for copper adapters
2387          * ONLY
2388          */
2389         switch (hw->media_type) {
2390         case e1000_media_type_copper:
2391                 if (hw->mac_type == e1000_ce4100)
2392                         hw->get_link_status = 1;
2393                 if (hw->get_link_status) {
2394                         e1000_check_for_link(hw);
2395                         link_active = !hw->get_link_status;
2396                 } else {
2397                         link_active = true;
2398                 }
2399                 break;
2400         case e1000_media_type_fiber:
2401                 e1000_check_for_link(hw);
2402                 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
2403                 break;
2404         case e1000_media_type_internal_serdes:
2405                 e1000_check_for_link(hw);
2406                 link_active = hw->serdes_has_link;
2407                 break;
2408         default:
2409                 break;
2410         }
2411
2412         return link_active;
2413 }
2414
2415 /**
2416  * e1000_watchdog - work function
2417  * @work: work struct contained inside adapter struct
2418  **/
2419 static void e1000_watchdog(struct work_struct *work)
2420 {
2421         struct e1000_adapter *adapter = container_of(work,
2422                                                      struct e1000_adapter,
2423                                                      watchdog_task.work);
2424         struct e1000_hw *hw = &adapter->hw;
2425         struct net_device *netdev = adapter->netdev;
2426         struct e1000_tx_ring *txdr = adapter->tx_ring;
2427         u32 link, tctl;
2428
2429         link = e1000_has_link(adapter);
2430         if ((netif_carrier_ok(netdev)) && link)
2431                 goto link_up;
2432
2433         if (link) {
2434                 if (!netif_carrier_ok(netdev)) {
2435                         u32 ctrl;
2436                         /* update snapshot of PHY registers on LSC */
2437                         e1000_get_speed_and_duplex(hw,
2438                                                    &adapter->link_speed,
2439                                                    &adapter->link_duplex);
2440
2441                         ctrl = er32(CTRL);
2442                         pr_info("%s NIC Link is Up %d Mbps %s, "
2443                                 "Flow Control: %s\n",
2444                                 netdev->name,
2445                                 adapter->link_speed,
2446                                 adapter->link_duplex == FULL_DUPLEX ?
2447                                 "Full Duplex" : "Half Duplex",
2448                                 ((ctrl & E1000_CTRL_TFCE) && (ctrl &
2449                                 E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2450                                 E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2451                                 E1000_CTRL_TFCE) ? "TX" : "None")));
2452
2453                         /* adjust timeout factor according to speed/duplex */
2454                         adapter->tx_timeout_factor = 1;
2455                         switch (adapter->link_speed) {
2456                         case SPEED_10:
2457                                 adapter->tx_timeout_factor = 16;
2458                                 break;
2459                         case SPEED_100:
2460                                 /* maybe add some timeout factor ? */
2461                                 break;
2462                         }
2463
2464                         /* enable transmits in the hardware */
2465                         tctl = er32(TCTL);
2466                         tctl |= E1000_TCTL_EN;
2467                         ew32(TCTL, tctl);
2468
2469                         netif_carrier_on(netdev);
2470                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2471                                 schedule_delayed_work(&adapter->phy_info_task,
2472                                                       2 * HZ);
2473                         adapter->smartspeed = 0;
2474                 }
2475         } else {
2476                 if (netif_carrier_ok(netdev)) {
2477                         adapter->link_speed = 0;
2478                         adapter->link_duplex = 0;
2479                         pr_info("%s NIC Link is Down\n",
2480                                 netdev->name);
2481                         netif_carrier_off(netdev);
2482
2483                         if (!test_bit(__E1000_DOWN, &adapter->flags))
2484                                 schedule_delayed_work(&adapter->phy_info_task,
2485                                                       2 * HZ);
2486                 }
2487
2488                 e1000_smartspeed(adapter);
2489         }
2490
2491 link_up:
2492         e1000_update_stats(adapter);
2493
2494         hw->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2495         adapter->tpt_old = adapter->stats.tpt;
2496         hw->collision_delta = adapter->stats.colc - adapter->colc_old;
2497         adapter->colc_old = adapter->stats.colc;
2498
2499         adapter->gorcl = adapter->stats.gorcl - adapter->gorcl_old;
2500         adapter->gorcl_old = adapter->stats.gorcl;
2501         adapter->gotcl = adapter->stats.gotcl - adapter->gotcl_old;
2502         adapter->gotcl_old = adapter->stats.gotcl;
2503
2504         e1000_update_adaptive(hw);
2505
2506         if (!netif_carrier_ok(netdev)) {
2507                 if (E1000_DESC_UNUSED(txdr) + 1 < txdr->count) {
2508                         /* We've lost link, so the controller stops DMA,
2509                          * but we've got queued Tx work that's never going
2510                          * to get done, so reset controller to flush Tx.
2511                          * (Do the reset outside of interrupt context).
2512                          */
2513                         adapter->tx_timeout_count++;
2514                         schedule_work(&adapter->reset_task);
2515                         /* exit immediately since reset is imminent */
2516                         return;
2517                 }
2518         }
2519
2520         /* Simple mode for Interrupt Throttle Rate (ITR) */
2521         if (hw->mac_type >= e1000_82540 && adapter->itr_setting == 4) {
2522                 /* Symmetric Tx/Rx gets a reduced ITR=2000;
2523                  * Total asymmetrical Tx or Rx gets ITR=8000;
2524                  * everyone else is between 2000-8000.
2525                  */
2526                 u32 goc = (adapter->gotcl + adapter->gorcl) / 10000;
2527                 u32 dif = (adapter->gotcl > adapter->gorcl ?
2528                             adapter->gotcl - adapter->gorcl :
2529                             adapter->gorcl - adapter->gotcl) / 10000;
2530                 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
2531
2532                 ew32(ITR, 1000000000 / (itr * 256));
2533         }
2534
2535         /* Cause software interrupt to ensure rx ring is cleaned */
2536         ew32(ICS, E1000_ICS_RXDMT0);
2537
2538         /* Force detection of hung controller every watchdog period */
2539         adapter->detect_tx_hung = true;
2540
2541         /* Reschedule the task */
2542         if (!test_bit(__E1000_DOWN, &adapter->flags))
2543                 schedule_delayed_work(&adapter->watchdog_task, 2 * HZ);
2544 }
2545
2546 enum latency_range {
2547         lowest_latency = 0,
2548         low_latency = 1,
2549         bulk_latency = 2,
2550         latency_invalid = 255
2551 };
2552
2553 /**
2554  * e1000_update_itr - update the dynamic ITR value based on statistics
2555  * @adapter: pointer to adapter
2556  * @itr_setting: current adapter->itr
2557  * @packets: the number of packets during this measurement interval
2558  * @bytes: the number of bytes during this measurement interval
2559  *
2560  *      Stores a new ITR value based on packets and byte
2561  *      counts during the last interrupt.  The advantage of per interrupt
2562  *      computation is faster updates and more accurate ITR for the current
2563  *      traffic pattern.  Constants in this function were computed
2564  *      based on theoretical maximum wire speed and thresholds were set based
2565  *      on testing data as well as attempting to minimize response time
2566  *      while increasing bulk throughput.
2567  *      this functionality is controlled by the InterruptThrottleRate module
2568  *      parameter (see e1000_param.c)
2569  **/
2570 static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2571                                      u16 itr_setting, int packets, int bytes)
2572 {
2573         unsigned int retval = itr_setting;
2574         struct e1000_hw *hw = &adapter->hw;
2575
2576         if (unlikely(hw->mac_type < e1000_82540))
2577                 goto update_itr_done;
2578
2579         if (packets == 0)
2580                 goto update_itr_done;
2581
2582         switch (itr_setting) {
2583         case lowest_latency:
2584                 /* jumbo frames get bulk treatment*/
2585                 if (bytes/packets > 8000)
2586                         retval = bulk_latency;
2587                 else if ((packets < 5) && (bytes > 512))
2588                         retval = low_latency;
2589                 break;
2590         case low_latency:  /* 50 usec aka 20000 ints/s */
2591                 if (bytes > 10000) {
2592                         /* jumbo frames need bulk latency setting */
2593                         if (bytes/packets > 8000)
2594                                 retval = bulk_latency;
2595                         else if ((packets < 10) || ((bytes/packets) > 1200))
2596                                 retval = bulk_latency;
2597                         else if ((packets > 35))
2598                                 retval = lowest_latency;
2599                 } else if (bytes/packets > 2000)
2600                         retval = bulk_latency;
2601                 else if (packets <= 2 && bytes < 512)
2602                         retval = lowest_latency;
2603                 break;
2604         case bulk_latency: /* 250 usec aka 4000 ints/s */
2605                 if (bytes > 25000) {
2606                         if (packets > 35)
2607                                 retval = low_latency;
2608                 } else if (bytes < 6000) {
2609                         retval = low_latency;
2610                 }
2611                 break;
2612         }
2613
2614 update_itr_done:
2615         return retval;
2616 }
2617
2618 static void e1000_set_itr(struct e1000_adapter *adapter)
2619 {
2620         struct e1000_hw *hw = &adapter->hw;
2621         u16 current_itr;
2622         u32 new_itr = adapter->itr;
2623
2624         if (unlikely(hw->mac_type < e1000_82540))
2625                 return;
2626
2627         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2628         if (unlikely(adapter->link_speed != SPEED_1000)) {
2629                 current_itr = 0;
2630                 new_itr = 4000;
2631                 goto set_itr_now;
2632         }
2633
2634         adapter->tx_itr = e1000_update_itr(adapter, adapter->tx_itr,
2635                                            adapter->total_tx_packets,
2636                                            adapter->total_tx_bytes);
2637         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2638         if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2639                 adapter->tx_itr = low_latency;
2640
2641         adapter->rx_itr = e1000_update_itr(adapter, adapter->rx_itr,
2642                                            adapter->total_rx_packets,
2643                                            adapter->total_rx_bytes);
2644         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2645         if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2646                 adapter->rx_itr = low_latency;
2647
2648         current_itr = max(adapter->rx_itr, adapter->tx_itr);
2649
2650         switch (current_itr) {
2651         /* counts and packets in update_itr are dependent on these numbers */
2652         case lowest_latency:
2653                 new_itr = 70000;
2654                 break;
2655         case low_latency:
2656                 new_itr = 20000; /* aka hwitr = ~200 */
2657                 break;
2658         case bulk_latency:
2659                 new_itr = 4000;
2660                 break;
2661         default:
2662                 break;
2663         }
2664
2665 set_itr_now:
2666         if (new_itr != adapter->itr) {
2667                 /* this attempts to bias the interrupt rate towards Bulk
2668                  * by adding intermediate steps when interrupt rate is
2669                  * increasing
2670                  */
2671                 new_itr = new_itr > adapter->itr ?
2672                           min(adapter->itr + (new_itr >> 2), new_itr) :
2673                           new_itr;
2674                 adapter->itr = new_itr;
2675                 ew32(ITR, 1000000000 / (new_itr * 256));
2676         }
2677 }
2678
2679 #define E1000_TX_FLAGS_CSUM             0x00000001
2680 #define E1000_TX_FLAGS_VLAN             0x00000002
2681 #define E1000_TX_FLAGS_TSO              0x00000004
2682 #define E1000_TX_FLAGS_IPV4             0x00000008
2683 #define E1000_TX_FLAGS_NO_FCS           0x00000010
2684 #define E1000_TX_FLAGS_VLAN_MASK        0xffff0000
2685 #define E1000_TX_FLAGS_VLAN_SHIFT       16
2686
2687 static int e1000_tso(struct e1000_adapter *adapter,
2688                      struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2689                      __be16 protocol)
2690 {
2691         struct e1000_context_desc *context_desc;
2692         struct e1000_tx_buffer *buffer_info;
2693         unsigned int i;
2694         u32 cmd_length = 0;
2695         u16 ipcse = 0, tucse, mss;
2696         u8 ipcss, ipcso, tucss, tucso, hdr_len;
2697
2698         if (skb_is_gso(skb)) {
2699                 int err;
2700
2701                 err = skb_cow_head(skb, 0);
2702                 if (err < 0)
2703                         return err;
2704
2705                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2706                 mss = skb_shinfo(skb)->gso_size;
2707                 if (protocol == htons(ETH_P_IP)) {
2708                         struct iphdr *iph = ip_hdr(skb);
2709                         iph->tot_len = 0;
2710                         iph->check = 0;
2711                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2712                                                                  iph->daddr, 0,
2713                                                                  IPPROTO_TCP,
2714                                                                  0);
2715                         cmd_length = E1000_TXD_CMD_IP;
2716                         ipcse = skb_transport_offset(skb) - 1;
2717                 } else if (skb_is_gso_v6(skb)) {
2718                         ipv6_hdr(skb)->payload_len = 0;
2719                         tcp_hdr(skb)->check =
2720                                 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2721                                                  &ipv6_hdr(skb)->daddr,
2722                                                  0, IPPROTO_TCP, 0);
2723                         ipcse = 0;
2724                 }
2725                 ipcss = skb_network_offset(skb);
2726                 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
2727                 tucss = skb_transport_offset(skb);
2728                 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
2729                 tucse = 0;
2730
2731                 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
2732                                E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
2733
2734                 i = tx_ring->next_to_use;
2735                 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2736                 buffer_info = &tx_ring->buffer_info[i];
2737
2738                 context_desc->lower_setup.ip_fields.ipcss  = ipcss;
2739                 context_desc->lower_setup.ip_fields.ipcso  = ipcso;
2740                 context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
2741                 context_desc->upper_setup.tcp_fields.tucss = tucss;
2742                 context_desc->upper_setup.tcp_fields.tucso = tucso;
2743                 context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
2744                 context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
2745                 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
2746                 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
2747
2748                 buffer_info->time_stamp = jiffies;
2749                 buffer_info->next_to_watch = i;
2750
2751                 if (++i == tx_ring->count)
2752                         i = 0;
2753
2754                 tx_ring->next_to_use = i;
2755
2756                 return true;
2757         }
2758         return false;
2759 }
2760
2761 static bool e1000_tx_csum(struct e1000_adapter *adapter,
2762                           struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2763                           __be16 protocol)
2764 {
2765         struct e1000_context_desc *context_desc;
2766         struct e1000_tx_buffer *buffer_info;
2767         unsigned int i;
2768         u8 css;
2769         u32 cmd_len = E1000_TXD_CMD_DEXT;
2770
2771         if (skb->ip_summed != CHECKSUM_PARTIAL)
2772                 return false;
2773
2774         switch (protocol) {
2775         case cpu_to_be16(ETH_P_IP):
2776                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2777                         cmd_len |= E1000_TXD_CMD_TCP;
2778                 break;
2779         case cpu_to_be16(ETH_P_IPV6):
2780                 /* XXX not handling all IPV6 headers */
2781                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2782                         cmd_len |= E1000_TXD_CMD_TCP;
2783                 break;
2784         default:
2785                 if (unlikely(net_ratelimit()))
2786                         e_warn(drv, "checksum_partial proto=%x!\n",
2787                                skb->protocol);
2788                 break;
2789         }
2790
2791         css = skb_checksum_start_offset(skb);
2792
2793         i = tx_ring->next_to_use;
2794         buffer_info = &tx_ring->buffer_info[i];
2795         context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2796
2797         context_desc->lower_setup.ip_config = 0;
2798         context_desc->upper_setup.tcp_fields.tucss = css;
2799         context_desc->upper_setup.tcp_fields.tucso =
2800                 css + skb->csum_offset;
2801         context_desc->upper_setup.tcp_fields.tucse = 0;
2802         context_desc->tcp_seg_setup.data = 0;
2803         context_desc->cmd_and_length = cpu_to_le32(cmd_len);
2804
2805         buffer_info->time_stamp = jiffies;
2806         buffer_info->next_to_watch = i;
2807
2808         if (unlikely(++i == tx_ring->count))
2809                 i = 0;
2810
2811         tx_ring->next_to_use = i;
2812
2813         return true;
2814 }
2815
2816 #define E1000_MAX_TXD_PWR       12
2817 #define E1000_MAX_DATA_PER_TXD  (1<<E1000_MAX_TXD_PWR)
2818
2819 static int e1000_tx_map(struct e1000_adapter *adapter,
2820                         struct e1000_tx_ring *tx_ring,
2821                         struct sk_buff *skb, unsigned int first,
2822                         unsigned int max_per_txd, unsigned int nr_frags,
2823                         unsigned int mss)
2824 {
2825         struct e1000_hw *hw = &adapter->hw;
2826         struct pci_dev *pdev = adapter->pdev;
2827         struct e1000_tx_buffer *buffer_info;
2828         unsigned int len = skb_headlen(skb);
2829         unsigned int offset = 0, size, count = 0, i;
2830         unsigned int f, bytecount, segs;
2831
2832         i = tx_ring->next_to_use;
2833
2834         while (len) {
2835                 buffer_info = &tx_ring->buffer_info[i];
2836                 size = min(len, max_per_txd);
2837                 /* Workaround for Controller erratum --
2838                  * descriptor for non-tso packet in a linear SKB that follows a
2839                  * tso gets written back prematurely before the data is fully
2840                  * DMA'd to the controller
2841                  */
2842                 if (!skb->data_len && tx_ring->last_tx_tso &&
2843                     !skb_is_gso(skb)) {
2844                         tx_ring->last_tx_tso = false;
2845                         size -= 4;
2846                 }
2847
2848                 /* Workaround for premature desc write-backs
2849                  * in TSO mode.  Append 4-byte sentinel desc
2850                  */
2851                 if (unlikely(mss && !nr_frags && size == len && size > 8))
2852                         size -= 4;
2853                 /* work-around for errata 10 and it applies
2854                  * to all controllers in PCI-X mode
2855                  * The fix is to make sure that the first descriptor of a
2856                  * packet is smaller than 2048 - 16 - 16 (or 2016) bytes
2857                  */
2858                 if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
2859                              (size > 2015) && count == 0))
2860                         size = 2015;
2861
2862                 /* Workaround for potential 82544 hang in PCI-X.  Avoid
2863                  * terminating buffers within evenly-aligned dwords.
2864                  */
2865                 if (unlikely(adapter->pcix_82544 &&
2866                    !((unsigned long)(skb->data + offset + size - 1) & 4) &&
2867                    size > 4))
2868                         size -= 4;
2869
2870                 buffer_info->length = size;
2871                 /* set time_stamp *before* dma to help avoid a possible race */
2872                 buffer_info->time_stamp = jiffies;
2873                 buffer_info->mapped_as_page = false;
2874                 buffer_info->dma = dma_map_single(&pdev->dev,
2875                                                   skb->data + offset,
2876                                                   size, DMA_TO_DEVICE);
2877                 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2878                         goto dma_error;
2879                 buffer_info->next_to_watch = i;
2880
2881                 len -= size;
2882                 offset += size;
2883                 count++;
2884                 if (len) {
2885                         i++;
2886                         if (unlikely(i == tx_ring->count))
2887                                 i = 0;
2888                 }
2889         }
2890
2891         for (f = 0; f < nr_frags; f++) {
2892                 const struct skb_frag_struct *frag;
2893
2894                 frag = &skb_shinfo(skb)->frags[f];
2895                 len = skb_frag_size(frag);
2896                 offset = 0;
2897
2898                 while (len) {
2899                         unsigned long bufend;
2900                         i++;
2901                         if (unlikely(i == tx_ring->count))
2902                                 i = 0;
2903
2904                         buffer_info = &tx_ring->buffer_info[i];
2905                         size = min(len, max_per_txd);
2906                         /* Workaround for premature desc write-backs
2907                          * in TSO mode.  Append 4-byte sentinel desc
2908                          */
2909                         if (unlikely(mss && f == (nr_frags-1) &&
2910                             size == len && size > 8))
2911                                 size -= 4;
2912                         /* Workaround for potential 82544 hang in PCI-X.
2913                          * Avoid terminating buffers within evenly-aligned
2914                          * dwords.
2915                          */
2916                         bufend = (unsigned long)
2917                                 page_to_phys(skb_frag_page(frag));
2918                         bufend += offset + size - 1;
2919                         if (unlikely(adapter->pcix_82544 &&
2920                                      !(bufend & 4) &&
2921                                      size > 4))
2922                                 size -= 4;
2923
2924                         buffer_info->length = size;
2925                         buffer_info->time_stamp = jiffies;
2926                         buffer_info->mapped_as_page = true;
2927                         buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
2928                                                 offset, size, DMA_TO_DEVICE);
2929                         if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2930                                 goto dma_error;
2931                         buffer_info->next_to_watch = i;
2932
2933                         len -= size;
2934                         offset += size;
2935                         count++;
2936                 }
2937         }
2938
2939         segs = skb_shinfo(skb)->gso_segs ?: 1;
2940         /* multiply data chunks by size of headers */
2941         bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
2942
2943         tx_ring->buffer_info[i].skb = skb;
2944         tx_ring->buffer_info[i].segs = segs;
2945         tx_ring->buffer_info[i].bytecount = bytecount;
2946         tx_ring->buffer_info[first].next_to_watch = i;
2947
2948         return count;
2949
2950 dma_error:
2951         dev_err(&pdev->dev, "TX DMA map failed\n");
2952         buffer_info->dma = 0;
2953         if (count)
2954                 count--;
2955
2956         while (count--) {
2957                 if (i == 0)
2958                         i += tx_ring->count;
2959                 i--;
2960                 buffer_info = &tx_ring->buffer_info[i];
2961                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
2962         }
2963
2964         return 0;
2965 }
2966
2967 static void e1000_tx_queue(struct e1000_adapter *adapter,
2968                            struct e1000_tx_ring *tx_ring, int tx_flags,
2969                            int count)
2970 {
2971         struct e1000_tx_desc *tx_desc = NULL;
2972         struct e1000_tx_buffer *buffer_info;
2973         u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
2974         unsigned int i;
2975
2976         if (likely(tx_flags & E1000_TX_FLAGS_TSO)) {
2977                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
2978                              E1000_TXD_CMD_TSE;
2979                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2980
2981                 if (likely(tx_flags & E1000_TX_FLAGS_IPV4))
2982                         txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2983         }
2984
2985         if (likely(tx_flags & E1000_TX_FLAGS_CSUM)) {
2986                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2987                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2988         }
2989
2990         if (unlikely(tx_flags & E1000_TX_FLAGS_VLAN)) {
2991                 txd_lower |= E1000_TXD_CMD_VLE;
2992                 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
2993         }
2994
2995         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
2996                 txd_lower &= ~(E1000_TXD_CMD_IFCS);
2997
2998         i = tx_ring->next_to_use;
2999
3000         while (count--) {
3001                 buffer_info = &tx_ring->buffer_info[i];
3002                 tx_desc = E1000_TX_DESC(*tx_ring, i);
3003                 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
3004                 tx_desc->lower.data =
3005                         cpu_to_le32(txd_lower | buffer_info->length);
3006                 tx_desc->upper.data = cpu_to_le32(txd_upper);
3007                 if (unlikely(++i == tx_ring->count))
3008                         i = 0;
3009         }
3010
3011         tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
3012
3013         /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
3014         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3015                 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
3016
3017         /* Force memory writes to complete before letting h/w
3018          * know there are new descriptors to fetch.  (Only
3019          * applicable for weak-ordered memory model archs,
3020          * such as IA-64).
3021          */
3022         wmb();
3023
3024         tx_ring->next_to_use = i;
3025 }
3026
3027 /* 82547 workaround to avoid controller hang in half-duplex environment.
3028  * The workaround is to avoid queuing a large packet that would span
3029  * the internal Tx FIFO ring boundary by notifying the stack to resend
3030  * the packet at a later time.  This gives the Tx FIFO an opportunity to
3031  * flush all packets.  When that occurs, we reset the Tx FIFO pointers
3032  * to the beginning of the Tx FIFO.
3033  */
3034
3035 #define E1000_FIFO_HDR                  0x10
3036 #define E1000_82547_PAD_LEN             0x3E0
3037
3038 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
3039                                        struct sk_buff *skb)
3040 {
3041         u32 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
3042         u32 skb_fifo_len = skb->len + E1000_FIFO_HDR;
3043
3044         skb_fifo_len = ALIGN(skb_fifo_len, E1000_FIFO_HDR);
3045
3046         if (adapter->link_duplex != HALF_DUPLEX)
3047                 goto no_fifo_stall_required;
3048
3049         if (atomic_read(&adapter->tx_fifo_stall))
3050                 return 1;
3051
3052         if (skb_fifo_len >= (E1000_82547_PAD_LEN + fifo_space)) {
3053                 atomic_set(&adapter->tx_fifo_stall, 1);
3054                 return 1;
3055         }
3056
3057 no_fifo_stall_required:
3058         adapter->tx_fifo_head += skb_fifo_len;
3059         if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
3060                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
3061         return 0;
3062 }
3063
3064 static int __e1000_maybe_stop_tx(struct net_device *netdev, int size)
3065 {
3066         struct e1000_adapter *adapter = netdev_priv(netdev);
3067         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3068
3069         netif_stop_queue(netdev);
3070         /* Herbert's original patch had:
3071          *  smp_mb__after_netif_stop_queue();
3072          * but since that doesn't exist yet, just open code it.
3073          */
3074         smp_mb();
3075
3076         /* We need to check again in a case another CPU has just
3077          * made room available.
3078          */
3079         if (likely(E1000_DESC_UNUSED(tx_ring) < size))
3080                 return -EBUSY;
3081
3082         /* A reprieve! */
3083         netif_start_queue(netdev);
3084         ++adapter->restart_queue;
3085         return 0;
3086 }
3087
3088 static int e1000_maybe_stop_tx(struct net_device *netdev,
3089                                struct e1000_tx_ring *tx_ring, int size)
3090 {
3091         if (likely(E1000_DESC_UNUSED(tx_ring) >= size))
3092                 return 0;
3093         return __e1000_maybe_stop_tx(netdev, size);
3094 }
3095
3096 #define TXD_USE_COUNT(S, X) (((S) + ((1 << (X)) - 1)) >> (X))
3097 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
3098                                     struct net_device *netdev)
3099 {
3100         struct e1000_adapter *adapter = netdev_priv(netdev);
3101         struct e1000_hw *hw = &adapter->hw;
3102         struct e1000_tx_ring *tx_ring;
3103         unsigned int first, max_per_txd = E1000_MAX_DATA_PER_TXD;
3104         unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
3105         unsigned int tx_flags = 0;
3106         unsigned int len = skb_headlen(skb);
3107         unsigned int nr_frags;
3108         unsigned int mss;
3109         int count = 0;
3110         int tso;
3111         unsigned int f;
3112         __be16 protocol = vlan_get_protocol(skb);
3113
3114         /* This goes back to the question of how to logically map a Tx queue
3115          * to a flow.  Right now, performance is impacted slightly negatively
3116          * if using multiple Tx queues.  If the stack breaks away from a
3117          * single qdisc implementation, we can look at this again.
3118          */
3119         tx_ring = adapter->tx_ring;
3120
3121         /* On PCI/PCI-X HW, if packet size is less than ETH_ZLEN,
3122          * packets may get corrupted during padding by HW.
3123          * To WA this issue, pad all small packets manually.
3124          */
3125         if (eth_skb_pad(skb))
3126                 return NETDEV_TX_OK;
3127
3128         mss = skb_shinfo(skb)->gso_size;
3129         /* The controller does a simple calculation to
3130          * make sure there is enough room in the FIFO before
3131          * initiating the DMA for each buffer.  The calc is:
3132          * 4 = ceil(buffer len/mss).  To make sure we don't
3133          * overrun the FIFO, adjust the max buffer len if mss
3134          * drops.
3135          */
3136         if (mss) {
3137                 u8 hdr_len;
3138                 max_per_txd = min(mss << 2, max_per_txd);
3139                 max_txd_pwr = fls(max_per_txd) - 1;
3140
3141                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3142                 if (skb->data_len && hdr_len == len) {
3143                         switch (hw->mac_type) {
3144                                 unsigned int pull_size;
3145                         case e1000_82544:
3146                                 /* Make sure we have room to chop off 4 bytes,
3147                                  * and that the end alignment will work out to
3148                                  * this hardware's requirements
3149                                  * NOTE: this is a TSO only workaround
3150                                  * if end byte alignment not correct move us
3151                                  * into the next dword
3152                                  */
3153                                 if ((unsigned long)(skb_tail_pointer(skb) - 1)
3154                                     & 4)
3155                                         break;
3156                                 /* fall through */
3157                                 pull_size = min((unsigned int)4, skb->data_len);
3158                                 if (!__pskb_pull_tail(skb, pull_size)) {
3159                                         e_err(drv, "__pskb_pull_tail "
3160                                               "failed.\n");
3161                                         dev_kfree_skb_any(skb);
3162                                         return NETDEV_TX_OK;
3163                                 }
3164                                 len = skb_headlen(skb);
3165                                 break;
3166                         default:
3167                                 /* do nothing */
3168                                 break;
3169                         }
3170                 }
3171         }
3172
3173         /* reserve a descriptor for the offload context */
3174         if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
3175                 count++;
3176         count++;
3177
3178         /* Controller Erratum workaround */
3179         if (!skb->data_len && tx_ring->last_tx_tso && !skb_is_gso(skb))
3180                 count++;
3181
3182         count += TXD_USE_COUNT(len, max_txd_pwr);
3183
3184         if (adapter->pcix_82544)
3185                 count++;
3186
3187         /* work-around for errata 10 and it applies to all controllers
3188          * in PCI-X mode, so add one more descriptor to the count
3189          */
3190         if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
3191                         (len > 2015)))
3192                 count++;
3193
3194         nr_frags = skb_shinfo(skb)->nr_frags;
3195         for (f = 0; f < nr_frags; f++)
3196                 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]),
3197                                        max_txd_pwr);
3198         if (adapter->pcix_82544)
3199                 count += nr_frags;
3200
3201         /* need: count + 2 desc gap to keep tail from touching
3202          * head, otherwise try next time
3203          */
3204         if (unlikely(e1000_maybe_stop_tx(netdev, tx_ring, count + 2)))
3205                 return NETDEV_TX_BUSY;
3206
3207         if (unlikely((hw->mac_type == e1000_82547) &&
3208                      (e1000_82547_fifo_workaround(adapter, skb)))) {
3209                 netif_stop_queue(netdev);
3210                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3211                         schedule_delayed_work(&adapter->fifo_stall_task, 1);
3212                 return NETDEV_TX_BUSY;
3213         }
3214
3215         if (skb_vlan_tag_present(skb)) {
3216                 tx_flags |= E1000_TX_FLAGS_VLAN;
3217                 tx_flags |= (skb_vlan_tag_get(skb) <<
3218                              E1000_TX_FLAGS_VLAN_SHIFT);
3219         }
3220
3221         first = tx_ring->next_to_use;
3222
3223         tso = e1000_tso(adapter, tx_ring, skb, protocol);
3224         if (tso < 0) {
3225                 dev_kfree_skb_any(skb);
3226                 return NETDEV_TX_OK;
3227         }
3228
3229         if (likely(tso)) {
3230                 if (likely(hw->mac_type != e1000_82544))
3231                         tx_ring->last_tx_tso = true;
3232                 tx_flags |= E1000_TX_FLAGS_TSO;
3233         } else if (likely(e1000_tx_csum(adapter, tx_ring, skb, protocol)))
3234                 tx_flags |= E1000_TX_FLAGS_CSUM;
3235
3236         if (protocol == htons(ETH_P_IP))
3237                 tx_flags |= E1000_TX_FLAGS_IPV4;
3238
3239         if (unlikely(skb->no_fcs))
3240                 tx_flags |= E1000_TX_FLAGS_NO_FCS;
3241
3242         count = e1000_tx_map(adapter, tx_ring, skb, first, max_per_txd,
3243                              nr_frags, mss);
3244
3245         if (count) {
3246                 /* The descriptors needed is higher than other Intel drivers
3247                  * due to a number of workarounds.  The breakdown is below:
3248                  * Data descriptors: MAX_SKB_FRAGS + 1
3249                  * Context Descriptor: 1
3250                  * Keep head from touching tail: 2
3251                  * Workarounds: 3
3252                  */
3253                 int desc_needed = MAX_SKB_FRAGS + 7;
3254
3255                 netdev_sent_queue(netdev, skb->len);
3256                 skb_tx_timestamp(skb);
3257
3258                 e1000_tx_queue(adapter, tx_ring, tx_flags, count);
3259
3260                 /* 82544 potentially requires twice as many data descriptors
3261                  * in order to guarantee buffers don't end on evenly-aligned
3262                  * dwords
3263                  */
3264                 if (adapter->pcix_82544)
3265                         desc_needed += MAX_SKB_FRAGS + 1;
3266
3267                 /* Make sure there is space in the ring for the next send. */
3268                 e1000_maybe_stop_tx(netdev, tx_ring, desc_needed);
3269
3270                 if (!skb->xmit_more ||
3271                     netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) {
3272                         writel(tx_ring->next_to_use, hw->hw_addr + tx_ring->tdt);
3273                         /* we need this if more than one processor can write to
3274                          * our tail at a time, it synchronizes IO on IA64/Altix
3275                          * systems
3276                          */
3277                         mmiowb();
3278                 }
3279         } else {
3280                 dev_kfree_skb_any(skb);
3281                 tx_ring->buffer_info[first].time_stamp = 0;
3282                 tx_ring->next_to_use = first;
3283         }
3284
3285         return NETDEV_TX_OK;
3286 }
3287
3288 #define NUM_REGS 38 /* 1 based count */
3289 static void e1000_regdump(struct e1000_adapter *adapter)
3290 {
3291         struct e1000_hw *hw = &adapter->hw;
3292         u32 regs[NUM_REGS];
3293         u32 *regs_buff = regs;
3294         int i = 0;
3295
3296         static const char * const reg_name[] = {
3297                 "CTRL",  "STATUS",
3298                 "RCTL", "RDLEN", "RDH", "RDT", "RDTR",
3299                 "TCTL", "TDBAL", "TDBAH", "TDLEN", "TDH", "TDT",
3300                 "TIDV", "TXDCTL", "TADV", "TARC0",
3301                 "TDBAL1", "TDBAH1", "TDLEN1", "TDH1", "TDT1",
3302                 "TXDCTL1", "TARC1",
3303                 "CTRL_EXT", "ERT", "RDBAL", "RDBAH",
3304                 "TDFH", "TDFT", "TDFHS", "TDFTS", "TDFPC",
3305                 "RDFH", "RDFT", "RDFHS", "RDFTS", "RDFPC"
3306         };
3307
3308         regs_buff[0]  = er32(CTRL);
3309         regs_buff[1]  = er32(STATUS);
3310
3311         regs_buff[2]  = er32(RCTL);
3312         regs_buff[3]  = er32(RDLEN);
3313         regs_buff[4]  = er32(RDH);
3314         regs_buff[5]  = er32(RDT);
3315         regs_buff[6]  = er32(RDTR);
3316
3317         regs_buff[7]  = er32(TCTL);
3318         regs_buff[8]  = er32(TDBAL);
3319         regs_buff[9]  = er32(TDBAH);
3320         regs_buff[10] = er32(TDLEN);
3321         regs_buff[11] = er32(TDH);
3322         regs_buff[12] = er32(TDT);
3323         regs_buff[13] = er32(TIDV);
3324         regs_buff[14] = er32(TXDCTL);
3325         regs_buff[15] = er32(TADV);
3326         regs_buff[16] = er32(TARC0);
3327
3328         regs_buff[17] = er32(TDBAL1);
3329         regs_buff[18] = er32(TDBAH1);
3330         regs_buff[19] = er32(TDLEN1);
3331         regs_buff[20] = er32(TDH1);
3332         regs_buff[21] = er32(TDT1);
3333         regs_buff[22] = er32(TXDCTL1);
3334         regs_buff[23] = er32(TARC1);
3335         regs_buff[24] = er32(CTRL_EXT);
3336         regs_buff[25] = er32(ERT);
3337         regs_buff[26] = er32(RDBAL0);
3338         regs_buff[27] = er32(RDBAH0);
3339         regs_buff[28] = er32(TDFH);
3340         regs_buff[29] = er32(TDFT);
3341         regs_buff[30] = er32(TDFHS);
3342         regs_buff[31] = er32(TDFTS);
3343         regs_buff[32] = er32(TDFPC);
3344         regs_buff[33] = er32(RDFH);
3345         regs_buff[34] = er32(RDFT);
3346         regs_buff[35] = er32(RDFHS);
3347         regs_buff[36] = er32(RDFTS);
3348         regs_buff[37] = er32(RDFPC);
3349
3350         pr_info("Register dump\n");
3351         for (i = 0; i < NUM_REGS; i++)
3352                 pr_info("%-15s  %08x\n", reg_name[i], regs_buff[i]);
3353 }
3354
3355 /*
3356  * e1000_dump: Print registers, tx ring and rx ring
3357  */
3358 static void e1000_dump(struct e1000_adapter *adapter)
3359 {
3360         /* this code doesn't handle multiple rings */
3361         struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3362         struct e1000_rx_ring *rx_ring = adapter->rx_ring;
3363         int i;
3364
3365         if (!netif_msg_hw(adapter))
3366                 return;
3367
3368         /* Print Registers */
3369         e1000_regdump(adapter);
3370
3371         /* transmit dump */
3372         pr_info("TX Desc ring0 dump\n");
3373
3374         /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
3375          *
3376          * Legacy Transmit Descriptor
3377          *   +--------------------------------------------------------------+
3378          * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
3379          *   +--------------------------------------------------------------+
3380          * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
3381          *   +--------------------------------------------------------------+
3382          *   63       48 47        36 35    32 31     24 23    16 15        0
3383          *
3384          * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
3385          *   63      48 47    40 39       32 31             16 15    8 7      0
3386          *   +----------------------------------------------------------------+
3387          * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
3388          *   +----------------------------------------------------------------+
3389          * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
3390          *   +----------------------------------------------------------------+
3391          *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
3392          *
3393          * Extended Data Descriptor (DTYP=0x1)
3394          *   +----------------------------------------------------------------+
3395          * 0 |                     Buffer Address [63:0]                      |
3396          *   +----------------------------------------------------------------+
3397          * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
3398          *   +----------------------------------------------------------------+
3399          *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
3400          */
3401         pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3402         pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3403
3404         if (!netif_msg_tx_done(adapter))
3405                 goto rx_ring_summary;
3406
3407         for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
3408                 struct e1000_tx_desc *tx_desc = E1000_TX_DESC(*tx_ring, i);
3409                 struct e1000_tx_buffer *buffer_info = &tx_ring->buffer_info[i];
3410                 struct my_u { __le64 a; __le64 b; };
3411                 struct my_u *u = (struct my_u *)tx_desc;
3412                 const char *type;
3413
3414                 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
3415                         type = "NTC/U";
3416                 else if (i == tx_ring->next_to_use)
3417                         type = "NTU";
3418                 else if (i == tx_ring->next_to_clean)
3419                         type = "NTC";
3420                 else
3421                         type = "";
3422
3423                 pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p %s\n",
3424                         ((le64_to_cpu(u->b) & (1<<20)) ? 'd' : 'c'), i,
3425                         le64_to_cpu(u->a), le64_to_cpu(u->b),
3426                         (u64)buffer_info->dma, buffer_info->length,
3427                         buffer_info->next_to_watch,
3428                         (u64)buffer_info->time_stamp, buffer_info->skb, type);
3429         }
3430
3431 rx_ring_summary:
3432         /* receive dump */
3433         pr_info("\nRX Desc ring dump\n");
3434
3435         /* Legacy Receive Descriptor Format
3436          *
3437          * +-----------------------------------------------------+
3438          * |                Buffer Address [63:0]                |
3439          * +-----------------------------------------------------+
3440          * | VLAN Tag | Errors | Status 0 | Packet csum | Length |
3441          * +-----------------------------------------------------+
3442          * 63       48 47    40 39      32 31         16 15      0
3443          */
3444         pr_info("R[desc]      [address 63:0  ] [vl er S cks ln] [bi->dma       ] [bi->skb]\n");
3445
3446         if (!netif_msg_rx_status(adapter))
3447                 goto exit;
3448
3449         for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) {
3450                 struct e1000_rx_desc *rx_desc = E1000_RX_DESC(*rx_ring, i);
3451                 struct e1000_rx_buffer *buffer_info = &rx_ring->buffer_info[i];
3452                 struct my_u { __le64 a; __le64 b; };
3453                 struct my_u *u = (struct my_u *)rx_desc;
3454                 const char *type;
3455
3456                 if (i == rx_ring->next_to_use)
3457                         type = "NTU";
3458                 else if (i == rx_ring->next_to_clean)
3459                         type = "NTC";
3460                 else
3461                         type = "";
3462
3463                 pr_info("R[0x%03X]     %016llX %016llX %016llX %p %s\n",
3464                         i, le64_to_cpu(u->a), le64_to_cpu(u->b),
3465                         (u64)buffer_info->dma, buffer_info->rxbuf.data, type);
3466         } /* for */
3467
3468         /* dump the descriptor caches */
3469         /* rx */
3470         pr_info("Rx descriptor cache in 64bit format\n");
3471         for (i = 0x6000; i <= 0x63FF ; i += 0x10) {
3472                 pr_info("R%04X: %08X|%08X %08X|%08X\n",
3473                         i,
3474                         readl(adapter->hw.hw_addr + i+4),
3475                         readl(adapter->hw.hw_addr + i),
3476                         readl(adapter->hw.hw_addr + i+12),
3477                         readl(adapter->hw.hw_addr + i+8));
3478         }
3479         /* tx */
3480         pr_info("Tx descriptor cache in 64bit format\n");
3481         for (i = 0x7000; i <= 0x73FF ; i += 0x10) {
3482                 pr_info("T%04X: %08X|%08X %08X|%08X\n",
3483                         i,
3484                         readl(adapter->hw.hw_addr + i+4),
3485                         readl(adapter->hw.hw_addr + i),
3486                         readl(adapter->hw.hw_addr + i+12),
3487                         readl(adapter->hw.hw_addr + i+8));
3488         }
3489 exit:
3490         return;
3491 }
3492
3493 /**
3494  * e1000_tx_timeout - Respond to a Tx Hang
3495  * @netdev: network interface device structure
3496  **/
3497 static void e1000_tx_timeout(struct net_device *netdev)
3498 {
3499         struct e1000_adapter *adapter = netdev_priv(netdev);
3500
3501         /* Do the reset outside of interrupt context */
3502         adapter->tx_timeout_count++;
3503         schedule_work(&adapter->reset_task);
3504 }
3505
3506 static void e1000_reset_task(struct work_struct *work)
3507 {
3508         struct e1000_adapter *adapter =
3509                 container_of(work, struct e1000_adapter, reset_task);
3510
3511         e_err(drv, "Reset adapter\n");
3512         e1000_reinit_locked(adapter);
3513 }
3514
3515 /**
3516  * e1000_change_mtu - Change the Maximum Transfer Unit
3517  * @netdev: network interface device structure
3518  * @new_mtu: new value for maximum frame size
3519  *
3520  * Returns 0 on success, negative on failure
3521  **/
3522 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
3523 {
3524         struct e1000_adapter *adapter = netdev_priv(netdev);
3525         struct e1000_hw *hw = &adapter->hw;
3526         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3527
3528         /* Adapter-specific max frame size limits. */
3529         switch (hw->mac_type) {
3530         case e1000_undefined ... e1000_82542_rev2_1:
3531                 if (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)) {
3532                         e_err(probe, "Jumbo Frames not supported.\n");
3533                         return -EINVAL;
3534                 }
3535                 break;
3536         default:
3537                 /* Capable of supporting up to MAX_JUMBO_FRAME_SIZE limit. */
3538                 break;
3539         }
3540
3541         while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
3542                 msleep(1);
3543         /* e1000_down has a dependency on max_frame_size */
3544         hw->max_frame_size = max_frame;
3545         if (netif_running(netdev)) {
3546                 /* prevent buffers from being reallocated */
3547                 adapter->alloc_rx_buf = e1000_alloc_dummy_rx_buffers;
3548                 e1000_down(adapter);
3549         }
3550
3551         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3552          * means we reserve 2 more, this pushes us to allocate from the next
3553          * larger slab size.
3554          * i.e. RXBUFFER_2048 --> size-4096 slab
3555          * however with the new *_jumbo_rx* routines, jumbo receives will use
3556          * fragmented skbs
3557          */
3558
3559         if (max_frame <= E1000_RXBUFFER_2048)
3560                 adapter->rx_buffer_len = E1000_RXBUFFER_2048;
3561         else
3562 #if (PAGE_SIZE >= E1000_RXBUFFER_16384)
3563                 adapter->rx_buffer_len = E1000_RXBUFFER_16384;
3564 #elif (PAGE_SIZE >= E1000_RXBUFFER_4096)
3565                 adapter->rx_buffer_len = PAGE_SIZE;
3566 #endif
3567
3568         /* adjust allocation if LPE protects us, and we aren't using SBP */
3569         if (!hw->tbi_compatibility_on &&
3570             ((max_frame == (ETH_FRAME_LEN + ETH_FCS_LEN)) ||
3571              (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE)))
3572                 adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3573
3574         pr_info("%s changing MTU from %d to %d\n",
3575                 netdev->name, netdev->mtu, new_mtu);
3576         netdev->mtu = new_mtu;
3577
3578         if (netif_running(netdev))
3579                 e1000_up(adapter);
3580         else
3581                 e1000_reset(adapter);
3582
3583         clear_bit(__E1000_RESETTING, &adapter->flags);
3584
3585         return 0;
3586 }
3587
3588 /**
3589  * e1000_update_stats - Update the board statistics counters
3590  * @adapter: board private structure
3591  **/
3592 void e1000_update_stats(struct e1000_adapter *adapter)
3593 {
3594         struct net_device *netdev = adapter->netdev;
3595         struct e1000_hw *hw = &adapter->hw;
3596         struct pci_dev *pdev = adapter->pdev;
3597         unsigned long flags;
3598         u16 phy_tmp;
3599
3600 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3601
3602         /* Prevent stats update while adapter is being reset, or if the pci
3603          * connection is down.
3604          */
3605         if (adapter->link_speed == 0)
3606                 return;
3607         if (pci_channel_offline(pdev))
3608                 return;
3609
3610         spin_lock_irqsave(&adapter->stats_lock, flags);
3611
3612         /* these counters are modified from e1000_tbi_adjust_stats,
3613          * called from the interrupt context, so they must only
3614          * be written while holding adapter->stats_lock
3615          */
3616
3617         adapter->stats.crcerrs += er32(CRCERRS);
3618         adapter->stats.gprc += er32(GPRC);
3619         adapter->stats.gorcl += er32(GORCL);
3620         adapter->stats.gorch += er32(GORCH);
3621         adapter->stats.bprc += er32(BPRC);
3622         adapter->stats.mprc += er32(MPRC);
3623         adapter->stats.roc += er32(ROC);
3624
3625         adapter->stats.prc64 += er32(PRC64);
3626         adapter->stats.prc127 += er32(PRC127);
3627         adapter->stats.prc255 += er32(PRC255);
3628         adapter->stats.prc511 += er32(PRC511);
3629         adapter->stats.prc1023 += er32(PRC1023);
3630         adapter->stats.prc1522 += er32(PRC1522);
3631
3632         adapter->stats.symerrs += er32(SYMERRS);
3633         adapter->stats.mpc += er32(MPC);
3634         adapter->stats.scc += er32(SCC);
3635         adapter->stats.ecol += er32(ECOL);
3636         adapter->stats.mcc += er32(MCC);
3637         adapter->stats.latecol += er32(LATECOL);
3638         adapter->stats.dc += er32(DC);
3639         adapter->stats.sec += er32(SEC);
3640         adapter->stats.rlec += er32(RLEC);
3641         adapter->stats.xonrxc += er32(XONRXC);
3642         adapter->stats.xontxc += er32(XONTXC);
3643         adapter->stats.xoffrxc += er32(XOFFRXC);
3644         adapter->stats.xofftxc += er32(XOFFTXC);
3645         adapter->stats.fcruc += er32(FCRUC);
3646         adapter->stats.gptc += er32(GPTC);
3647         adapter->stats.gotcl += er32(GOTCL);
3648         adapter->stats.gotch += er32(GOTCH);
3649         adapter->stats.rnbc += er32(RNBC);
3650         adapter->stats.ruc += er32(RUC);
3651         adapter->stats.rfc += er32(RFC);
3652         adapter->stats.rjc += er32(RJC);
3653         adapter->stats.torl += er32(TORL);
3654         adapter->stats.torh += er32(TORH);
3655         adapter->stats.totl += er32(TOTL);
3656         adapter->stats.toth += er32(TOTH);
3657         adapter->stats.tpr += er32(TPR);
3658
3659         adapter->stats.ptc64 += er32(PTC64);
3660         adapter->stats.ptc127 += er32(PTC127);
3661         adapter->stats.ptc255 += er32(PTC255);
3662         adapter->stats.ptc511 += er32(PTC511);
3663         adapter->stats.ptc1023 += er32(PTC1023);
3664         adapter->stats.ptc1522 += er32(PTC1522);
3665
3666         adapter->stats.mptc += er32(MPTC);
3667         adapter->stats.bptc += er32(BPTC);
3668
3669         /* used for adaptive IFS */
3670
3671         hw->tx_packet_delta = er32(TPT);
3672         adapter->stats.tpt += hw->tx_packet_delta;
3673         hw->collision_delta = er32(COLC);
3674         adapter->stats.colc += hw->collision_delta;
3675
3676         if (hw->mac_type >= e1000_82543) {
3677                 adapter->stats.algnerrc += er32(ALGNERRC);
3678                 adapter->stats.rxerrc += er32(RXERRC);
3679                 adapter->stats.tncrs += er32(TNCRS);
3680                 adapter->stats.cexterr += er32(CEXTERR);
3681                 adapter->stats.tsctc += er32(TSCTC);
3682                 adapter->stats.tsctfc += er32(TSCTFC);
3683         }
3684
3685         /* Fill out the OS statistics structure */
3686         netdev->stats.multicast = adapter->stats.mprc;
3687         netdev->stats.collisions = adapter->stats.colc;
3688
3689         /* Rx Errors */
3690
3691         /* RLEC on some newer hardware can be incorrect so build
3692          * our own version based on RUC and ROC
3693          */
3694         netdev->stats.rx_errors = adapter->stats.rxerrc +
3695                 adapter->stats.crcerrs + adapter->stats.algnerrc +
3696                 adapter->stats.ruc + adapter->stats.roc +
3697                 adapter->stats.cexterr;
3698         adapter->stats.rlerrc = adapter->stats.ruc + adapter->stats.roc;
3699         netdev->stats.rx_length_errors = adapter->stats.rlerrc;
3700         netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3701         netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3702         netdev->stats.rx_missed_errors = adapter->stats.mpc;
3703
3704         /* Tx Errors */
3705         adapter->stats.txerrc = adapter->stats.ecol + adapter->stats.latecol;
3706         netdev->stats.tx_errors = adapter->stats.txerrc;
3707         netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3708         netdev->stats.tx_window_errors = adapter->stats.latecol;
3709         netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
3710         if (hw->bad_tx_carr_stats_fd &&
3711             adapter->link_duplex == FULL_DUPLEX) {
3712                 netdev->stats.tx_carrier_errors = 0;
3713                 adapter->stats.tncrs = 0;
3714         }
3715
3716         /* Tx Dropped needs to be maintained elsewhere */
3717
3718         /* Phy Stats */
3719         if (hw->media_type == e1000_media_type_copper) {
3720                 if ((adapter->link_speed == SPEED_1000) &&
3721                    (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3722                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3723                         adapter->phy_stats.idle_errors += phy_tmp;
3724                 }
3725
3726                 if ((hw->mac_type <= e1000_82546) &&
3727                    (hw->phy_type == e1000_phy_m88) &&
3728                    !e1000_read_phy_reg(hw, M88E1000_RX_ERR_CNTR, &phy_tmp))
3729                         adapter->phy_stats.receive_errors += phy_tmp;
3730         }
3731
3732         /* Management Stats */
3733         if (hw->has_smbus) {
3734                 adapter->stats.mgptc += er32(MGTPTC);
3735                 adapter->stats.mgprc += er32(MGTPRC);
3736                 adapter->stats.mgpdc += er32(MGTPDC);
3737         }
3738
3739         spin_unlock_irqrestore(&adapter->stats_lock, flags);
3740 }
3741
3742 /**
3743  * e1000_intr - Interrupt Handler
3744  * @irq: interrupt number
3745  * @data: pointer to a network interface device structure
3746  **/
3747 static irqreturn_t e1000_intr(int irq, void *data)
3748 {
3749         struct net_device *netdev = data;
3750         struct e1000_adapter *adapter = netdev_priv(netdev);
3751         struct e1000_hw *hw = &adapter->hw;
3752         u32 icr = er32(ICR);
3753
3754         if (unlikely((!icr)))
3755                 return IRQ_NONE;  /* Not our interrupt */
3756
3757         /* we might have caused the interrupt, but the above
3758          * read cleared it, and just in case the driver is
3759          * down there is nothing to do so return handled
3760          */
3761         if (unlikely(test_bit(__E1000_DOWN, &adapter->flags)))
3762                 return IRQ_HANDLED;
3763
3764         if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
3765                 hw->get_link_status = 1;
3766                 /* guard against interrupt when we're going down */
3767                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3768                         schedule_delayed_work(&adapter->watchdog_task, 1);
3769         }
3770
3771         /* disable interrupts, without the synchronize_irq bit */
3772         ew32(IMC, ~0);
3773         E1000_WRITE_FLUSH();
3774
3775         if (likely(napi_schedule_prep(&adapter->napi))) {
3776                 adapter->total_tx_bytes = 0;
3777                 adapter->total_tx_packets = 0;
3778                 adapter->total_rx_bytes = 0;
3779                 adapter->total_rx_packets = 0;
3780                 __napi_schedule(&adapter->napi);
3781         } else {
3782                 /* this really should not happen! if it does it is basically a
3783                  * bug, but not a hard error, so enable ints and continue
3784                  */
3785                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3786                         e1000_irq_enable(adapter);
3787         }
3788
3789         return IRQ_HANDLED;
3790 }
3791
3792 /**
3793  * e1000_clean - NAPI Rx polling callback
3794  * @adapter: board private structure
3795  **/
3796 static int e1000_clean(struct napi_struct *napi, int budget)
3797 {
3798         struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
3799                                                      napi);
3800         int tx_clean_complete = 0, work_done = 0;
3801
3802         tx_clean_complete = e1000_clean_tx_irq(adapter, &adapter->tx_ring[0]);
3803
3804         adapter->clean_rx(adapter, &adapter->rx_ring[0], &work_done, budget);
3805
3806         if (!tx_clean_complete || work_done == budget)
3807                 return budget;
3808
3809         /* Exit the polling mode, but don't re-enable interrupts if stack might
3810          * poll us due to busy-polling
3811          */
3812         if (likely(napi_complete_done(napi, work_done))) {
3813                 if (likely(adapter->itr_setting & 3))
3814                         e1000_set_itr(adapter);
3815                 if (!test_bit(__E1000_DOWN, &adapter->flags))
3816                         e1000_irq_enable(adapter);
3817         }
3818
3819         return work_done;
3820 }
3821
3822 /**
3823  * e1000_clean_tx_irq - Reclaim resources after transmit completes
3824  * @adapter: board private structure
3825  **/
3826 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
3827                                struct e1000_tx_ring *tx_ring)
3828 {
3829         struct e1000_hw *hw = &adapter->hw;
3830         struct net_device *netdev = adapter->netdev;
3831         struct e1000_tx_desc *tx_desc, *eop_desc;
3832         struct e1000_tx_buffer *buffer_info;
3833         unsigned int i, eop;
3834         unsigned int count = 0;
3835         unsigned int total_tx_bytes = 0, total_tx_packets = 0;
3836         unsigned int bytes_compl = 0, pkts_compl = 0;
3837
3838         i = tx_ring->next_to_clean;
3839         eop = tx_ring->buffer_info[i].next_to_watch;
3840         eop_desc = E1000_TX_DESC(*tx_ring, eop);
3841
3842         while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
3843                (count < tx_ring->count)) {
3844                 bool cleaned = false;
3845                 dma_rmb();      /* read buffer_info after eop_desc */
3846                 for ( ; !cleaned; count++) {
3847                         tx_desc = E1000_TX_DESC(*tx_ring, i);
3848                         buffer_info = &tx_ring->buffer_info[i];
3849                         cleaned = (i == eop);
3850
3851                         if (cleaned) {
3852                                 total_tx_packets += buffer_info->segs;
3853                                 total_tx_bytes += buffer_info->bytecount;
3854                                 if (buffer_info->skb) {
3855                                         bytes_compl += buffer_info->skb->len;
3856                                         pkts_compl++;
3857                                 }
3858
3859                         }
3860                         e1000_unmap_and_free_tx_resource(adapter, buffer_info);
3861                         tx_desc->upper.data = 0;
3862
3863                         if (unlikely(++i == tx_ring->count))
3864                                 i = 0;
3865                 }
3866
3867                 eop = tx_ring->buffer_info[i].next_to_watch;
3868                 eop_desc = E1000_TX_DESC(*tx_ring, eop);
3869         }
3870
3871         /* Synchronize with E1000_DESC_UNUSED called from e1000_xmit_frame,
3872          * which will reuse the cleaned buffers.
3873          */
3874         smp_store_release(&tx_ring->next_to_clean, i);
3875
3876         netdev_completed_queue(netdev, pkts_compl, bytes_compl);
3877
3878 #define TX_WAKE_THRESHOLD 32
3879         if (unlikely(count && netif_carrier_ok(netdev) &&
3880                      E1000_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD)) {
3881                 /* Make sure that anybody stopping the queue after this
3882                  * sees the new next_to_clean.
3883                  */
3884                 smp_mb();
3885
3886                 if (netif_queue_stopped(netdev) &&
3887                     !(test_bit(__E1000_DOWN, &adapter->flags))) {
3888                         netif_wake_queue(netdev);
3889                         ++adapter->restart_queue;
3890                 }
3891         }
3892
3893         if (adapter->detect_tx_hung) {
3894                 /* Detect a transmit hang in hardware, this serializes the
3895                  * check with the clearing of time_stamp and movement of i
3896                  */
3897                 adapter->detect_tx_hung = false;
3898                 if (tx_ring->buffer_info[eop].time_stamp &&
3899                     time_after(jiffies, tx_ring->buffer_info[eop].time_stamp +
3900                                (adapter->tx_timeout_factor * HZ)) &&
3901                     !(er32(STATUS) & E1000_STATUS_TXOFF)) {
3902
3903                         /* detected Tx unit hang */
3904                         e_err(drv, "Detected Tx Unit Hang\n"
3905                               "  Tx Queue             <%lu>\n"
3906                               "  TDH                  <%x>\n"
3907                               "  TDT                  <%x>\n"
3908                               "  next_to_use          <%x>\n"
3909                               "  next_to_clean        <%x>\n"
3910                               "buffer_info[next_to_clean]\n"
3911                               "  time_stamp           <%lx>\n"
3912                               "  next_to_watch        <%x>\n"
3913                               "  jiffies              <%lx>\n"
3914                               "  next_to_watch.status <%x>\n",
3915                                 (unsigned long)(tx_ring - adapter->tx_ring),
3916                                 readl(hw->hw_addr + tx_ring->tdh),
3917                                 readl(hw->hw_addr + tx_ring->tdt),
3918                                 tx_ring->next_to_use,
3919                                 tx_ring->next_to_clean,
3920                                 tx_ring->buffer_info[eop].time_stamp,
3921                                 eop,
3922                                 jiffies,
3923                                 eop_desc->upper.fields.status);
3924                         e1000_dump(adapter);
3925                         netif_stop_queue(netdev);
3926                 }
3927         }
3928         adapter->total_tx_bytes += total_tx_bytes;
3929         adapter->total_tx_packets += total_tx_packets;
3930         netdev->stats.tx_bytes += total_tx_bytes;
3931         netdev->stats.tx_packets += total_tx_packets;
3932         return count < tx_ring->count;
3933 }
3934
3935 /**
3936  * e1000_rx_checksum - Receive Checksum Offload for 82543
3937  * @adapter:     board private structure
3938  * @status_err:  receive descriptor status and error fields
3939  * @csum:        receive descriptor csum field
3940  * @sk_buff:     socket buffer with received data
3941  **/
3942 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
3943                               u32 csum, struct sk_buff *skb)
3944 {
3945         struct e1000_hw *hw = &adapter->hw;
3946         u16 status = (u16)status_err;
3947         u8 errors = (u8)(status_err >> 24);
3948
3949         skb_checksum_none_assert(skb);
3950
3951         /* 82543 or newer only */
3952         if (unlikely(hw->mac_type < e1000_82543))
3953                 return;
3954         /* Ignore Checksum bit is set */
3955         if (unlikely(status & E1000_RXD_STAT_IXSM))
3956                 return;
3957         /* TCP/UDP checksum error bit is set */
3958         if (unlikely(errors & E1000_RXD_ERR_TCPE)) {
3959                 /* let the stack verify checksum errors */
3960                 adapter->hw_csum_err++;
3961                 return;
3962         }
3963         /* TCP/UDP Checksum has not been calculated */
3964         if (!(status & E1000_RXD_STAT_TCPCS))
3965                 return;
3966
3967         /* It must be a TCP or UDP packet with a valid checksum */
3968         if (likely(status & E1000_RXD_STAT_TCPCS)) {
3969                 /* TCP checksum is good */
3970                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3971         }
3972         adapter->hw_csum_good++;
3973 }
3974
3975 /**
3976  * e1000_consume_page - helper function for jumbo Rx path
3977  **/
3978 static void e1000_consume_page(struct e1000_rx_buffer *bi, struct sk_buff *skb,
3979                                u16 length)
3980 {
3981         bi->rxbuf.page = NULL;
3982         skb->len += length;
3983         skb->data_len += length;
3984         skb->truesize += PAGE_SIZE;
3985 }
3986
3987 /**
3988  * e1000_receive_skb - helper function to handle rx indications
3989  * @adapter: board private structure
3990  * @status: descriptor status field as written by hardware
3991  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
3992  * @skb: pointer to sk_buff to be indicated to stack
3993  */
3994 static void e1000_receive_skb(struct e1000_adapter *adapter, u8 status,
3995                               __le16 vlan, struct sk_buff *skb)
3996 {
3997         skb->protocol = eth_type_trans(skb, adapter->netdev);
3998
3999         if (status & E1000_RXD_STAT_VP) {
4000                 u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4001
4002                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4003         }
4004         napi_gro_receive(&adapter->napi, skb);
4005 }
4006
4007 /**
4008  * e1000_tbi_adjust_stats
4009  * @hw: Struct containing variables accessed by shared code
4010  * @frame_len: The length of the frame in question
4011  * @mac_addr: The Ethernet destination address of the frame in question
4012  *
4013  * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT
4014  */
4015 static void e1000_tbi_adjust_stats(struct e1000_hw *hw,
4016                                    struct e1000_hw_stats *stats,
4017                                    u32 frame_len, const u8 *mac_addr)
4018 {
4019         u64 carry_bit;
4020
4021         /* First adjust the frame length. */
4022         frame_len--;
4023         /* We need to adjust the statistics counters, since the hardware
4024          * counters overcount this packet as a CRC error and undercount
4025          * the packet as a good packet
4026          */
4027         /* This packet should not be counted as a CRC error. */
4028         stats->crcerrs--;
4029         /* This packet does count as a Good Packet Received. */
4030         stats->gprc++;
4031
4032         /* Adjust the Good Octets received counters */
4033         carry_bit = 0x80000000 & stats->gorcl;
4034         stats->gorcl += frame_len;
4035         /* If the high bit of Gorcl (the low 32 bits of the Good Octets
4036          * Received Count) was one before the addition,
4037          * AND it is zero after, then we lost the carry out,
4038          * need to add one to Gorch (Good Octets Received Count High).
4039          * This could be simplified if all environments supported
4040          * 64-bit integers.
4041          */
4042         if (carry_bit && ((stats->gorcl & 0x80000000) == 0))
4043                 stats->gorch++;
4044         /* Is this a broadcast or multicast?  Check broadcast first,
4045          * since the test for a multicast frame will test positive on
4046          * a broadcast frame.
4047          */
4048         if (is_broadcast_ether_addr(mac_addr))
4049                 stats->bprc++;
4050         else if (is_multicast_ether_addr(mac_addr))
4051                 stats->mprc++;
4052
4053         if (frame_len == hw->max_frame_size) {
4054                 /* In this case, the hardware has overcounted the number of
4055                  * oversize frames.
4056                  */
4057                 if (stats->roc > 0)
4058                         stats->roc--;
4059         }
4060
4061         /* Adjust the bin counters when the extra byte put the frame in the
4062          * wrong bin. Remember that the frame_len was adjusted above.
4063          */
4064         if (frame_len == 64) {
4065                 stats->prc64++;
4066                 stats->prc127--;
4067         } else if (frame_len == 127) {
4068                 stats->prc127++;
4069                 stats->prc255--;
4070         } else if (frame_len == 255) {
4071                 stats->prc255++;
4072                 stats->prc511--;
4073         } else if (frame_len == 511) {
4074                 stats->prc511++;
4075                 stats->prc1023--;
4076         } else if (frame_len == 1023) {
4077                 stats->prc1023++;
4078                 stats->prc1522--;
4079         } else if (frame_len == 1522) {
4080                 stats->prc1522++;
4081         }
4082 }
4083
4084 static bool e1000_tbi_should_accept(struct e1000_adapter *adapter,
4085                                     u8 status, u8 errors,
4086                                     u32 length, const u8 *data)
4087 {
4088         struct e1000_hw *hw = &adapter->hw;
4089         u8 last_byte = *(data + length - 1);
4090
4091         if (TBI_ACCEPT(hw, status, errors, length, last_byte)) {
4092                 unsigned long irq_flags;
4093
4094                 spin_lock_irqsave(&adapter->stats_lock, irq_flags);
4095                 e1000_tbi_adjust_stats(hw, &adapter->stats, length, data);
4096                 spin_unlock_irqrestore(&adapter->stats_lock, irq_flags);
4097
4098                 return true;
4099         }
4100
4101         return false;
4102 }
4103
4104 static struct sk_buff *e1000_alloc_rx_skb(struct e1000_adapter *adapter,
4105                                           unsigned int bufsz)
4106 {
4107         struct sk_buff *skb = napi_alloc_skb(&adapter->napi, bufsz);
4108
4109         if (unlikely(!skb))
4110                 adapter->alloc_rx_buff_failed++;
4111         return skb;
4112 }
4113
4114 /**
4115  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
4116  * @adapter: board private structure
4117  * @rx_ring: ring to clean
4118  * @work_done: amount of napi work completed this call
4119  * @work_to_do: max amount of work allowed for this call to do
4120  *
4121  * the return value indicates whether actual cleaning was done, there
4122  * is no guarantee that everything was cleaned
4123  */
4124 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
4125                                      struct e1000_rx_ring *rx_ring,
4126                                      int *work_done, int work_to_do)
4127 {
4128         struct net_device *netdev = adapter->netdev;
4129         struct pci_dev *pdev = adapter->pdev;
4130         struct e1000_rx_desc *rx_desc, *next_rxd;
4131         struct e1000_rx_buffer *buffer_info, *next_buffer;
4132         u32 length;
4133         unsigned int i;
4134         int cleaned_count = 0;
4135         bool cleaned = false;
4136         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4137
4138         i = rx_ring->next_to_clean;
4139         rx_desc = E1000_RX_DESC(*rx_ring, i);
4140         buffer_info = &rx_ring->buffer_info[i];
4141
4142         while (rx_desc->status & E1000_RXD_STAT_DD) {
4143                 struct sk_buff *skb;
4144                 u8 status;
4145
4146                 if (*work_done >= work_to_do)
4147                         break;
4148                 (*work_done)++;
4149                 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4150
4151                 status = rx_desc->status;
4152
4153                 if (++i == rx_ring->count)
4154                         i = 0;
4155
4156                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4157                 prefetch(next_rxd);
4158
4159                 next_buffer = &rx_ring->buffer_info[i];
4160
4161                 cleaned = true;
4162                 cleaned_count++;
4163                 dma_unmap_page(&pdev->dev, buffer_info->dma,
4164                                adapter->rx_buffer_len, DMA_FROM_DEVICE);
4165                 buffer_info->dma = 0;
4166
4167                 length = le16_to_cpu(rx_desc->length);
4168
4169                 /* errors is only valid for DD + EOP descriptors */
4170                 if (unlikely((status & E1000_RXD_STAT_EOP) &&
4171                     (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) {
4172                         u8 *mapped = page_address(buffer_info->rxbuf.page);
4173
4174                         if (e1000_tbi_should_accept(adapter, status,
4175                                                     rx_desc->errors,
4176                                                     length, mapped)) {
4177                                 length--;
4178                         } else if (netdev->features & NETIF_F_RXALL) {
4179                                 goto process_skb;
4180                         } else {
4181                                 /* an error means any chain goes out the window
4182                                  * too
4183                                  */
4184                                 if (rx_ring->rx_skb_top)
4185                                         dev_kfree_skb(rx_ring->rx_skb_top);
4186                                 rx_ring->rx_skb_top = NULL;
4187                                 goto next_desc;
4188                         }
4189                 }
4190
4191 #define rxtop rx_ring->rx_skb_top
4192 process_skb:
4193                 if (!(status & E1000_RXD_STAT_EOP)) {
4194                         /* this descriptor is only the beginning (or middle) */
4195                         if (!rxtop) {
4196                                 /* this is the beginning of a chain */
4197                                 rxtop = napi_get_frags(&adapter->napi);
4198                                 if (!rxtop)
4199                                         break;
4200
4201                                 skb_fill_page_desc(rxtop, 0,
4202                                                    buffer_info->rxbuf.page,
4203                                                    0, length);
4204                         } else {
4205                                 /* this is the middle of a chain */
4206                                 skb_fill_page_desc(rxtop,
4207                                     skb_shinfo(rxtop)->nr_frags,
4208                                     buffer_info->rxbuf.page, 0, length);
4209                         }
4210                         e1000_consume_page(buffer_info, rxtop, length);
4211                         goto next_desc;
4212                 } else {
4213                         if (rxtop) {
4214                                 /* end of the chain */
4215                                 skb_fill_page_desc(rxtop,
4216                                     skb_shinfo(rxtop)->nr_frags,
4217                                     buffer_info->rxbuf.page, 0, length);
4218                                 skb = rxtop;
4219                                 rxtop = NULL;
4220                                 e1000_consume_page(buffer_info, skb, length);
4221                         } else {
4222                                 struct page *p;
4223                                 /* no chain, got EOP, this buf is the packet
4224                                  * copybreak to save the put_page/alloc_page
4225                                  */
4226                                 p = buffer_info->rxbuf.page;
4227                                 if (length <= copybreak) {
4228                                         u8 *vaddr;
4229
4230                                         if (likely(!(netdev->features & NETIF_F_RXFCS)))
4231                                                 length -= 4;
4232                                         skb = e1000_alloc_rx_skb(adapter,
4233                                                                  length);
4234                                         if (!skb)
4235                                                 break;
4236
4237                                         vaddr = kmap_atomic(p);
4238                                         memcpy(skb_tail_pointer(skb), vaddr,
4239                                                length);
4240                                         kunmap_atomic(vaddr);
4241                                         /* re-use the page, so don't erase
4242                                          * buffer_info->rxbuf.page
4243                                          */
4244                                         skb_put(skb, length);
4245                                         e1000_rx_checksum(adapter,
4246                                                           status | rx_desc->errors << 24,
4247                                                           le16_to_cpu(rx_desc->csum), skb);
4248
4249                                         total_rx_bytes += skb->len;
4250                                         total_rx_packets++;
4251
4252                                         e1000_receive_skb(adapter, status,
4253                                                           rx_desc->special, skb);
4254                                         goto next_desc;
4255                                 } else {
4256                                         skb = napi_get_frags(&adapter->napi);
4257                                         if (!skb) {
4258                                                 adapter->alloc_rx_buff_failed++;
4259                                                 break;
4260                                         }
4261                                         skb_fill_page_desc(skb, 0, p, 0,
4262                                                            length);
4263                                         e1000_consume_page(buffer_info, skb,
4264                                                            length);
4265                                 }
4266                         }
4267                 }
4268
4269                 /* Receive Checksum Offload XXX recompute due to CRC strip? */
4270                 e1000_rx_checksum(adapter,
4271                                   (u32)(status) |
4272                                   ((u32)(rx_desc->errors) << 24),
4273                                   le16_to_cpu(rx_desc->csum), skb);
4274
4275                 total_rx_bytes += (skb->len - 4); /* don't count FCS */
4276                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4277                         pskb_trim(skb, skb->len - 4);
4278                 total_rx_packets++;
4279
4280                 if (status & E1000_RXD_STAT_VP) {
4281                         __le16 vlan = rx_desc->special;
4282                         u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4283
4284                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4285                 }
4286
4287                 napi_gro_frags(&adapter->napi);
4288
4289 next_desc:
4290                 rx_desc->status = 0;
4291
4292                 /* return some buffers to hardware, one at a time is too slow */
4293                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4294                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4295                         cleaned_count = 0;
4296                 }
4297
4298                 /* use prefetched values */
4299                 rx_desc = next_rxd;
4300                 buffer_info = next_buffer;
4301         }
4302         rx_ring->next_to_clean = i;
4303
4304         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4305         if (cleaned_count)
4306                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4307
4308         adapter->total_rx_packets += total_rx_packets;
4309         adapter->total_rx_bytes += total_rx_bytes;
4310         netdev->stats.rx_bytes += total_rx_bytes;
4311         netdev->stats.rx_packets += total_rx_packets;
4312         return cleaned;
4313 }
4314
4315 /* this should improve performance for small packets with large amounts
4316  * of reassembly being done in the stack
4317  */
4318 static struct sk_buff *e1000_copybreak(struct e1000_adapter *adapter,
4319                                        struct e1000_rx_buffer *buffer_info,
4320                                        u32 length, const void *data)
4321 {
4322         struct sk_buff *skb;
4323
4324         if (length > copybreak)
4325                 return NULL;
4326
4327         skb = e1000_alloc_rx_skb(adapter, length);
4328         if (!skb)
4329                 return NULL;
4330
4331         dma_sync_single_for_cpu(&adapter->pdev->dev, buffer_info->dma,
4332                                 length, DMA_FROM_DEVICE);
4333
4334         skb_put_data(skb, data, length);
4335
4336         return skb;
4337 }
4338
4339 /**
4340  * e1000_clean_rx_irq - Send received data up the network stack; legacy
4341  * @adapter: board private structure
4342  * @rx_ring: ring to clean
4343  * @work_done: amount of napi work completed this call
4344  * @work_to_do: max amount of work allowed for this call to do
4345  */
4346 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
4347                                struct e1000_rx_ring *rx_ring,
4348                                int *work_done, int work_to_do)
4349 {
4350         struct net_device *netdev = adapter->netdev;
4351         struct pci_dev *pdev = adapter->pdev;
4352         struct e1000_rx_desc *rx_desc, *next_rxd;
4353         struct e1000_rx_buffer *buffer_info, *next_buffer;
4354         u32 length;
4355         unsigned int i;
4356         int cleaned_count = 0;
4357         bool cleaned = false;
4358         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4359
4360         i = rx_ring->next_to_clean;
4361         rx_desc = E1000_RX_DESC(*rx_ring, i);
4362         buffer_info = &rx_ring->buffer_info[i];
4363
4364         while (rx_desc->status & E1000_RXD_STAT_DD) {
4365                 struct sk_buff *skb;
4366                 u8 *data;
4367                 u8 status;
4368
4369                 if (*work_done >= work_to_do)
4370                         break;
4371                 (*work_done)++;
4372                 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4373
4374                 status = rx_desc->status;
4375                 length = le16_to_cpu(rx_desc->length);
4376
4377                 data = buffer_info->rxbuf.data;
4378                 prefetch(data);
4379                 skb = e1000_copybreak(adapter, buffer_info, length, data);
4380                 if (!skb) {
4381                         unsigned int frag_len = e1000_frag_len(adapter);
4382
4383                         skb = build_skb(data - E1000_HEADROOM, frag_len);
4384                         if (!skb) {
4385                                 adapter->alloc_rx_buff_failed++;
4386                                 break;
4387                         }
4388
4389                         skb_reserve(skb, E1000_HEADROOM);
4390                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4391                                          adapter->rx_buffer_len,
4392                                          DMA_FROM_DEVICE);
4393                         buffer_info->dma = 0;
4394                         buffer_info->rxbuf.data = NULL;
4395                 }
4396
4397                 if (++i == rx_ring->count)
4398                         i = 0;
4399
4400                 next_rxd = E1000_RX_DESC(*rx_ring, i);
4401                 prefetch(next_rxd);
4402
4403                 next_buffer = &rx_ring->buffer_info[i];
4404
4405                 cleaned = true;
4406                 cleaned_count++;
4407
4408                 /* !EOP means multiple descriptors were used to store a single
4409                  * packet, if thats the case we need to toss it.  In fact, we
4410                  * to toss every packet with the EOP bit clear and the next
4411                  * frame that _does_ have the EOP bit set, as it is by
4412                  * definition only a frame fragment
4413                  */
4414                 if (unlikely(!(status & E1000_RXD_STAT_EOP)))
4415                         adapter->discarding = true;
4416
4417                 if (adapter->discarding) {
4418                         /* All receives must fit into a single buffer */
4419                         netdev_dbg(netdev, "Receive packet consumed multiple buffers\n");
4420                         dev_kfree_skb(skb);
4421                         if (status & E1000_RXD_STAT_EOP)
4422                                 adapter->discarding = false;
4423                         goto next_desc;
4424                 }
4425
4426                 if (unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
4427                         if (e1000_tbi_should_accept(adapter, status,
4428                                                     rx_desc->errors,
4429                                                     length, data)) {
4430                                 length--;
4431                         } else if (netdev->features & NETIF_F_RXALL) {
4432                                 goto process_skb;
4433                         } else {
4434                                 dev_kfree_skb(skb);
4435                                 goto next_desc;
4436                         }
4437                 }
4438
4439 process_skb:
4440                 total_rx_bytes += (length - 4); /* don't count FCS */
4441                 total_rx_packets++;
4442
4443                 if (likely(!(netdev->features & NETIF_F_RXFCS)))
4444                         /* adjust length to remove Ethernet CRC, this must be
4445                          * done after the TBI_ACCEPT workaround above
4446                          */
4447                         length -= 4;
4448
4449                 if (buffer_info->rxbuf.data == NULL)
4450                         skb_put(skb, length);
4451                 else /* copybreak skb */
4452                         skb_trim(skb, length);
4453
4454                 /* Receive Checksum Offload */
4455                 e1000_rx_checksum(adapter,
4456                                   (u32)(status) |
4457                                   ((u32)(rx_desc->errors) << 24),
4458                                   le16_to_cpu(rx_desc->csum), skb);
4459
4460                 e1000_receive_skb(adapter, status, rx_desc->special, skb);
4461
4462 next_desc:
4463                 rx_desc->status = 0;
4464
4465                 /* return some buffers to hardware, one at a time is too slow */
4466                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4467                         adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4468                         cleaned_count = 0;
4469                 }
4470
4471                 /* use prefetched values */
4472                 rx_desc = next_rxd;
4473                 buffer_info = next_buffer;
4474         }
4475         rx_ring->next_to_clean = i;
4476
4477         cleaned_count = E1000_DESC_UNUSED(rx_ring);
4478         if (cleaned_count)
4479                 adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4480
4481         adapter->total_rx_packets += total_rx_packets;
4482         adapter->total_rx_bytes += total_rx_bytes;
4483         netdev->stats.rx_bytes += total_rx_bytes;
4484         netdev->stats.rx_packets += total_rx_packets;
4485         return cleaned;
4486 }
4487
4488 /**
4489  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
4490  * @adapter: address of board private structure
4491  * @rx_ring: pointer to receive ring structure
4492  * @cleaned_count: number of buffers to allocate this pass
4493  **/
4494 static void
4495 e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
4496                              struct e1000_rx_ring *rx_ring, int cleaned_count)
4497 {
4498         struct pci_dev *pdev = adapter->pdev;
4499         struct e1000_rx_desc *rx_desc;
4500         struct e1000_rx_buffer *buffer_info;
4501         unsigned int i;
4502
4503         i = rx_ring->next_to_use;
4504         buffer_info = &rx_ring->buffer_info[i];
4505
4506         while (cleaned_count--) {
4507                 /* allocate a new page if necessary */
4508                 if (!buffer_info->rxbuf.page) {
4509                         buffer_info->rxbuf.page = alloc_page(GFP_ATOMIC);
4510                         if (unlikely(!buffer_info->rxbuf.page)) {
4511                                 adapter->alloc_rx_buff_failed++;
4512                                 break;
4513                         }
4514                 }
4515
4516                 if (!buffer_info->dma) {
4517                         buffer_info->dma = dma_map_page(&pdev->dev,
4518                                                         buffer_info->rxbuf.page, 0,
4519                                                         adapter->rx_buffer_len,
4520                                                         DMA_FROM_DEVICE);
4521                         if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4522                                 put_page(buffer_info->rxbuf.page);
4523                                 buffer_info->rxbuf.page = NULL;
4524                                 buffer_info->dma = 0;
4525                                 adapter->alloc_rx_buff_failed++;
4526                                 break;
4527                         }
4528                 }
4529
4530                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4531                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4532
4533                 if (unlikely(++i == rx_ring->count))
4534                         i = 0;
4535                 buffer_info = &rx_ring->buffer_info[i];
4536         }
4537
4538         if (likely(rx_ring->next_to_use != i)) {
4539                 rx_ring->next_to_use = i;
4540                 if (unlikely(i-- == 0))
4541                         i = (rx_ring->count - 1);
4542
4543                 /* Force memory writes to complete before letting h/w
4544                  * know there are new descriptors to fetch.  (Only
4545                  * applicable for weak-ordered memory model archs,
4546                  * such as IA-64).
4547                  */
4548                 wmb();
4549                 writel(i, adapter->hw.hw_addr + rx_ring->rdt);
4550         }
4551 }
4552
4553 /**
4554  * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
4555  * @adapter: address of board private structure
4556  **/
4557 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
4558                                    struct e1000_rx_ring *rx_ring,
4559                                    int cleaned_count)
4560 {
4561         struct e1000_hw *hw = &adapter->hw;
4562         struct pci_dev *pdev = adapter->pdev;
4563         struct e1000_rx_desc *rx_desc;
4564         struct e1000_rx_buffer *buffer_info;
4565         unsigned int i;
4566         unsigned int bufsz = adapter->rx_buffer_len;
4567
4568         i = rx_ring->next_to_use;
4569         buffer_info = &rx_ring->buffer_info[i];
4570
4571         while (cleaned_count--) {
4572                 void *data;
4573
4574                 if (buffer_info->rxbuf.data)
4575                         goto skip;
4576
4577                 data = e1000_alloc_frag(adapter);
4578                 if (!data) {
4579                         /* Better luck next round */
4580                         adapter->alloc_rx_buff_failed++;
4581                         break;
4582                 }
4583
4584                 /* Fix for errata 23, can't cross 64kB boundary */
4585                 if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4586                         void *olddata = data;
4587                         e_err(rx_err, "skb align check failed: %u bytes at "
4588                               "%p\n", bufsz, data);
4589                         /* Try again, without freeing the previous */
4590                         data = e1000_alloc_frag(adapter);
4591                         /* Failed allocation, critical failure */
4592                         if (!data) {
4593                                 skb_free_frag(olddata);
4594                                 adapter->alloc_rx_buff_failed++;
4595                                 break;
4596                         }
4597
4598                         if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4599                                 /* give up */
4600                                 skb_free_frag(data);
4601                                 skb_free_frag(olddata);
4602                                 adapter->alloc_rx_buff_failed++;
4603                                 break;
4604                         }
4605
4606                         /* Use new allocation */
4607                         skb_free_frag(olddata);
4608                 }
4609                 buffer_info->dma = dma_map_single(&pdev->dev,
4610                                                   data,
4611                                                   adapter->rx_buffer_len,
4612                                                   DMA_FROM_DEVICE);
4613                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4614                         skb_free_frag(data);
4615                         buffer_info->dma = 0;
4616                         adapter->alloc_rx_buff_failed++;
4617                         break;
4618                 }
4619
4620                 /* XXX if it was allocated cleanly it will never map to a
4621                  * boundary crossing
4622                  */
4623
4624                 /* Fix for errata 23, can't cross 64kB boundary */
4625                 if (!e1000_check_64k_bound(adapter,
4626                                         (void *)(unsigned long)buffer_info->dma,
4627                                         adapter->rx_buffer_len)) {
4628                         e_err(rx_err, "dma align check failed: %u bytes at "
4629                               "%p\n", adapter->rx_buffer_len,
4630                               (void *)(unsigned long)buffer_info->dma);
4631
4632                         dma_unmap_single(&pdev->dev, buffer_info->dma,
4633                                          adapter->rx_buffer_len,
4634                                          DMA_FROM_DEVICE);
4635
4636                         skb_free_frag(data);
4637                         buffer_info->rxbuf.data = NULL;
4638                         buffer_info->dma = 0;
4639
4640                         adapter->alloc_rx_buff_failed++;
4641                         break;
4642                 }
4643                 buffer_info->rxbuf.data = data;
4644  skip:
4645                 rx_desc = E1000_RX_DESC(*rx_ring, i);
4646                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4647
4648                 if (unlikely(++i == rx_ring->count))
4649                         i = 0;
4650                 buffer_info = &rx_ring->buffer_info[i];
4651         }
4652
4653         if (likely(rx_ring->next_to_use != i)) {
4654                 rx_ring->next_to_use = i;
4655                 if (unlikely(i-- == 0))
4656                         i = (rx_ring->count - 1);
4657
4658                 /* Force memory writes to complete before letting h/w
4659                  * know there are new descriptors to fetch.  (Only
4660                  * applicable for weak-ordered memory model archs,
4661                  * such as IA-64).
4662                  */
4663                 wmb();
4664                 writel(i, hw->hw_addr + rx_ring->rdt);
4665         }
4666 }
4667
4668 /**
4669  * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers.
4670  * @adapter:
4671  **/
4672 static void e1000_smartspeed(struct e1000_adapter *adapter)
4673 {
4674         struct e1000_hw *hw = &adapter->hw;
4675         u16 phy_status;
4676         u16 phy_ctrl;
4677
4678         if ((hw->phy_type != e1000_phy_igp) || !hw->autoneg ||
4679            !(hw->autoneg_advertised & ADVERTISE_1000_FULL))
4680                 return;
4681
4682         if (adapter->smartspeed == 0) {
4683                 /* If Master/Slave config fault is asserted twice,
4684                  * we assume back-to-back
4685                  */
4686                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4687                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4688                         return;
4689                 e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4690                 if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4691                         return;
4692                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4693                 if (phy_ctrl & CR_1000T_MS_ENABLE) {
4694                         phy_ctrl &= ~CR_1000T_MS_ENABLE;
4695                         e1000_write_phy_reg(hw, PHY_1000T_CTRL,
4696                                             phy_ctrl);
4697                         adapter->smartspeed++;
4698                         if (!e1000_phy_setup_autoneg(hw) &&
4699                            !e1000_read_phy_reg(hw, PHY_CTRL,
4700                                                &phy_ctrl)) {
4701                                 phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4702                                              MII_CR_RESTART_AUTO_NEG);
4703                                 e1000_write_phy_reg(hw, PHY_CTRL,
4704                                                     phy_ctrl);
4705                         }
4706                 }
4707                 return;
4708         } else if (adapter->smartspeed == E1000_SMARTSPEED_DOWNSHIFT) {
4709                 /* If still no link, perhaps using 2/3 pair cable */
4710                 e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4711                 phy_ctrl |= CR_1000T_MS_ENABLE;
4712                 e1000_write_phy_reg(hw, PHY_1000T_CTRL, phy_ctrl);
4713                 if (!e1000_phy_setup_autoneg(hw) &&
4714                    !e1000_read_phy_reg(hw, PHY_CTRL, &phy_ctrl)) {
4715                         phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4716                                      MII_CR_RESTART_AUTO_NEG);
4717                         e1000_write_phy_reg(hw, PHY_CTRL, phy_ctrl);
4718                 }
4719         }
4720         /* Restart process after E1000_SMARTSPEED_MAX iterations */
4721         if (adapter->smartspeed++ == E1000_SMARTSPEED_MAX)
4722                 adapter->smartspeed = 0;
4723 }
4724
4725 /**
4726  * e1000_ioctl -
4727  * @netdev:
4728  * @ifreq:
4729  * @cmd:
4730  **/
4731 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
4732 {
4733         switch (cmd) {
4734         case SIOCGMIIPHY:
4735         case SIOCGMIIREG:
4736         case SIOCSMIIREG:
4737                 return e1000_mii_ioctl(netdev, ifr, cmd);
4738         default:
4739                 return -EOPNOTSUPP;
4740         }
4741 }
4742
4743 /**
4744  * e1000_mii_ioctl -
4745  * @netdev:
4746  * @ifreq:
4747  * @cmd:
4748  **/
4749 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
4750                            int cmd)
4751 {
4752         struct e1000_adapter *adapter = netdev_priv(netdev);
4753         struct e1000_hw *hw = &adapter->hw;
4754         struct mii_ioctl_data *data = if_mii(ifr);
4755         int retval;
4756         u16 mii_reg;
4757         unsigned long flags;
4758
4759         if (hw->media_type != e1000_media_type_copper)
4760                 return -EOPNOTSUPP;
4761
4762         switch (cmd) {
4763         case SIOCGMIIPHY:
4764                 data->phy_id = hw->phy_addr;
4765                 break;
4766         case SIOCGMIIREG:
4767                 spin_lock_irqsave(&adapter->stats_lock, flags);
4768                 if (e1000_read_phy_reg(hw, data->reg_num & 0x1F,
4769                                    &data->val_out)) {
4770                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4771                         return -EIO;
4772                 }
4773                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4774                 break;
4775         case SIOCSMIIREG:
4776                 if (data->reg_num & ~(0x1F))
4777                         return -EFAULT;
4778                 mii_reg = data->val_in;
4779                 spin_lock_irqsave(&adapter->stats_lock, flags);
4780                 if (e1000_write_phy_reg(hw, data->reg_num,
4781                                         mii_reg)) {
4782                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4783                         return -EIO;
4784                 }
4785                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4786                 if (hw->media_type == e1000_media_type_copper) {
4787                         switch (data->reg_num) {
4788                         case PHY_CTRL:
4789                                 if (mii_reg & MII_CR_POWER_DOWN)
4790                                         break;
4791                                 if (mii_reg & MII_CR_AUTO_NEG_EN) {
4792                                         hw->autoneg = 1;
4793                                         hw->autoneg_advertised = 0x2F;
4794                                 } else {
4795                                         u32 speed;
4796                                         if (mii_reg & 0x40)
4797                                                 speed = SPEED_1000;
4798                                         else if (mii_reg & 0x2000)
4799                                                 speed = SPEED_100;
4800                                         else
4801                                                 speed = SPEED_10;
4802                                         retval = e1000_set_spd_dplx(
4803                                                 adapter, speed,
4804                                                 ((mii_reg & 0x100)
4805                                                  ? DUPLEX_FULL :
4806                                                  DUPLEX_HALF));
4807                                         if (retval)
4808                                                 return retval;
4809                                 }
4810                                 if (netif_running(adapter->netdev))
4811                                         e1000_reinit_locked(adapter);
4812                                 else
4813                                         e1000_reset(adapter);
4814                                 break;
4815                         case M88E1000_PHY_SPEC_CTRL:
4816                         case M88E1000_EXT_PHY_SPEC_CTRL:
4817                                 if (e1000_phy_reset(hw))
4818                                         return -EIO;
4819                                 break;
4820                         }
4821                 } else {
4822                         switch (data->reg_num) {
4823                         case PHY_CTRL:
4824                                 if (mii_reg & MII_CR_POWER_DOWN)
4825                                         break;
4826                                 if (netif_running(adapter->netdev))
4827                                         e1000_reinit_locked(adapter);
4828                                 else
4829                                         e1000_reset(adapter);
4830                                 break;
4831                         }
4832                 }
4833                 break;
4834         default:
4835                 return -EOPNOTSUPP;
4836         }
4837         return E1000_SUCCESS;
4838 }
4839
4840 void e1000_pci_set_mwi(struct e1000_hw *hw)
4841 {
4842         struct e1000_adapter *adapter = hw->back;
4843         int ret_val = pci_set_mwi(adapter->pdev);
4844
4845         if (ret_val)
4846                 e_err(probe, "Error in setting MWI\n");
4847 }
4848
4849 void e1000_pci_clear_mwi(struct e1000_hw *hw)
4850 {
4851         struct e1000_adapter *adapter = hw->back;
4852
4853         pci_clear_mwi(adapter->pdev);
4854 }
4855
4856 int e1000_pcix_get_mmrbc(struct e1000_hw *hw)
4857 {
4858         struct e1000_adapter *adapter = hw->back;
4859         return pcix_get_mmrbc(adapter->pdev);
4860 }
4861
4862 void e1000_pcix_set_mmrbc(struct e1000_hw *hw, int mmrbc)
4863 {
4864         struct e1000_adapter *adapter = hw->back;
4865         pcix_set_mmrbc(adapter->pdev, mmrbc);
4866 }
4867
4868 void e1000_io_write(struct e1000_hw *hw, unsigned long port, u32 value)
4869 {
4870         outl(value, port);
4871 }
4872
4873 static bool e1000_vlan_used(struct e1000_adapter *adapter)
4874 {
4875         u16 vid;
4876
4877         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
4878                 return true;
4879         return false;
4880 }
4881
4882 static void __e1000_vlan_mode(struct e1000_adapter *adapter,
4883                               netdev_features_t features)
4884 {
4885         struct e1000_hw *hw = &adapter->hw;
4886         u32 ctrl;
4887
4888         ctrl = er32(CTRL);
4889         if (features & NETIF_F_HW_VLAN_CTAG_RX) {
4890                 /* enable VLAN tag insert/strip */
4891                 ctrl |= E1000_CTRL_VME;
4892         } else {
4893                 /* disable VLAN tag insert/strip */
4894                 ctrl &= ~E1000_CTRL_VME;
4895         }
4896         ew32(CTRL, ctrl);
4897 }
4898 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
4899                                      bool filter_on)
4900 {
4901         struct e1000_hw *hw = &adapter->hw;
4902         u32 rctl;
4903
4904         if (!test_bit(__E1000_DOWN, &adapter->flags))
4905                 e1000_irq_disable(adapter);
4906
4907         __e1000_vlan_mode(adapter, adapter->netdev->features);
4908         if (filter_on) {
4909                 /* enable VLAN receive filtering */
4910                 rctl = er32(RCTL);
4911                 rctl &= ~E1000_RCTL_CFIEN;
4912                 if (!(adapter->netdev->flags & IFF_PROMISC))
4913                         rctl |= E1000_RCTL_VFE;
4914                 ew32(RCTL, rctl);
4915                 e1000_update_mng_vlan(adapter);
4916         } else {
4917                 /* disable VLAN receive filtering */
4918                 rctl = er32(RCTL);
4919                 rctl &= ~E1000_RCTL_VFE;
4920                 ew32(RCTL, rctl);
4921         }
4922
4923         if (!test_bit(__E1000_DOWN, &adapter->flags))
4924                 e1000_irq_enable(adapter);
4925 }
4926
4927 static void e1000_vlan_mode(struct net_device *netdev,
4928                             netdev_features_t features)
4929 {
4930         struct e1000_adapter *adapter = netdev_priv(netdev);
4931
4932         if (!test_bit(__E1000_DOWN, &adapter->flags))
4933                 e1000_irq_disable(adapter);
4934
4935         __e1000_vlan_mode(adapter, features);
4936
4937         if (!test_bit(__E1000_DOWN, &adapter->flags))
4938                 e1000_irq_enable(adapter);
4939 }
4940
4941 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
4942                                  __be16 proto, u16 vid)
4943 {
4944         struct e1000_adapter *adapter = netdev_priv(netdev);
4945         struct e1000_hw *hw = &adapter->hw;
4946         u32 vfta, index;
4947
4948         if ((hw->mng_cookie.status &
4949              E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
4950             (vid == adapter->mng_vlan_id))
4951                 return 0;
4952
4953         if (!e1000_vlan_used(adapter))
4954                 e1000_vlan_filter_on_off(adapter, true);
4955
4956         /* add VID to filter table */
4957         index = (vid >> 5) & 0x7F;
4958         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4959         vfta |= (1 << (vid & 0x1F));
4960         e1000_write_vfta(hw, index, vfta);
4961
4962         set_bit(vid, adapter->active_vlans);
4963
4964         return 0;
4965 }
4966
4967 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
4968                                   __be16 proto, u16 vid)
4969 {
4970         struct e1000_adapter *adapter = netdev_priv(netdev);
4971         struct e1000_hw *hw = &adapter->hw;
4972         u32 vfta, index;
4973
4974         if (!test_bit(__E1000_DOWN, &adapter->flags))
4975                 e1000_irq_disable(adapter);
4976         if (!test_bit(__E1000_DOWN, &adapter->flags))
4977                 e1000_irq_enable(adapter);
4978
4979         /* remove VID from filter table */
4980         index = (vid >> 5) & 0x7F;
4981         vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4982         vfta &= ~(1 << (vid & 0x1F));
4983         e1000_write_vfta(hw, index, vfta);
4984
4985         clear_bit(vid, adapter->active_vlans);
4986
4987         if (!e1000_vlan_used(adapter))
4988                 e1000_vlan_filter_on_off(adapter, false);
4989
4990         return 0;
4991 }
4992
4993 static void e1000_restore_vlan(struct e1000_adapter *adapter)
4994 {
4995         u16 vid;
4996
4997         if (!e1000_vlan_used(adapter))
4998                 return;
4999
5000         e1000_vlan_filter_on_off(adapter, true);
5001         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
5002                 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
5003 }
5004
5005 int e1000_set_spd_dplx(struct e1000_adapter *adapter, u32 spd, u8 dplx)
5006 {
5007         struct e1000_hw *hw = &adapter->hw;
5008
5009         hw->autoneg = 0;
5010
5011         /* Make sure dplx is at most 1 bit and lsb of speed is not set
5012          * for the switch() below to work
5013          */
5014         if ((spd & 1) || (dplx & ~1))
5015                 goto err_inval;
5016
5017         /* Fiber NICs only allow 1000 gbps Full duplex */
5018         if ((hw->media_type == e1000_media_type_fiber) &&
5019             spd != SPEED_1000 &&
5020             dplx != DUPLEX_FULL)
5021                 goto err_inval;
5022
5023         switch (spd + dplx) {
5024         case SPEED_10 + DUPLEX_HALF:
5025                 hw->forced_speed_duplex = e1000_10_half;
5026                 break;
5027         case SPEED_10 + DUPLEX_FULL:
5028                 hw->forced_speed_duplex = e1000_10_full;
5029                 break;
5030         case SPEED_100 + DUPLEX_HALF:
5031                 hw->forced_speed_duplex = e1000_100_half;
5032                 break;
5033         case SPEED_100 + DUPLEX_FULL:
5034                 hw->forced_speed_duplex = e1000_100_full;
5035                 break;
5036         case SPEED_1000 + DUPLEX_FULL:
5037                 hw->autoneg = 1;
5038                 hw->autoneg_advertised = ADVERTISE_1000_FULL;
5039                 break;
5040         case SPEED_1000 + DUPLEX_HALF: /* not supported */
5041         default:
5042                 goto err_inval;
5043         }
5044
5045         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
5046         hw->mdix = AUTO_ALL_MODES;
5047
5048         return 0;
5049
5050 err_inval:
5051         e_err(probe, "Unsupported Speed/Duplex configuration\n");
5052         return -EINVAL;
5053 }
5054
5055 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
5056 {
5057         struct net_device *netdev = pci_get_drvdata(pdev);
5058         struct e1000_adapter *adapter = netdev_priv(netdev);
5059         struct e1000_hw *hw = &adapter->hw;
5060         u32 ctrl, ctrl_ext, rctl, status;
5061         u32 wufc = adapter->wol;
5062 #ifdef CONFIG_PM
5063         int retval = 0;
5064 #endif
5065
5066         netif_device_detach(netdev);
5067
5068         if (netif_running(netdev)) {
5069                 int count = E1000_CHECK_RESET_COUNT;
5070
5071                 while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
5072                         usleep_range(10000, 20000);
5073
5074                 WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
5075                 e1000_down(adapter);
5076         }
5077
5078 #ifdef CONFIG_PM
5079         retval = pci_save_state(pdev);
5080         if (retval)
5081                 return retval;
5082 #endif
5083
5084         status = er32(STATUS);
5085         if (status & E1000_STATUS_LU)
5086                 wufc &= ~E1000_WUFC_LNKC;
5087
5088         if (wufc) {
5089                 e1000_setup_rctl(adapter);
5090                 e1000_set_rx_mode(netdev);
5091
5092                 rctl = er32(RCTL);
5093
5094                 /* turn on all-multi mode if wake on multicast is enabled */
5095                 if (wufc & E1000_WUFC_MC)
5096                         rctl |= E1000_RCTL_MPE;
5097
5098                 /* enable receives in the hardware */
5099                 ew32(RCTL, rctl | E1000_RCTL_EN);
5100
5101                 if (hw->mac_type >= e1000_82540) {
5102                         ctrl = er32(CTRL);
5103                         /* advertise wake from D3Cold */
5104                         #define E1000_CTRL_ADVD3WUC 0x00100000
5105                         /* phy power management enable */
5106                         #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5107                         ctrl |= E1000_CTRL_ADVD3WUC |
5108                                 E1000_CTRL_EN_PHY_PWR_MGMT;
5109                         ew32(CTRL, ctrl);
5110                 }
5111
5112                 if (hw->media_type == e1000_media_type_fiber ||
5113                     hw->media_type == e1000_media_type_internal_serdes) {
5114                         /* keep the laser running in D3 */
5115                         ctrl_ext = er32(CTRL_EXT);
5116                         ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
5117                         ew32(CTRL_EXT, ctrl_ext);
5118                 }
5119
5120                 ew32(WUC, E1000_WUC_PME_EN);
5121                 ew32(WUFC, wufc);
5122         } else {
5123                 ew32(WUC, 0);
5124                 ew32(WUFC, 0);
5125         }
5126
5127         e1000_release_manageability(adapter);
5128
5129         *enable_wake = !!wufc;
5130
5131         /* make sure adapter isn't asleep if manageability is enabled */
5132         if (adapter->en_mng_pt)
5133                 *enable_wake = true;
5134
5135         if (netif_running(netdev))
5136                 e1000_free_irq(adapter);
5137
5138         if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5139                 pci_disable_device(pdev);
5140
5141         return 0;
5142 }
5143
5144 #ifdef CONFIG_PM
5145 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state)
5146 {
5147         int retval;
5148         bool wake;
5149
5150         retval = __e1000_shutdown(pdev, &wake);
5151         if (retval)
5152                 return retval;
5153
5154         if (wake) {
5155                 pci_prepare_to_sleep(pdev);
5156         } else {
5157                 pci_wake_from_d3(pdev, false);
5158                 pci_set_power_state(pdev, PCI_D3hot);
5159         }
5160
5161         return 0;
5162 }
5163
5164 static int e1000_resume(struct pci_dev *pdev)
5165 {
5166         struct net_device *netdev = pci_get_drvdata(pdev);
5167         struct e1000_adapter *adapter = netdev_priv(netdev);
5168         struct e1000_hw *hw = &adapter->hw;
5169         u32 err;
5170
5171         pci_set_power_state(pdev, PCI_D0);
5172         pci_restore_state(pdev);
5173         pci_save_state(pdev);
5174
5175         if (adapter->need_ioport)
5176                 err = pci_enable_device(pdev);
5177         else
5178                 err = pci_enable_device_mem(pdev);
5179         if (err) {
5180                 pr_err("Cannot enable PCI device from suspend\n");
5181                 return err;
5182         }
5183
5184         /* flush memory to make sure state is correct */
5185         smp_mb__before_atomic();
5186         clear_bit(__E1000_DISABLED, &adapter->flags);
5187         pci_set_master(pdev);
5188
5189         pci_enable_wake(pdev, PCI_D3hot, 0);
5190         pci_enable_wake(pdev, PCI_D3cold, 0);
5191
5192         if (netif_running(netdev)) {
5193                 err = e1000_request_irq(adapter);
5194                 if (err)
5195                         return err;
5196         }
5197
5198         e1000_power_up_phy(adapter);
5199         e1000_reset(adapter);
5200         ew32(WUS, ~0);
5201
5202         e1000_init_manageability(adapter);
5203
5204         if (netif_running(netdev))
5205                 e1000_up(adapter);
5206
5207         netif_device_attach(netdev);
5208
5209         return 0;
5210 }
5211 #endif
5212
5213 static void e1000_shutdown(struct pci_dev *pdev)
5214 {
5215         bool wake;
5216
5217         __e1000_shutdown(pdev, &wake);
5218
5219         if (system_state == SYSTEM_POWER_OFF) {
5220                 pci_wake_from_d3(pdev, wake);
5221                 pci_set_power_state(pdev, PCI_D3hot);
5222         }
5223 }
5224
5225 #ifdef CONFIG_NET_POLL_CONTROLLER
5226 /* Polling 'interrupt' - used by things like netconsole to send skbs
5227  * without having to re-enable interrupts. It's not called while
5228  * the interrupt routine is executing.
5229  */
5230 static void e1000_netpoll(struct net_device *netdev)
5231 {
5232         struct e1000_adapter *adapter = netdev_priv(netdev);
5233
5234         if (disable_hardirq(adapter->pdev->irq))
5235                 e1000_intr(adapter->pdev->irq, netdev);
5236         enable_irq(adapter->pdev->irq);
5237 }
5238 #endif
5239
5240 /**
5241  * e1000_io_error_detected - called when PCI error is detected
5242  * @pdev: Pointer to PCI device
5243  * @state: The current pci connection state
5244  *
5245  * This function is called after a PCI bus error affecting
5246  * this device has been detected.
5247  */
5248 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
5249                                                 pci_channel_state_t state)
5250 {
5251         struct net_device *netdev = pci_get_drvdata(pdev);
5252         struct e1000_adapter *adapter = netdev_priv(netdev);
5253
5254         netif_device_detach(netdev);
5255
5256         if (state == pci_channel_io_perm_failure)
5257                 return PCI_ERS_RESULT_DISCONNECT;
5258
5259         if (netif_running(netdev))
5260                 e1000_down(adapter);
5261
5262         if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5263                 pci_disable_device(pdev);
5264
5265         /* Request a slot slot reset. */
5266         return PCI_ERS_RESULT_NEED_RESET;
5267 }
5268
5269 /**
5270  * e1000_io_slot_reset - called after the pci bus has been reset.
5271  * @pdev: Pointer to PCI device
5272  *
5273  * Restart the card from scratch, as if from a cold-boot. Implementation
5274  * resembles the first-half of the e1000_resume routine.
5275  */
5276 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
5277 {
5278         struct net_device *netdev = pci_get_drvdata(pdev);
5279         struct e1000_adapter *adapter = netdev_priv(netdev);
5280         struct e1000_hw *hw = &adapter->hw;
5281         int err;
5282
5283         if (adapter->need_ioport)
5284                 err = pci_enable_device(pdev);
5285         else
5286                 err = pci_enable_device_mem(pdev);
5287         if (err) {
5288                 pr_err("Cannot re-enable PCI device after reset.\n");
5289                 return PCI_ERS_RESULT_DISCONNECT;
5290         }
5291
5292         /* flush memory to make sure state is correct */
5293         smp_mb__before_atomic();
5294         clear_bit(__E1000_DISABLED, &adapter->flags);
5295         pci_set_master(pdev);
5296
5297         pci_enable_wake(pdev, PCI_D3hot, 0);
5298         pci_enable_wake(pdev, PCI_D3cold, 0);
5299
5300         e1000_reset(adapter);
5301         ew32(WUS, ~0);
5302
5303         return PCI_ERS_RESULT_RECOVERED;
5304 }
5305
5306 /**
5307  * e1000_io_resume - called when traffic can start flowing again.
5308  * @pdev: Pointer to PCI device
5309  *
5310  * This callback is called when the error recovery driver tells us that
5311  * its OK to resume normal operation. Implementation resembles the
5312  * second-half of the e1000_resume routine.
5313  */
5314 static void e1000_io_resume(struct pci_dev *pdev)
5315 {
5316         struct net_device *netdev = pci_get_drvdata(pdev);
5317         struct e1000_adapter *adapter = netdev_priv(netdev);
5318
5319         e1000_init_manageability(adapter);
5320
5321         if (netif_running(netdev)) {
5322                 if (e1000_up(adapter)) {
5323                         pr_info("can't bring device back up after reset\n");
5324                         return;
5325                 }
5326         }
5327
5328         netif_device_attach(netdev);
5329 }
5330
5331 /* e1000_main.c */