drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  36
  37 #include <linux/bitmap.h>
  38 #include <linux/crc32.h>
  39 #include <linux/ctype.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/err.h>
  42 #include <linux/etherdevice.h>
  43 #include <linux/firmware.h>
  44 #include <linux/if.h>
  45 #include <linux/if_vlan.h>
  46 #include <linux/init.h>
  47 #include <linux/log2.h>
  48 #include <linux/mdio.h>
  49 #include <linux/module.h>
  50 #include <linux/moduleparam.h>
  51 #include <linux/mutex.h>
  52 #include <linux/netdevice.h>
  53 #include <linux/pci.h>
  54 #include <linux/aer.h>
  55 #include <linux/rtnetlink.h>
  56 #include <linux/sched.h>
  57 #include <linux/seq_file.h>
  58 #include <linux/sockios.h>
  59 #include <linux/vmalloc.h>
  60 #include <linux/workqueue.h>
  61 #include <net/neighbour.h>
  62 #include <net/netevent.h>
  63 #include <asm/uaccess.h>
  64
  65 #include "cxgb4.h"
  66 #include "t4_regs.h"
  67 #include "t4_msg.h"
  68 #include "t4fw_api.h"
  69 #include "l2t.h"
  70
  71 #define DRV_VERSION "1.3.0-ko"
  72 #define DRV_DESC "Chelsio T4 Network Driver"
  73
  74 /*
  75  * Max interrupt hold-off timer value in us.  Queues fall back to this value
  76  * under extreme memory pressure so it's largish to give the system time to
  77  * recover.
  78  */
  79 #define MAX_SGE_TIMERVAL 200U
  80
  81 enum {
  82         /*
  83          * Physical Function provisioning constants.
  84          */
  85         PFRES_NVI = 4,                  /* # of Virtual Interfaces */
  86         PFRES_NETHCTRL = 128,           /* # of EQs used for ETH or CTRL Qs */
  87         PFRES_NIQFLINT = 128,           /* # of ingress Qs/w Free List(s)/intr
  88                                          */
  89         PFRES_NEQ = 256,                /* # of egress queues */
  90         PFRES_NIQ = 0,                  /* # of ingress queues */
  91         PFRES_TC = 0,                   /* PCI-E traffic class */
  92         PFRES_NEXACTF = 128,            /* # of exact MPS filters */
  93
  94         PFRES_R_CAPS = FW_CMD_CAP_PF,
  95         PFRES_WX_CAPS = FW_CMD_CAP_PF,
  96
  97 #ifdef CONFIG_PCI_IOV
  98         /*
  99          * Virtual Function provisioning constants.  We need two extra Ingress
 100          * Queues with Interrupt capability to serve as the VF's Firmware
 101          * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
 102          * neither will have Free Lists associated with them).  For each
 103          * Ethernet/Control Egress Queue and for each Free List, we need an
 104          * Egress Context.
 105          */
 106         VFRES_NPORTS = 1,               /* # of "ports" per VF */
 107         VFRES_NQSETS = 2,               /* # of "Queue Sets" per VF */
 108
 109         VFRES_NVI = VFRES_NPORTS,       /* # of Virtual Interfaces */
 110         VFRES_NETHCTRL = VFRES_NQSETS,  /* # of EQs used for ETH or CTRL Qs */
 111         VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
 112         VFRES_NEQ = VFRES_NQSETS*2,     /* # of egress queues */
 113         VFRES_NIQ = 0,                  /* # of non-fl/int ingress queues */
 114         VFRES_TC = 0,                   /* PCI-E traffic class */
 115         VFRES_NEXACTF = 16,             /* # of exact MPS filters */
 116
 117         VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
 118         VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
 119 #endif
 120 };
 121
 122 /*
 123  * Provide a Port Access Rights Mask for the specified PF/VF.  This is very
 124  * static and likely not to be useful in the long run.  We really need to
 125  * implement some form of persistent configuration which the firmware
 126  * controls.
 127  */
 128 static unsigned int pfvfres_pmask(struct adapter *adapter,
 129                                   unsigned int pf, unsigned int vf)
 130 {
 131         unsigned int portn, portvec;
 132
 133         /*
 134          * Give PF's access to all of the ports.
 135          */
 136         if (vf == 0)
 137                 return FW_PFVF_CMD_PMASK_MASK;
 138
 139         /*
 140          * For VFs, we'll assign them access to the ports based purely on the
 141          * PF.  We assign active ports in order, wrapping around if there are
 142          * fewer active ports than PFs: e.g. active port[pf % nports].
 143          * Unfortunately the adapter's port_info structs haven't been
 144          * initialized yet so we have to compute this.
 145          */
 146         if (adapter->params.nports == 0)
 147                 return 0;
 148
 149         portn = pf % adapter->params.nports;
 150         portvec = adapter->params.portvec;
 151         for (;;) {
 152                 /*
 153                  * Isolate the lowest set bit in the port vector.  If we're at
 154                  * the port number that we want, return that as the pmask.
 155                  * otherwise mask that bit out of the port vector and
 156                  * decrement our port number ...
 157                  */
 158                 unsigned int pmask = portvec ^ (portvec & (portvec-1));
 159                 if (portn == 0)
 160                         return pmask;
 161                 portn--;
 162                 portvec &= ~pmask;
 163         }
 164         /*NOTREACHED*/
 165 }
 166
 167 enum {
 168         MAX_TXQ_ENTRIES      = 16384,
 169         MAX_CTRL_TXQ_ENTRIES = 1024,
 170         MAX_RSPQ_ENTRIES     = 16384,
 171         MAX_RX_BUFFERS       = 16384,
 172         MIN_TXQ_ENTRIES      = 32,
 173         MIN_CTRL_TXQ_ENTRIES = 32,
 174         MIN_RSPQ_ENTRIES     = 128,
 175         MIN_FL_ENTRIES       = 16
 176 };
 177
 178 /* Host shadow copy of ingress filter entry.  This is in host native format
 179  * and doesn't match the ordering or bit order, etc. of the hardware of the
 180  * firmware command.  The use of bit-field structure elements is purely to
 181  * remind ourselves of the field size limitations and save memory in the case
 182  * where the filter table is large.
 183  */
 184 struct filter_entry {
 185         /* Administrative fields for filter.
 186          */
 187         u32 valid:1;            /* filter allocated and valid */
 188         u32 locked:1;           /* filter is administratively locked */
 189
 190         u32 pending:1;          /* filter action is pending firmware reply */
 191         u32 smtidx:8;           /* Source MAC Table index for smac */
 192         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 193
 194         /* The filter itself.  Most of this is a straight copy of information
 195          * provided by the extended ioctl().  Some fields are translated to
 196          * internal forms -- for instance the Ingress Queue ID passed in from
 197          * the ioctl() is translated into the Absolute Ingress Queue ID.
 198          */
 199         struct ch_filter_specification fs;
 200 };
 201
 202 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 203                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 204                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 205
 206 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
 207
 208 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
 209         CH_DEVICE(0xa000, 0),  /* PE10K */
 210         CH_DEVICE(0x4001, -1),
 211         CH_DEVICE(0x4002, -1),
 212         CH_DEVICE(0x4003, -1),
 213         CH_DEVICE(0x4004, -1),
 214         CH_DEVICE(0x4005, -1),
 215         CH_DEVICE(0x4006, -1),
 216         CH_DEVICE(0x4007, -1),
 217         CH_DEVICE(0x4008, -1),
 218         CH_DEVICE(0x4009, -1),
 219         CH_DEVICE(0x400a, -1),
 220         CH_DEVICE(0x4401, 4),
 221         CH_DEVICE(0x4402, 4),
 222         CH_DEVICE(0x4403, 4),
 223         CH_DEVICE(0x4404, 4),
 224         CH_DEVICE(0x4405, 4),
 225         CH_DEVICE(0x4406, 4),
 226         CH_DEVICE(0x4407, 4),
 227         CH_DEVICE(0x4408, 4),
 228         CH_DEVICE(0x4409, 4),
 229         CH_DEVICE(0x440a, 4),
 230         CH_DEVICE(0x440d, 4),
 231         CH_DEVICE(0x440e, 4),
 232         { 0, }
 233 };
 234
 235 #define FW_FNAME "cxgb4/t4fw.bin"
 236 #define FW5_FNAME "cxgb4/t5fw.bin"
 237 #define FW_CFNAME "cxgb4/t4-config.txt"
 238 #define FW5_CFNAME "cxgb4/t5-config.txt"
 239
 240 MODULE_DESCRIPTION(DRV_DESC);
 241 MODULE_AUTHOR("Chelsio Communications");
 242 MODULE_LICENSE("Dual BSD/GPL");
 243 MODULE_VERSION(DRV_VERSION);
 244 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
 245 MODULE_FIRMWARE(FW_FNAME);
 246 MODULE_FIRMWARE(FW5_FNAME);
 247
 248 /*
 249  * Normally we're willing to become the firmware's Master PF but will be happy
 250  * if another PF has already become the Master and initialized the adapter.
 251  * Setting "force_init" will cause this driver to forcibly establish itself as
 252  * the Master PF and initialize the adapter.
 253  */
 254 static uint force_init;
 255
 256 module_param(force_init, uint, 0644);
 257 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
 258
 259 /*
 260  * Normally if the firmware we connect to has Configuration File support, we
 261  * use that and only fall back to the old Driver-based initialization if the
 262  * Configuration File fails for some reason.  If force_old_init is set, then
 263  * we'll always use the old Driver-based initialization sequence.
 264  */
 265 static uint force_old_init;
 266
 267 module_param(force_old_init, uint, 0644);
 268 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");
 269
 270 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 271
 272 module_param(dflt_msg_enable, int, 0644);
 273 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
 274
 275 /*
 276  * The driver uses the best interrupt scheme available on a platform in the
 277  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 278  * of these schemes the driver may consider as follows:
 279  *
 280  * msi = 2: choose from among all three options
 281  * msi = 1: only consider MSI and INTx interrupts
 282  * msi = 0: force INTx interrupts
 283  */
 284 static int msi = 2;
 285
 286 module_param(msi, int, 0644);
 287 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
 288
 289 /*
 290  * Queue interrupt hold-off timer values.  Queues default to the first of these
 291  * upon creation.
 292  */
 293 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
 294
 295 module_param_array(intr_holdoff, uint, NULL, 0644);
 296 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
 297                  "0..4 in microseconds");
 298
 299 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
 300
 301 module_param_array(intr_cnt, uint, NULL, 0644);
 302 MODULE_PARM_DESC(intr_cnt,
 303                  "thresholds 1..3 for queue interrupt packet counters");
 304
 305 /*
 306  * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
 307  * offset by 2 bytes in order to have the IP headers line up on 4-byte
 308  * boundaries.  This is a requirement for many architectures which will throw
 309  * a machine check fault if an attempt is made to access one of the 4-byte IP
 310  * header fields on a non-4-byte boundary.  And it's a major performance issue
 311  * even on some architectures which allow it like some implementations of the
 312  * x86 ISA.  However, some architectures don't mind this and for some very
 313  * edge-case performance sensitive applications (like forwarding large volumes
 314  * of small packets), setting this DMA offset to 0 will decrease the number of
 315  * PCI-E Bus transfers enough to measurably affect performance.
 316  */
 317 static int rx_dma_offset = 2;
 318
 319 static bool vf_acls;
 320
 321 #ifdef CONFIG_PCI_IOV
 322 module_param(vf_acls, bool, 0644);
 323 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
 324
 325 /* Since T5 has more num of PFs, using NUM_OF_PF_WITH_SRIOV_T5
 326  * macro as num_vf array size
 327  */
 328 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV_T5];
 329
 330 module_param_array(num_vf, uint, NULL, 0644);
 331 MODULE_PARM_DESC(num_vf,
 332                  "number of VFs for each of PFs 0-3 for T4 and PFs 0-7 for T5");
 333 #endif
 334
 335 /*
 336  * The filter TCAM has a fixed portion and a variable portion.  The fixed
 337  * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
 338  * ports.  The variable portion is 36 bits which can include things like Exact
 339  * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
 340  * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
 341  * far exceed the 36-bit budget for this "compressed" header portion of the
 342  * filter.  Thus, we have a scarce resource which must be carefully managed.
 343  *
 344  * By default we set this up to mostly match the set of filter matching
 345  * capabilities of T3 but with accommodations for some of T4's more
 346  * interesting features:
 347  *
 348  *   { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
 349  *     [Inner] VLAN (17), Port (3), FCoE (1) }
 350  */
 351 enum {
 352         TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,
 353         TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT,
 354         TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT,
 355 };
 356
 357 static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
 358
 359 module_param(tp_vlan_pri_map, uint, 0644);
 360 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration");
 361
 362 static struct dentry *cxgb4_debugfs_root;
 363
 364 static LIST_HEAD(adapter_list);
 365 static DEFINE_MUTEX(uld_mutex);
 366 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
 367 static const char *uld_str[] = { "RDMA", "iSCSI" };
 368
 369 static void link_report(struct net_device *dev)
 370 {
 371         if (!netif_carrier_ok(dev))
 372                 netdev_info(dev, "link down\n");
 373         else {
 374                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
 375
 376                 const char *s = "10Mbps";
 377                 const struct port_info *p = netdev_priv(dev);
 378
 379                 switch (p->link_cfg.speed) {
 380                 case SPEED_10000:
 381                         s = "10Gbps";
 382                         break;
 383                 case SPEED_1000:
 384                         s = "1000Mbps";
 385                         break;
 386                 case SPEED_100:
 387                         s = "100Mbps";
 388                         break;
 389                 }
 390
 391                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
 392                             fc[p->link_cfg.fc]);
 393         }
 394 }
 395
 396 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 397 {
 398         struct net_device *dev = adapter->port[port_id];
 399
 400         /* Skip changes from disabled ports. */
 401         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
 402                 if (link_stat)
 403                         netif_carrier_on(dev);
 404                 else
 405                         netif_carrier_off(dev);
 406
 407                 link_report(dev);
 408         }
 409 }
 410
 411 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 412 {
 413         static const char *mod_str[] = {
 414                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 415         };
 416
 417         const struct net_device *dev = adap->port[port_id];
 418         const struct port_info *pi = netdev_priv(dev);
 419
 420         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 421                 netdev_info(dev, "port module unplugged\n");
 422         else if (pi->mod_type < ARRAY_SIZE(mod_str))
 423                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
 424 }
 425
 426 /*
 427  * Configure the exact and hash address filters to handle a port's multicast
 428  * and secondary unicast MAC addresses.
 429  */
 430 static int set_addr_filters(const struct net_device *dev, bool sleep)
 431 {
 432         u64 mhash = 0;
 433         u64 uhash = 0;
 434         bool free = true;
 435         u16 filt_idx[7];
 436         const u8 *addr[7];
 437         int ret, naddr = 0;
 438         const struct netdev_hw_addr *ha;
 439         int uc_cnt = netdev_uc_count(dev);
 440         int mc_cnt = netdev_mc_count(dev);
 441         const struct port_info *pi = netdev_priv(dev);
 442         unsigned int mb = pi->adapter->fn;
 443
 444         /* first do the secondary unicast addresses */
 445         netdev_for_each_uc_addr(ha, dev) {
 446                 addr[naddr++] = ha->addr;
 447                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 448                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 449                                         naddr, addr, filt_idx, &uhash, sleep);
 450                         if (ret < 0)
 451                                 return ret;
 452
 453                         free = false;
 454                         naddr = 0;
 455                 }
 456         }
 457
 458         /* next set up the multicast addresses */
 459         netdev_for_each_mc_addr(ha, dev) {
 460                 addr[naddr++] = ha->addr;
 461                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 462                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 463                                         naddr, addr, filt_idx, &mhash, sleep);
 464                         if (ret < 0)
 465                                 return ret;
 466
 467                         free = false;
 468                         naddr = 0;
 469                 }
 470         }
 471
 472         return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
 473                                 uhash | mhash, sleep);
 474 }
 475
 476 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
 477 module_param(dbfifo_int_thresh, int, 0644);
 478 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
 479
 480 /*
 481  * usecs to sleep while draining the dbfifo
 482  */
 483 static int dbfifo_drain_delay = 1000;
 484 module_param(dbfifo_drain_delay, int, 0644);
 485 MODULE_PARM_DESC(dbfifo_drain_delay,
 486                  "usecs to sleep while draining the dbfifo");
 487
 488 /*
 489  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 490  * If @mtu is -1 it is left unchanged.
 491  */
 492 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 493 {
 494         int ret;
 495         struct port_info *pi = netdev_priv(dev);
 496
 497         ret = set_addr_filters(dev, sleep_ok);
 498         if (ret == 0)
 499                 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
 500                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
 501                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
 502                                     sleep_ok);
 503         return ret;
 504 }
 505
 506 static struct workqueue_struct *workq;
 507
 508 /**
 509  *      link_start - enable a port
 510  *      @dev: the port to enable
 511  *
 512  *      Performs the MAC and PHY actions needed to enable a port.
 513  */
 514 static int link_start(struct net_device *dev)
 515 {
 516         int ret;
 517         struct port_info *pi = netdev_priv(dev);
 518         unsigned int mb = pi->adapter->fn;
 519
 520         /*
 521          * We do not set address filters and promiscuity here, the stack does
 522          * that step explicitly.
 523          */
 524         ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
 525                             !!(dev->features & NETIF_F_HW_VLAN_RX), true);
 526         if (ret == 0) {
 527                 ret = t4_change_mac(pi->adapter, mb, pi->viid,
 528                                     pi->xact_addr_filt, dev->dev_addr, true,
 529                                     true);
 530                 if (ret >= 0) {
 531                         pi->xact_addr_filt = ret;
 532                         ret = 0;
 533                 }
 534         }
 535         if (ret == 0)
 536                 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
 537                                     &pi->link_cfg);
 538         if (ret == 0)
 539                 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
 540         return ret;
 541 }
 542
 543 /* Clear a filter and release any of its resources that we own.  This also
 544  * clears the filter's "pending" status.
 545  */
 546 static void clear_filter(struct adapter *adap, struct filter_entry *f)
 547 {
 548         /* If the new or old filter have loopback rewriteing rules then we'll
 549          * need to free any existing Layer Two Table (L2T) entries of the old
 550          * filter rule.  The firmware will handle freeing up any Source MAC
 551          * Table (SMT) entries used for rewriting Source MAC Addresses in
 552          * loopback rules.
 553          */
 554         if (f->l2t)
 555                 cxgb4_l2t_release(f->l2t);
 556
 557         /* The zeroing of the filter rule below clears the filter valid,
 558          * pending, locked flags, l2t pointer, etc. so it's all we need for
 559          * this operation.
 560          */
 561         memset(f, 0, sizeof(*f));
 562 }
 563
 564 /* Handle a filter write/deletion reply.
 565  */
 566 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
 567 {
 568         unsigned int idx = GET_TID(rpl);
 569         unsigned int nidx = idx - adap->tids.ftid_base;
 570         unsigned int ret;
 571         struct filter_entry *f;
 572
 573         if (idx >= adap->tids.ftid_base && nidx <
 574            (adap->tids.nftids + adap->tids.nsftids)) {
 575                 idx = nidx;
 576                 ret = GET_TCB_COOKIE(rpl->cookie);
 577                 f = &adap->tids.ftid_tab[idx];
 578
 579                 if (ret == FW_FILTER_WR_FLT_DELETED) {
 580                         /* Clear the filter when we get confirmation from the
 581                          * hardware that the filter has been deleted.
 582                          */
 583                         clear_filter(adap, f);
 584                 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
 585                         dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
 586                                 idx);
 587                         clear_filter(adap, f);
 588                 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
 589                         f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
 590                         f->pending = 0;  /* asynchronous setup completed */
 591                         f->valid = 1;
 592                 } else {
 593                         /* Something went wrong.  Issue a warning about the
 594                          * problem and clear everything out.
 595                          */
 596                         dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
 597                                 idx, ret);
 598                         clear_filter(adap, f);
 599                 }
 600         }
 601 }
 602
 603 /* Response queue handler for the FW event queue.
 604  */
 605 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 606                           const struct pkt_gl *gl)
 607 {
 608         u8 opcode = ((const struct rss_header *)rsp)->opcode;
 609
 610         rsp++;                                          /* skip RSS header */
 611         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
 612                 const struct cpl_sge_egr_update *p = (void *)rsp;
 613                 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
 614                 struct sge_txq *txq;
 615
 616                 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
 617                 txq->restarts++;
 618                 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
 619                         struct sge_eth_txq *eq;
 620
 621                         eq = container_of(txq, struct sge_eth_txq, q);
 622                         netif_tx_wake_queue(eq->txq);
 623                 } else {
 624                         struct sge_ofld_txq *oq;
 625
 626                         oq = container_of(txq, struct sge_ofld_txq, q);
 627                         tasklet_schedule(&oq->qresume_tsk);
 628                 }
 629         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
 630                 const struct cpl_fw6_msg *p = (void *)rsp;
 631
 632                 if (p->type == 0)
 633                         t4_handle_fw_rpl(q->adap, p->data);
 634         } else if (opcode == CPL_L2T_WRITE_RPL) {
 635                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
 636
 637                 do_l2t_write_rpl(q->adap, p);
 638         } else if (opcode == CPL_SET_TCB_RPL) {
 639                 const struct cpl_set_tcb_rpl *p = (void *)rsp;
 640
 641                 filter_rpl(q->adap, p);
 642         } else
 643                 dev_err(q->adap->pdev_dev,
 644                         "unexpected CPL %#x on FW event queue\n", opcode);
 645         return 0;
 646 }
 647
 648 /**
 649  *      uldrx_handler - response queue handler for ULD queues
 650  *      @q: the response queue that received the packet
 651  *      @rsp: the response queue descriptor holding the offload message
 652  *      @gl: the gather list of packet fragments
 653  *
 654  *      Deliver an ingress offload packet to a ULD.  All processing is done by
 655  *      the ULD, we just maintain statistics.
 656  */
 657 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 658                          const struct pkt_gl *gl)
 659 {
 660         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 661
 662         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
 663                 rxq->stats.nomem++;
 664                 return -1;
 665         }
 666         if (gl == NULL)
 667                 rxq->stats.imm++;
 668         else if (gl == CXGB4_MSG_AN)
 669                 rxq->stats.an++;
 670         else
 671                 rxq->stats.pkts++;
 672         return 0;
 673 }
 674
 675 static void disable_msi(struct adapter *adapter)
 676 {
 677         if (adapter->flags & USING_MSIX) {
 678                 pci_disable_msix(adapter->pdev);
 679                 adapter->flags &= ~USING_MSIX;
 680         } else if (adapter->flags & USING_MSI) {
 681                 pci_disable_msi(adapter->pdev);
 682                 adapter->flags &= ~USING_MSI;
 683         }
 684 }
 685
 686 /*
 687  * Interrupt handler for non-data events used with MSI-X.
 688  */
 689 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
 690 {
 691         struct adapter *adap = cookie;
 692
 693         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
 694         if (v & PFSW) {
 695                 adap->swintr = 1;
 696                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
 697         }
 698         t4_slow_intr_handler(adap);
 699         return IRQ_HANDLED;
 700 }
 701
 702 /*
 703  * Name the MSI-X interrupts.
 704  */
 705 static void name_msix_vecs(struct adapter *adap)
 706 {
 707         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
 708
 709         /* non-data interrupts */
 710         snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
 711
 712         /* FW events */
 713         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
 714                  adap->port[0]->name);
 715
 716         /* Ethernet queues */
 717         for_each_port(adap, j) {
 718                 struct net_device *d = adap->port[j];
 719                 const struct port_info *pi = netdev_priv(d);
 720
 721                 for (i = 0; i < pi->nqsets; i++, msi_idx++)
 722                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 723                                  d->name, i);
 724         }
 725
 726         /* offload queues */
 727         for_each_ofldrxq(&adap->sge, i)
 728                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
 729                          adap->port[0]->name, i);
 730
 731         for_each_rdmarxq(&adap->sge, i)
 732                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
 733                          adap->port[0]->name, i);
 734 }
 735
 736 static int request_msix_queue_irqs(struct adapter *adap)
 737 {
 738         struct sge *s = &adap->sge;
 739         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi_index = 2;
 740
 741         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
 742                           adap->msix_info[1].desc, &s->fw_evtq);
 743         if (err)
 744                 return err;
 745
 746         for_each_ethrxq(s, ethqidx) {
 747                 err = request_irq(adap->msix_info[msi_index].vec,
 748                                   t4_sge_intr_msix, 0,
 749                                   adap->msix_info[msi_index].desc,
 750                                   &s->ethrxq[ethqidx].rspq);
 751                 if (err)
 752                         goto unwind;
 753                 msi_index++;
 754         }
 755         for_each_ofldrxq(s, ofldqidx) {
 756                 err = request_irq(adap->msix_info[msi_index].vec,
 757                                   t4_sge_intr_msix, 0,
 758                                   adap->msix_info[msi_index].desc,
 759                                   &s->ofldrxq[ofldqidx].rspq);
 760                 if (err)
 761                         goto unwind;
 762                 msi_index++;
 763         }
 764         for_each_rdmarxq(s, rdmaqidx) {
 765                 err = request_irq(adap->msix_info[msi_index].vec,
 766                                   t4_sge_intr_msix, 0,
 767                                   adap->msix_info[msi_index].desc,
 768                                   &s->rdmarxq[rdmaqidx].rspq);
 769                 if (err)
 770                         goto unwind;
 771                 msi_index++;
 772         }
 773         return 0;
 774
 775 unwind:
 776         while (--rdmaqidx >= 0)
 777                 free_irq(adap->msix_info[--msi_index].vec,
 778                          &s->rdmarxq[rdmaqidx].rspq);
 779         while (--ofldqidx >= 0)
 780                 free_irq(adap->msix_info[--msi_index].vec,
 781                          &s->ofldrxq[ofldqidx].rspq);
 782         while (--ethqidx >= 0)
 783                 free_irq(adap->msix_info[--msi_index].vec,
 784                          &s->ethrxq[ethqidx].rspq);
 785         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 786         return err;
 787 }
 788
 789 static void free_msix_queue_irqs(struct adapter *adap)
 790 {
 791         int i, msi_index = 2;
 792         struct sge *s = &adap->sge;
 793
 794         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 795         for_each_ethrxq(s, i)
 796                 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
 797         for_each_ofldrxq(s, i)
 798                 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
 799         for_each_rdmarxq(s, i)
 800                 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
 801 }
 802
 803 /**
 804  *      write_rss - write the RSS table for a given port
 805  *      @pi: the port
 806  *      @queues: array of queue indices for RSS
 807  *
 808  *      Sets up the portion of the HW RSS table for the port's VI to distribute
 809  *      packets to the Rx queues in @queues.
 810  */
 811 static int write_rss(const struct port_info *pi, const u16 *queues)
 812 {
 813         u16 *rss;
 814         int i, err;
 815         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
 816
 817         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
 818         if (!rss)
 819                 return -ENOMEM;
 820
 821         /* map the queue indices to queue ids */
 822         for (i = 0; i < pi->rss_size; i++, queues++)
 823                 rss[i] = q[*queues].rspq.abs_id;
 824
 825         err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
 826                                   pi->rss_size, rss, pi->rss_size);
 827         kfree(rss);
 828         return err;
 829 }
 830
 831 /**
 832  *      setup_rss - configure RSS
 833  *      @adap: the adapter
 834  *
 835  *      Sets up RSS for each port.
 836  */
 837 static int setup_rss(struct adapter *adap)
 838 {
 839         int i, err;
 840
 841         for_each_port(adap, i) {
 842                 const struct port_info *pi = adap2pinfo(adap, i);
 843
 844                 err = write_rss(pi, pi->rss);
 845                 if (err)
 846                         return err;
 847         }
 848         return 0;
 849 }
 850
 851 /*
 852  * Return the channel of the ingress queue with the given qid.
 853  */
 854 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
 855 {
 856         qid -= p->ingr_start;
 857         return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
 858 }
 859
 860 /*
 861  * Wait until all NAPI handlers are descheduled.
 862  */
 863 static void quiesce_rx(struct adapter *adap)
 864 {
 865         int i;
 866
 867         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 868                 struct sge_rspq *q = adap->sge.ingr_map[i];
 869
 870                 if (q && q->handler)
 871                         napi_disable(&q->napi);
 872         }
 873 }
 874
 875 /*
 876  * Enable NAPI scheduling and interrupt generation for all Rx queues.
 877  */
 878 static void enable_rx(struct adapter *adap)
 879 {
 880         int i;
 881
 882         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 883                 struct sge_rspq *q = adap->sge.ingr_map[i];
 884
 885                 if (!q)
 886                         continue;
 887                 if (q->handler)
 888                         napi_enable(&q->napi);
 889                 /* 0-increment GTS to start the timer and enable interrupts */
 890                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
 891                              SEINTARM(q->intr_params) |
 892                              INGRESSQID(q->cntxt_id));
 893         }
 894 }
 895
 896 /**
 897  *      setup_sge_queues - configure SGE Tx/Rx/response queues
 898  *      @adap: the adapter
 899  *
 900  *      Determines how many sets of SGE queues to use and initializes them.
 901  *      We support multiple queue sets per port if we have MSI-X, otherwise
 902  *      just one queue set per port.
 903  */
 904 static int setup_sge_queues(struct adapter *adap)
 905 {
 906         int err, msi_idx, i, j;
 907         struct sge *s = &adap->sge;
 908
 909         bitmap_zero(s->starving_fl, MAX_EGRQ);
 910         bitmap_zero(s->txq_maperr, MAX_EGRQ);
 911
 912         if (adap->flags & USING_MSIX)
 913                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
 914         else {
 915                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
 916                                        NULL, NULL);
 917                 if (err)
 918                         return err;
 919                 msi_idx = -((int)s->intrq.abs_id + 1);
 920         }
 921
 922         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
 923                                msi_idx, NULL, fwevtq_handler);
 924         if (err) {
 925 freeout:        t4_free_sge_resources(adap);
 926                 return err;
 927         }
 928
 929         for_each_port(adap, i) {
 930                 struct net_device *dev = adap->port[i];
 931                 struct port_info *pi = netdev_priv(dev);
 932                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
 933                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
 934
 935                 for (j = 0; j < pi->nqsets; j++, q++) {
 936                         if (msi_idx > 0)
 937                                 msi_idx++;
 938                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
 939                                                msi_idx, &q->fl,
 940                                                t4_ethrx_handler);
 941                         if (err)
 942                                 goto freeout;
 943                         q->rspq.idx = j;
 944                         memset(&q->stats, 0, sizeof(q->stats));
 945                 }
 946                 for (j = 0; j < pi->nqsets; j++, t++) {
 947                         err = t4_sge_alloc_eth_txq(adap, t, dev,
 948                                         netdev_get_tx_queue(dev, j),
 949                                         s->fw_evtq.cntxt_id);
 950                         if (err)
 951                                 goto freeout;
 952                 }
 953         }
 954
 955         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
 956         for_each_ofldrxq(s, i) {
 957                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
 958                 struct net_device *dev = adap->port[i / j];
 959
 960                 if (msi_idx > 0)
 961                         msi_idx++;
 962                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
 963                                        &q->fl, uldrx_handler);
 964                 if (err)
 965                         goto freeout;
 966                 memset(&q->stats, 0, sizeof(q->stats));
 967                 s->ofld_rxq[i] = q->rspq.abs_id;
 968                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
 969                                             s->fw_evtq.cntxt_id);
 970                 if (err)
 971                         goto freeout;
 972         }
 973
 974         for_each_rdmarxq(s, i) {
 975                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
 976
 977                 if (msi_idx > 0)
 978                         msi_idx++;
 979                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
 980                                        msi_idx, &q->fl, uldrx_handler);
 981                 if (err)
 982                         goto freeout;
 983                 memset(&q->stats, 0, sizeof(q->stats));
 984                 s->rdma_rxq[i] = q->rspq.abs_id;
 985         }
 986
 987         for_each_port(adap, i) {
 988                 /*
 989                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
 990                  * have RDMA queues, and that's the right value.
 991                  */
 992                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
 993                                             s->fw_evtq.cntxt_id,
 994                                             s->rdmarxq[i].rspq.cntxt_id);
 995                 if (err)
 996                         goto freeout;
 997         }
 998
 999         t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
1000                      RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
1001                      QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
1002         return 0;
1003 }
1004
1005 /*
1006  * Returns 0 if new FW was successfully loaded, a positive errno if a load was
1007  * started but failed, and a negative errno if flash load couldn't start.
1008  */
1009 static int upgrade_fw(struct adapter *adap)
1010 {
1011         int ret;
1012         u32 vers, exp_major;
1013         const struct fw_hdr *hdr;
1014         const struct firmware *fw;
1015         struct device *dev = adap->pdev_dev;
1016         char *fw_file_name;
1017
1018         switch (CHELSIO_CHIP_VERSION(adap->chip)) {
1019         case CHELSIO_T4:
1020                 fw_file_name = FW_FNAME;
1021                 exp_major = FW_VERSION_MAJOR;
1022                 break;
1023         case CHELSIO_T5:
1024                 fw_file_name = FW5_FNAME;
1025                 exp_major = FW_VERSION_MAJOR_T5;
1026                 break;
1027         default:
1028                 dev_err(dev, "Unsupported chip type, %x\n", adap->chip);
1029                 return -EINVAL;
1030         }
1031
1032         ret = request_firmware(&fw, fw_file_name, dev);
1033         if (ret < 0) {
1034                 dev_err(dev, "unable to load firmware image %s, error %d\n",
1035                         fw_file_name, ret);
1036                 return ret;
1037         }
1038
1039         hdr = (const struct fw_hdr *)fw->data;
1040         vers = ntohl(hdr->fw_ver);
1041         if (FW_HDR_FW_VER_MAJOR_GET(vers) != exp_major) {
1042                 ret = -EINVAL;              /* wrong major version, won't do */
1043                 goto out;
1044         }
1045
1046         /*
1047          * If the flash FW is unusable or we found something newer, load it.
1048          */
1049         if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != exp_major ||
1050             vers > adap->params.fw_vers) {
1051                 dev_info(dev, "upgrading firmware ...\n");
1052                 ret = t4_fw_upgrade(adap, adap->mbox, fw->data, fw->size,
1053                                     /*force=*/false);
1054                 if (!ret)
1055                         dev_info(dev,
1056                                  "firmware upgraded to version %pI4 from %s\n",
1057                                  &hdr->fw_ver, fw_file_name);
1058                 else
1059                         dev_err(dev, "firmware upgrade failed! err=%d\n", -ret);
1060         } else {
1061                 /*
1062                  * Tell our caller that we didn't upgrade the firmware.
1063                  */
1064                 ret = -EINVAL;
1065         }
1066
1067 out:    release_firmware(fw);
1068         return ret;
1069 }
1070
1071 /*
1072  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1073  * The allocated memory is cleared.
1074  */
1075 void *t4_alloc_mem(size_t size)
1076 {
1077         void *p = kzalloc(size, GFP_KERNEL);
1078
1079         if (!p)
1080                 p = vzalloc(size);
1081         return p;
1082 }
1083
1084 /*
1085  * Free memory allocated through alloc_mem().
1086  */
1087 static void t4_free_mem(void *addr)
1088 {
1089         if (is_vmalloc_addr(addr))
1090                 vfree(addr);
1091         else
1092                 kfree(addr);
1093 }
1094
1095 /* Send a Work Request to write the filter at a specified index.  We construct
1096  * a Firmware Filter Work Request to have the work done and put the indicated
1097  * filter into "pending" mode which will prevent any further actions against
1098  * it till we get a reply from the firmware on the completion status of the
1099  * request.
1100  */
1101 static int set_filter_wr(struct adapter *adapter, int fidx)
1102 {
1103         struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1104         struct sk_buff *skb;
1105         struct fw_filter_wr *fwr;
1106         unsigned int ftid;
1107
1108         /* If the new filter requires loopback Destination MAC and/or VLAN
1109          * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1110          * the filter.
1111          */
1112         if (f->fs.newdmac || f->fs.newvlan) {
1113                 /* allocate L2T entry for new filter */
1114                 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1115                 if (f->l2t == NULL)
1116                         return -EAGAIN;
1117                 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1118                                         f->fs.eport, f->fs.dmac)) {
1119                         cxgb4_l2t_release(f->l2t);
1120                         f->l2t = NULL;
1121                         return -ENOMEM;
1122                 }
1123         }
1124
1125         ftid = adapter->tids.ftid_base + fidx;
1126
1127         skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1128         fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1129         memset(fwr, 0, sizeof(*fwr));
1130
1131         /* It would be nice to put most of the following in t4_hw.c but most
1132          * of the work is translating the cxgbtool ch_filter_specification
1133          * into the Work Request and the definition of that structure is
1134          * currently in cxgbtool.h which isn't appropriate to pull into the
1135          * common code.  We may eventually try to come up with a more neutral
1136          * filter specification structure but for now it's easiest to simply
1137          * put this fairly direct code in line ...
1138          */
1139         fwr->op_pkd = htonl(FW_WR_OP(FW_FILTER_WR));
1140         fwr->len16_pkd = htonl(FW_WR_LEN16(sizeof(*fwr)/16));
1141         fwr->tid_to_iq =
1142                 htonl(V_FW_FILTER_WR_TID(ftid) |
1143                       V_FW_FILTER_WR_RQTYPE(f->fs.type) |
1144                       V_FW_FILTER_WR_NOREPLY(0) |
1145                       V_FW_FILTER_WR_IQ(f->fs.iq));
1146         fwr->del_filter_to_l2tix =
1147                 htonl(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
1148                       V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
1149                       V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
1150                       V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
1151                       V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
1152                       V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
1153                       V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
1154                       V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
1155                       V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
1156                                              f->fs.newvlan == VLAN_REWRITE) |
1157                       V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
1158                                             f->fs.newvlan == VLAN_REWRITE) |
1159                       V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
1160                       V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
1161                       V_FW_FILTER_WR_PRIO(f->fs.prio) |
1162                       V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
1163         fwr->ethtype = htons(f->fs.val.ethtype);
1164         fwr->ethtypem = htons(f->fs.mask.ethtype);
1165         fwr->frag_to_ovlan_vldm =
1166                 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
1167                  V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
1168                  V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.ivlan_vld) |
1169                  V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.ovlan_vld) |
1170                  V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.ivlan_vld) |
1171                  V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.ovlan_vld));
1172         fwr->smac_sel = 0;
1173         fwr->rx_chan_rx_rpl_iq =
1174                 htons(V_FW_FILTER_WR_RX_CHAN(0) |
1175                       V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id));
1176         fwr->maci_to_matchtypem =
1177                 htonl(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
1178                       V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
1179                       V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
1180                       V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
1181                       V_FW_FILTER_WR_PORT(f->fs.val.iport) |
1182                       V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
1183                       V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
1184                       V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
1185         fwr->ptcl = f->fs.val.proto;
1186         fwr->ptclm = f->fs.mask.proto;
1187         fwr->ttyp = f->fs.val.tos;
1188         fwr->ttypm = f->fs.mask.tos;
1189         fwr->ivlan = htons(f->fs.val.ivlan);
1190         fwr->ivlanm = htons(f->fs.mask.ivlan);
1191         fwr->ovlan = htons(f->fs.val.ovlan);
1192         fwr->ovlanm = htons(f->fs.mask.ovlan);
1193         memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1194         memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1195         memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1196         memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1197         fwr->lp = htons(f->fs.val.lport);
1198         fwr->lpm = htons(f->fs.mask.lport);
1199         fwr->fp = htons(f->fs.val.fport);
1200         fwr->fpm = htons(f->fs.mask.fport);
1201         if (f->fs.newsmac)
1202                 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1203
1204         /* Mark the filter as "pending" and ship off the Filter Work Request.
1205          * When we get the Work Request Reply we'll clear the pending status.
1206          */
1207         f->pending = 1;
1208         set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1209         t4_ofld_send(adapter, skb);
1210         return 0;
1211 }
1212
1213 /* Delete the filter at a specified index.
1214  */
1215 static int del_filter_wr(struct adapter *adapter, int fidx)
1216 {
1217         struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1218         struct sk_buff *skb;
1219         struct fw_filter_wr *fwr;
1220         unsigned int len, ftid;
1221
1222         len = sizeof(*fwr);
1223         ftid = adapter->tids.ftid_base + fidx;
1224
1225         skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1226         fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1227         t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1228
1229         /* Mark the filter as "pending" and ship off the Filter Work Request.
1230          * When we get the Work Request Reply we'll clear the pending status.
1231          */
1232         f->pending = 1;
1233         t4_mgmt_tx(adapter, skb);
1234         return 0;
1235 }
1236
1237 static inline int is_offload(const struct adapter *adap)
1238 {
1239         return adap->params.offload;
1240 }
1241
1242 /*
1243  * Implementation of ethtool operations.
1244  */
1245
1246 static u32 get_msglevel(struct net_device *dev)
1247 {
1248         return netdev2adap(dev)->msg_enable;
1249 }
1250
1251 static void set_msglevel(struct net_device *dev, u32 val)
1252 {
1253         netdev2adap(dev)->msg_enable = val;
1254 }
1255
1256 static char stats_strings[][ETH_GSTRING_LEN] = {
1257         "TxOctetsOK         ",
1258         "TxFramesOK         ",
1259         "TxBroadcastFrames  ",
1260         "TxMulticastFrames  ",
1261         "TxUnicastFrames    ",
1262         "TxErrorFrames      ",
1263
1264         "TxFrames64         ",
1265         "TxFrames65To127    ",
1266         "TxFrames128To255   ",
1267         "TxFrames256To511   ",
1268         "TxFrames512To1023  ",
1269         "TxFrames1024To1518 ",
1270         "TxFrames1519ToMax  ",
1271
1272         "TxFramesDropped    ",
1273         "TxPauseFrames      ",
1274         "TxPPP0Frames       ",
1275         "TxPPP1Frames       ",
1276         "TxPPP2Frames       ",
1277         "TxPPP3Frames       ",
1278         "TxPPP4Frames       ",
1279         "TxPPP5Frames       ",
1280         "TxPPP6Frames       ",
1281         "TxPPP7Frames       ",
1282
1283         "RxOctetsOK         ",
1284         "RxFramesOK         ",
1285         "RxBroadcastFrames  ",
1286         "RxMulticastFrames  ",
1287         "RxUnicastFrames    ",
1288
1289         "RxFramesTooLong    ",
1290         "RxJabberErrors     ",
1291         "RxFCSErrors        ",
1292         "RxLengthErrors     ",
1293         "RxSymbolErrors     ",
1294         "RxRuntFrames       ",
1295
1296         "RxFrames64         ",
1297         "RxFrames65To127    ",
1298         "RxFrames128To255   ",
1299         "RxFrames256To511   ",
1300         "RxFrames512To1023  ",
1301         "RxFrames1024To1518 ",
1302         "RxFrames1519ToMax  ",
1303
1304         "RxPauseFrames      ",
1305         "RxPPP0Frames       ",
1306         "RxPPP1Frames       ",
1307         "RxPPP2Frames       ",
1308         "RxPPP3Frames       ",
1309         "RxPPP4Frames       ",
1310         "RxPPP5Frames       ",
1311         "RxPPP6Frames       ",
1312         "RxPPP7Frames       ",
1313
1314         "RxBG0FramesDropped ",
1315         "RxBG1FramesDropped ",
1316         "RxBG2FramesDropped ",
1317         "RxBG3FramesDropped ",
1318         "RxBG0FramesTrunc   ",
1319         "RxBG1FramesTrunc   ",
1320         "RxBG2FramesTrunc   ",
1321         "RxBG3FramesTrunc   ",
1322
1323         "TSO                ",
1324         "TxCsumOffload      ",
1325         "RxCsumGood         ",
1326         "VLANextractions    ",
1327         "VLANinsertions     ",
1328         "GROpackets         ",
1329         "GROmerged          ",
1330 };
1331
1332 static int get_sset_count(struct net_device *dev, int sset)
1333 {
1334         switch (sset) {
1335         case ETH_SS_STATS:
1336                 return ARRAY_SIZE(stats_strings);
1337         default:
1338                 return -EOPNOTSUPP;
1339         }
1340 }
1341
1342 #define T4_REGMAP_SIZE (160 * 1024)
1343
1344 static int get_regs_len(struct net_device *dev)
1345 {
1346         return T4_REGMAP_SIZE;
1347 }
1348
1349 static int get_eeprom_len(struct net_device *dev)
1350 {
1351         return EEPROMSIZE;
1352 }
1353
1354 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1355 {
1356         struct adapter *adapter = netdev2adap(dev);
1357
1358         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1359         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1360         strlcpy(info->bus_info, pci_name(adapter->pdev),
1361                 sizeof(info->bus_info));
1362
1363         if (adapter->params.fw_vers)
1364                 snprintf(info->fw_version, sizeof(info->fw_version),
1365                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
1366                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1367                         FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1368                         FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1369                         FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1370                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1371                         FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1372                         FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1373                         FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1374 }
1375
1376 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1377 {
1378         if (stringset == ETH_SS_STATS)
1379                 memcpy(data, stats_strings, sizeof(stats_strings));
1380 }
1381
1382 /*
1383  * port stats maintained per queue of the port.  They should be in the same
1384  * order as in stats_strings above.
1385  */
1386 struct queue_port_stats {
1387         u64 tso;
1388         u64 tx_csum;
1389         u64 rx_csum;
1390         u64 vlan_ex;
1391         u64 vlan_ins;
1392         u64 gro_pkts;
1393         u64 gro_merged;
1394 };
1395
1396 static void collect_sge_port_stats(const struct adapter *adap,
1397                 const struct port_info *p, struct queue_port_stats *s)
1398 {
1399         int i;
1400         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1401         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1402
1403         memset(s, 0, sizeof(*s));
1404         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1405                 s->tso += tx->tso;
1406                 s->tx_csum += tx->tx_cso;
1407                 s->rx_csum += rx->stats.rx_cso;
1408                 s->vlan_ex += rx->stats.vlan_ex;
1409                 s->vlan_ins += tx->vlan_ins;
1410                 s->gro_pkts += rx->stats.lro_pkts;
1411                 s->gro_merged += rx->stats.lro_merged;
1412         }
1413 }
1414
1415 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1416                       u64 *data)
1417 {
1418         struct port_info *pi = netdev_priv(dev);
1419         struct adapter *adapter = pi->adapter;
1420
1421         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1422
1423         data += sizeof(struct port_stats) / sizeof(u64);
1424         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1425 }
1426
1427 /*
1428  * Return a version number to identify the type of adapter.  The scheme is:
1429  * - bits 0..9: chip version
1430  * - bits 10..15: chip revision
1431  * - bits 16..23: register dump version
1432  */
1433 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1434 {
1435         return CHELSIO_CHIP_VERSION(ap->chip) |
1436                 (CHELSIO_CHIP_RELEASE(ap->chip) << 10) | (1 << 16);
1437 }
1438
1439 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1440                            unsigned int end)
1441 {
1442         u32 *p = buf + start;
1443
1444         for ( ; start <= end; start += sizeof(u32))
1445                 *p++ = t4_read_reg(ap, start);
1446 }
1447
1448 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1449                      void *buf)
1450 {
1451         static const unsigned int reg_ranges[] = {
1452                 0x1008, 0x1108,
1453                 0x1180, 0x11b4,
1454                 0x11fc, 0x123c,
1455                 0x1300, 0x173c,
1456                 0x1800, 0x18fc,
1457                 0x3000, 0x30d8,
1458                 0x30e0, 0x5924,
1459                 0x5960, 0x59d4,
1460                 0x5a00, 0x5af8,
1461                 0x6000, 0x6098,
1462                 0x6100, 0x6150,
1463                 0x6200, 0x6208,
1464                 0x6240, 0x6248,
1465                 0x6280, 0x6338,
1466                 0x6370, 0x638c,
1467                 0x6400, 0x643c,
1468                 0x6500, 0x6524,
1469                 0x6a00, 0x6a38,
1470                 0x6a60, 0x6a78,
1471                 0x6b00, 0x6b84,
1472                 0x6bf0, 0x6c84,
1473                 0x6cf0, 0x6d84,
1474                 0x6df0, 0x6e84,
1475                 0x6ef0, 0x6f84,
1476                 0x6ff0, 0x7084,
1477                 0x70f0, 0x7184,
1478                 0x71f0, 0x7284,
1479                 0x72f0, 0x7384,
1480                 0x73f0, 0x7450,
1481                 0x7500, 0x7530,
1482                 0x7600, 0x761c,
1483                 0x7680, 0x76cc,
1484                 0x7700, 0x7798,
1485                 0x77c0, 0x77fc,
1486                 0x7900, 0x79fc,
1487                 0x7b00, 0x7c38,
1488                 0x7d00, 0x7efc,
1489                 0x8dc0, 0x8e1c,
1490                 0x8e30, 0x8e78,
1491                 0x8ea0, 0x8f6c,
1492                 0x8fc0, 0x9074,
1493                 0x90fc, 0x90fc,
1494                 0x9400, 0x9458,
1495                 0x9600, 0x96bc,
1496                 0x9800, 0x9808,
1497                 0x9820, 0x983c,
1498                 0x9850, 0x9864,
1499                 0x9c00, 0x9c6c,
1500                 0x9c80, 0x9cec,
1501                 0x9d00, 0x9d6c,
1502                 0x9d80, 0x9dec,
1503                 0x9e00, 0x9e6c,
1504                 0x9e80, 0x9eec,
1505                 0x9f00, 0x9f6c,
1506                 0x9f80, 0x9fec,
1507                 0xd004, 0xd03c,
1508                 0xdfc0, 0xdfe0,
1509                 0xe000, 0xea7c,
1510                 0xf000, 0x11190,
1511                 0x19040, 0x1906c,
1512                 0x19078, 0x19080,
1513                 0x1908c, 0x19124,
1514                 0x19150, 0x191b0,
1515                 0x191d0, 0x191e8,
1516                 0x19238, 0x1924c,
1517                 0x193f8, 0x19474,
1518                 0x19490, 0x194f8,
1519                 0x19800, 0x19f30,
1520                 0x1a000, 0x1a06c,
1521                 0x1a0b0, 0x1a120,
1522                 0x1a128, 0x1a138,
1523                 0x1a190, 0x1a1c4,
1524                 0x1a1fc, 0x1a1fc,
1525                 0x1e040, 0x1e04c,
1526                 0x1e284, 0x1e28c,
1527                 0x1e2c0, 0x1e2c0,
1528                 0x1e2e0, 0x1e2e0,
1529                 0x1e300, 0x1e384,
1530                 0x1e3c0, 0x1e3c8,
1531                 0x1e440, 0x1e44c,
1532                 0x1e684, 0x1e68c,
1533                 0x1e6c0, 0x1e6c0,
1534                 0x1e6e0, 0x1e6e0,
1535                 0x1e700, 0x1e784,
1536                 0x1e7c0, 0x1e7c8,
1537                 0x1e840, 0x1e84c,
1538                 0x1ea84, 0x1ea8c,
1539                 0x1eac0, 0x1eac0,
1540                 0x1eae0, 0x1eae0,
1541                 0x1eb00, 0x1eb84,
1542                 0x1ebc0, 0x1ebc8,
1543                 0x1ec40, 0x1ec4c,
1544                 0x1ee84, 0x1ee8c,
1545                 0x1eec0, 0x1eec0,
1546                 0x1eee0, 0x1eee0,
1547                 0x1ef00, 0x1ef84,
1548                 0x1efc0, 0x1efc8,
1549                 0x1f040, 0x1f04c,
1550                 0x1f284, 0x1f28c,
1551                 0x1f2c0, 0x1f2c0,
1552                 0x1f2e0, 0x1f2e0,
1553                 0x1f300, 0x1f384,
1554                 0x1f3c0, 0x1f3c8,
1555                 0x1f440, 0x1f44c,
1556                 0x1f684, 0x1f68c,
1557                 0x1f6c0, 0x1f6c0,
1558                 0x1f6e0, 0x1f6e0,
1559                 0x1f700, 0x1f784,
1560                 0x1f7c0, 0x1f7c8,
1561                 0x1f840, 0x1f84c,
1562                 0x1fa84, 0x1fa8c,
1563                 0x1fac0, 0x1fac0,
1564                 0x1fae0, 0x1fae0,
1565                 0x1fb00, 0x1fb84,
1566                 0x1fbc0, 0x1fbc8,
1567                 0x1fc40, 0x1fc4c,
1568                 0x1fe84, 0x1fe8c,
1569                 0x1fec0, 0x1fec0,
1570                 0x1fee0, 0x1fee0,
1571                 0x1ff00, 0x1ff84,
1572                 0x1ffc0, 0x1ffc8,
1573                 0x20000, 0x2002c,
1574                 0x20100, 0x2013c,
1575                 0x20190, 0x201c8,
1576                 0x20200, 0x20318,
1577                 0x20400, 0x20528,
1578                 0x20540, 0x20614,
1579                 0x21000, 0x21040,
1580                 0x2104c, 0x21060,
1581                 0x210c0, 0x210ec,
1582                 0x21200, 0x21268,
1583                 0x21270, 0x21284,
1584                 0x212fc, 0x21388,
1585                 0x21400, 0x21404,
1586                 0x21500, 0x21518,
1587                 0x2152c, 0x2153c,
1588                 0x21550, 0x21554,
1589                 0x21600, 0x21600,
1590                 0x21608, 0x21628,
1591                 0x21630, 0x2163c,
1592                 0x21700, 0x2171c,
1593                 0x21780, 0x2178c,
1594                 0x21800, 0x21c38,
1595                 0x21c80, 0x21d7c,
1596                 0x21e00, 0x21e04,
1597                 0x22000, 0x2202c,
1598                 0x22100, 0x2213c,
1599                 0x22190, 0x221c8,
1600                 0x22200, 0x22318,
1601                 0x22400, 0x22528,
1602                 0x22540, 0x22614,
1603                 0x23000, 0x23040,
1604                 0x2304c, 0x23060,
1605                 0x230c0, 0x230ec,
1606                 0x23200, 0x23268,
1607                 0x23270, 0x23284,
1608                 0x232fc, 0x23388,
1609                 0x23400, 0x23404,
1610                 0x23500, 0x23518,
1611                 0x2352c, 0x2353c,
1612                 0x23550, 0x23554,
1613                 0x23600, 0x23600,
1614                 0x23608, 0x23628,
1615                 0x23630, 0x2363c,
1616                 0x23700, 0x2371c,
1617                 0x23780, 0x2378c,
1618                 0x23800, 0x23c38,
1619                 0x23c80, 0x23d7c,
1620                 0x23e00, 0x23e04,
1621                 0x24000, 0x2402c,
1622                 0x24100, 0x2413c,
1623                 0x24190, 0x241c8,
1624                 0x24200, 0x24318,
1625                 0x24400, 0x24528,
1626                 0x24540, 0x24614,
1627                 0x25000, 0x25040,
1628                 0x2504c, 0x25060,
1629                 0x250c0, 0x250ec,
1630                 0x25200, 0x25268,
1631                 0x25270, 0x25284,
1632                 0x252fc, 0x25388,
1633                 0x25400, 0x25404,
1634                 0x25500, 0x25518,
1635                 0x2552c, 0x2553c,
1636                 0x25550, 0x25554,
1637                 0x25600, 0x25600,
1638                 0x25608, 0x25628,
1639                 0x25630, 0x2563c,
1640                 0x25700, 0x2571c,
1641                 0x25780, 0x2578c,
1642                 0x25800, 0x25c38,
1643                 0x25c80, 0x25d7c,
1644                 0x25e00, 0x25e04,
1645                 0x26000, 0x2602c,
1646                 0x26100, 0x2613c,
1647                 0x26190, 0x261c8,
1648                 0x26200, 0x26318,
1649                 0x26400, 0x26528,
1650                 0x26540, 0x26614,
1651                 0x27000, 0x27040,
1652                 0x2704c, 0x27060,
1653                 0x270c0, 0x270ec,
1654                 0x27200, 0x27268,
1655                 0x27270, 0x27284,
1656                 0x272fc, 0x27388,
1657                 0x27400, 0x27404,
1658                 0x27500, 0x27518,
1659                 0x2752c, 0x2753c,
1660                 0x27550, 0x27554,
1661                 0x27600, 0x27600,
1662                 0x27608, 0x27628,
1663                 0x27630, 0x2763c,
1664                 0x27700, 0x2771c,
1665                 0x27780, 0x2778c,
1666                 0x27800, 0x27c38,
1667                 0x27c80, 0x27d7c,
1668                 0x27e00, 0x27e04
1669         };
1670
1671         int i;
1672         struct adapter *ap = netdev2adap(dev);
1673
1674         regs->version = mk_adap_vers(ap);
1675
1676         memset(buf, 0, T4_REGMAP_SIZE);
1677         for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1678                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1679 }
1680
1681 static int restart_autoneg(struct net_device *dev)
1682 {
1683         struct port_info *p = netdev_priv(dev);
1684
1685         if (!netif_running(dev))
1686                 return -EAGAIN;
1687         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1688                 return -EINVAL;
1689         t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
1690         return 0;
1691 }
1692
1693 static int identify_port(struct net_device *dev,
1694                          enum ethtool_phys_id_state state)
1695 {
1696         unsigned int val;
1697         struct adapter *adap = netdev2adap(dev);
1698
1699         if (state == ETHTOOL_ID_ACTIVE)
1700                 val = 0xffff;
1701         else if (state == ETHTOOL_ID_INACTIVE)
1702                 val = 0;
1703         else
1704                 return -EINVAL;
1705
1706         return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
1707 }
1708
1709 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1710 {
1711         unsigned int v = 0;
1712
1713         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1714             type == FW_PORT_TYPE_BT_XAUI) {
1715                 v |= SUPPORTED_TP;
1716                 if (caps & FW_PORT_CAP_SPEED_100M)
1717                         v |= SUPPORTED_100baseT_Full;
1718                 if (caps & FW_PORT_CAP_SPEED_1G)
1719                         v |= SUPPORTED_1000baseT_Full;
1720                 if (caps & FW_PORT_CAP_SPEED_10G)
1721                         v |= SUPPORTED_10000baseT_Full;
1722         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1723                 v |= SUPPORTED_Backplane;
1724                 if (caps & FW_PORT_CAP_SPEED_1G)
1725                         v |= SUPPORTED_1000baseKX_Full;
1726                 if (caps & FW_PORT_CAP_SPEED_10G)
1727                         v |= SUPPORTED_10000baseKX4_Full;
1728         } else if (type == FW_PORT_TYPE_KR)
1729                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1730         else if (type == FW_PORT_TYPE_BP_AP)
1731                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1732                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
1733         else if (type == FW_PORT_TYPE_BP4_AP)
1734                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1735                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
1736                      SUPPORTED_10000baseKX4_Full;
1737         else if (type == FW_PORT_TYPE_FIBER_XFI ||
1738                  type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
1739                 v |= SUPPORTED_FIBRE;
1740
1741         if (caps & FW_PORT_CAP_ANEG)
1742                 v |= SUPPORTED_Autoneg;
1743         return v;
1744 }
1745
1746 static unsigned int to_fw_linkcaps(unsigned int caps)
1747 {
1748         unsigned int v = 0;
1749
1750         if (caps & ADVERTISED_100baseT_Full)
1751                 v |= FW_PORT_CAP_SPEED_100M;
1752         if (caps & ADVERTISED_1000baseT_Full)
1753                 v |= FW_PORT_CAP_SPEED_1G;
1754         if (caps & ADVERTISED_10000baseT_Full)
1755                 v |= FW_PORT_CAP_SPEED_10G;
1756         return v;
1757 }
1758
1759 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1760 {
1761         const struct port_info *p = netdev_priv(dev);
1762
1763         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1764             p->port_type == FW_PORT_TYPE_BT_XFI ||
1765             p->port_type == FW_PORT_TYPE_BT_XAUI)
1766                 cmd->port = PORT_TP;
1767         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1768                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1769                 cmd->port = PORT_FIBRE;
1770         else if (p->port_type == FW_PORT_TYPE_SFP) {
1771                 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1772                     p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1773                         cmd->port = PORT_DA;
1774                 else
1775                         cmd->port = PORT_FIBRE;
1776         } else
1777                 cmd->port = PORT_OTHER;
1778
1779         if (p->mdio_addr >= 0) {
1780                 cmd->phy_address = p->mdio_addr;
1781                 cmd->transceiver = XCVR_EXTERNAL;
1782                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1783                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1784         } else {
1785                 cmd->phy_address = 0;  /* not really, but no better option */
1786                 cmd->transceiver = XCVR_INTERNAL;
1787                 cmd->mdio_support = 0;
1788         }
1789
1790         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1791         cmd->advertising = from_fw_linkcaps(p->port_type,
1792                                             p->link_cfg.advertising);
1793         ethtool_cmd_speed_set(cmd,
1794                               netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
1795         cmd->duplex = DUPLEX_FULL;
1796         cmd->autoneg = p->link_cfg.autoneg;
1797         cmd->maxtxpkt = 0;
1798         cmd->maxrxpkt = 0;
1799         return 0;
1800 }
1801
1802 static unsigned int speed_to_caps(int speed)
1803 {
1804         if (speed == SPEED_100)
1805                 return FW_PORT_CAP_SPEED_100M;
1806         if (speed == SPEED_1000)
1807                 return FW_PORT_CAP_SPEED_1G;
1808         if (speed == SPEED_10000)
1809                 return FW_PORT_CAP_SPEED_10G;
1810         return 0;
1811 }
1812
1813 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1814 {
1815         unsigned int cap;
1816         struct port_info *p = netdev_priv(dev);
1817         struct link_config *lc = &p->link_cfg;
1818         u32 speed = ethtool_cmd_speed(cmd);
1819
1820         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
1821                 return -EINVAL;
1822
1823         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1824                 /*
1825                  * PHY offers a single speed.  See if that's what's
1826                  * being requested.
1827                  */
1828                 if (cmd->autoneg == AUTONEG_DISABLE &&
1829                     (lc->supported & speed_to_caps(speed)))
1830                         return 0;
1831                 return -EINVAL;
1832         }
1833
1834         if (cmd->autoneg == AUTONEG_DISABLE) {
1835                 cap = speed_to_caps(speed);
1836
1837                 if (!(lc->supported & cap) || (speed == SPEED_1000) ||
1838                     (speed == SPEED_10000))
1839                         return -EINVAL;
1840                 lc->requested_speed = cap;
1841                 lc->advertising = 0;
1842         } else {
1843                 cap = to_fw_linkcaps(cmd->advertising);
1844                 if (!(lc->supported & cap))
1845                         return -EINVAL;
1846                 lc->requested_speed = 0;
1847                 lc->advertising = cap | FW_PORT_CAP_ANEG;
1848         }
1849         lc->autoneg = cmd->autoneg;
1850
1851         if (netif_running(dev))
1852                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1853                                      lc);
1854         return 0;
1855 }
1856
1857 static void get_pauseparam(struct net_device *dev,
1858                            struct ethtool_pauseparam *epause)
1859 {
1860         struct port_info *p = netdev_priv(dev);
1861
1862         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1863         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1864         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1865 }
1866
1867 static int set_pauseparam(struct net_device *dev,
1868                           struct ethtool_pauseparam *epause)
1869 {
1870         struct port_info *p = netdev_priv(dev);
1871         struct link_config *lc = &p->link_cfg;
1872
1873         if (epause->autoneg == AUTONEG_DISABLE)
1874                 lc->requested_fc = 0;
1875         else if (lc->supported & FW_PORT_CAP_ANEG)
1876                 lc->requested_fc = PAUSE_AUTONEG;
1877         else
1878                 return -EINVAL;
1879
1880         if (epause->rx_pause)
1881                 lc->requested_fc |= PAUSE_RX;
1882         if (epause->tx_pause)
1883                 lc->requested_fc |= PAUSE_TX;
1884         if (netif_running(dev))
1885                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1886                                      lc);
1887         return 0;
1888 }
1889
1890 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1891 {
1892         const struct port_info *pi = netdev_priv(dev);
1893         const struct sge *s = &pi->adapter->sge;
1894
1895         e->rx_max_pending = MAX_RX_BUFFERS;
1896         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1897         e->rx_jumbo_max_pending = 0;
1898         e->tx_max_pending = MAX_TXQ_ENTRIES;
1899
1900         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1901         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1902         e->rx_jumbo_pending = 0;
1903         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1904 }
1905
1906 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1907 {
1908         int i;
1909         const struct port_info *pi = netdev_priv(dev);
1910         struct adapter *adapter = pi->adapter;
1911         struct sge *s = &adapter->sge;
1912
1913         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1914             e->tx_pending > MAX_TXQ_ENTRIES ||
1915             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1916             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1917             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1918                 return -EINVAL;
1919
1920         if (adapter->flags & FULL_INIT_DONE)
1921                 return -EBUSY;
1922
1923         for (i = 0; i < pi->nqsets; ++i) {
1924                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1925                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1926                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1927         }
1928         return 0;
1929 }
1930
1931 static int closest_timer(const struct sge *s, int time)
1932 {
1933         int i, delta, match = 0, min_delta = INT_MAX;
1934
1935         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1936                 delta = time - s->timer_val[i];
1937                 if (delta < 0)
1938                         delta = -delta;
1939                 if (delta < min_delta) {
1940                         min_delta = delta;
1941                         match = i;
1942                 }
1943         }
1944         return match;
1945 }
1946
1947 static int closest_thres(const struct sge *s, int thres)
1948 {
1949         int i, delta, match = 0, min_delta = INT_MAX;
1950
1951         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1952                 delta = thres - s->counter_val[i];
1953                 if (delta < 0)
1954                         delta = -delta;
1955                 if (delta < min_delta) {
1956                         min_delta = delta;
1957                         match = i;
1958                 }
1959         }
1960         return match;
1961 }
1962
1963 /*
1964  * Return a queue's interrupt hold-off time in us.  0 means no timer.
1965  */
1966 static unsigned int qtimer_val(const struct adapter *adap,
1967                                const struct sge_rspq *q)
1968 {
1969         unsigned int idx = q->intr_params >> 1;
1970
1971         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1972 }
1973
1974 /**
1975  *      set_rxq_intr_params - set a queue's interrupt holdoff parameters
1976  *      @adap: the adapter
1977  *      @q: the Rx queue
1978  *      @us: the hold-off time in us, or 0 to disable timer
1979  *      @cnt: the hold-off packet count, or 0 to disable counter
1980  *
1981  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
1982  *      one of the two needs to be enabled for the queue to generate interrupts.
1983  */
1984 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1985                                unsigned int us, unsigned int cnt)
1986 {
1987         if ((us | cnt) == 0)
1988                 cnt = 1;
1989
1990         if (cnt) {
1991                 int err;
1992                 u32 v, new_idx;
1993
1994                 new_idx = closest_thres(&adap->sge, cnt);
1995                 if (q->desc && q->pktcnt_idx != new_idx) {
1996                         /* the queue has already been created, update it */
1997                         v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1998                             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1999                             FW_PARAMS_PARAM_YZ(q->cntxt_id);
2000                         err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2001                                             &new_idx);
2002                         if (err)
2003                                 return err;
2004                 }
2005                 q->pktcnt_idx = new_idx;
2006         }
2007
2008         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2009         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2010         return 0;
2011 }
2012
2013 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2014 {
2015         const struct port_info *pi = netdev_priv(dev);
2016         struct adapter *adap = pi->adapter;
2017         struct sge_rspq *q;
2018         int i;
2019         int r = 0;
2020
2021         for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) {
2022                 q = &adap->sge.ethrxq[i].rspq;
2023                 r = set_rxq_intr_params(adap, q, c->rx_coalesce_usecs,
2024                         c->rx_max_coalesced_frames);
2025                 if (r) {
2026                         dev_err(&dev->dev, "failed to set coalesce %d\n", r);
2027                         break;
2028                 }
2029         }
2030         return r;
2031 }
2032
2033 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2034 {
2035         const struct port_info *pi = netdev_priv(dev);
2036         const struct adapter *adap = pi->adapter;
2037         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2038
2039         c->rx_coalesce_usecs = qtimer_val(adap, rq);
2040         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2041                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
2042         return 0;
2043 }
2044
2045 /**
2046  *      eeprom_ptov - translate a physical EEPROM address to virtual
2047  *      @phys_addr: the physical EEPROM address
2048  *      @fn: the PCI function number
2049  *      @sz: size of function-specific area
2050  *
2051  *      Translate a physical EEPROM address to virtual.  The first 1K is
2052  *      accessed through virtual addresses starting at 31K, the rest is
2053  *      accessed through virtual addresses starting at 0.
2054  *
2055  *      The mapping is as follows:
2056  *      [0..1K) -> [31K..32K)
2057  *      [1K..1K+A) -> [31K-A..31K)
2058  *      [1K+A..ES) -> [0..ES-A-1K)
2059  *
2060  *      where A = @fn * @sz, and ES = EEPROM size.
2061  */
2062 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
2063 {
2064         fn *= sz;
2065         if (phys_addr < 1024)
2066                 return phys_addr + (31 << 10);
2067         if (phys_addr < 1024 + fn)
2068                 return 31744 - fn + phys_addr - 1024;
2069         if (phys_addr < EEPROMSIZE)
2070                 return phys_addr - 1024 - fn;
2071         return -EINVAL;
2072 }
2073
2074 /*
2075  * The next two routines implement eeprom read/write from physical addresses.
2076  */
2077 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2078 {
2079         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2080
2081         if (vaddr >= 0)
2082                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2083         return vaddr < 0 ? vaddr : 0;
2084 }
2085
2086 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2087 {
2088         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2089
2090         if (vaddr >= 0)
2091                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2092         return vaddr < 0 ? vaddr : 0;
2093 }
2094
2095 #define EEPROM_MAGIC 0x38E2F10C
2096
2097 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2098                       u8 *data)
2099 {
2100         int i, err = 0;
2101         struct adapter *adapter = netdev2adap(dev);
2102
2103         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2104         if (!buf)
2105                 return -ENOMEM;
2106
2107         e->magic = EEPROM_MAGIC;
2108         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2109                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2110
2111         if (!err)
2112                 memcpy(data, buf + e->offset, e->len);
2113         kfree(buf);
2114         return err;
2115 }
2116
2117 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2118                       u8 *data)
2119 {
2120         u8 *buf;
2121         int err = 0;
2122         u32 aligned_offset, aligned_len, *p;
2123         struct adapter *adapter = netdev2adap(dev);
2124
2125         if (eeprom->magic != EEPROM_MAGIC)
2126                 return -EINVAL;
2127
2128         aligned_offset = eeprom->offset & ~3;
2129         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2130
2131         if (adapter->fn > 0) {
2132                 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2133
2134                 if (aligned_offset < start ||
2135                     aligned_offset + aligned_len > start + EEPROMPFSIZE)
2136                         return -EPERM;
2137         }
2138
2139         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2140                 /*
2141                  * RMW possibly needed for first or last words.
2142                  */
2143                 buf = kmalloc(aligned_len, GFP_KERNEL);
2144                 if (!buf)
2145                         return -ENOMEM;
2146                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2147                 if (!err && aligned_len > 4)
2148                         err = eeprom_rd_phys(adapter,
2149                                              aligned_offset + aligned_len - 4,
2150                                              (u32 *)&buf[aligned_len - 4]);
2151                 if (err)
2152                         goto out;
2153                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2154         } else
2155                 buf = data;
2156
2157         err = t4_seeprom_wp(adapter, false);
2158         if (err)
2159                 goto out;
2160
2161         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2162                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2163                 aligned_offset += 4;
2164         }
2165
2166         if (!err)
2167                 err = t4_seeprom_wp(adapter, true);
2168 out:
2169         if (buf != data)
2170                 kfree(buf);
2171         return err;
2172 }
2173
2174 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2175 {
2176         int ret;
2177         const struct firmware *fw;
2178         struct adapter *adap = netdev2adap(netdev);
2179
2180         ef->data[sizeof(ef->data) - 1] = '\0';
2181         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2182         if (ret < 0)
2183                 return ret;
2184
2185         ret = t4_load_fw(adap, fw->data, fw->size);
2186         release_firmware(fw);
2187         if (!ret)
2188                 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
2189         return ret;
2190 }
2191
2192 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2193 #define BCAST_CRC 0xa0ccc1a6
2194
2195 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2196 {
2197         wol->supported = WAKE_BCAST | WAKE_MAGIC;
2198         wol->wolopts = netdev2adap(dev)->wol;
2199         memset(&wol->sopass, 0, sizeof(wol->sopass));
2200 }
2201
2202 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2203 {
2204         int err = 0;
2205         struct port_info *pi = netdev_priv(dev);
2206
2207         if (wol->wolopts & ~WOL_SUPPORTED)
2208                 return -EINVAL;
2209         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2210                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2211         if (wol->wolopts & WAKE_BCAST) {
2212                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2213                                         ~0ULL, 0, false);
2214                 if (!err)
2215                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2216                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
2217         } else
2218                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2219         return err;
2220 }
2221
2222 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2223 {
2224         const struct port_info *pi = netdev_priv(dev);
2225         netdev_features_t changed = dev->features ^ features;
2226         int err;
2227
2228         if (!(changed & NETIF_F_HW_VLAN_RX))
2229                 return 0;
2230
2231         err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2232                             -1, -1, -1,
2233                             !!(features & NETIF_F_HW_VLAN_RX), true);
2234         if (unlikely(err))
2235                 dev->features = features ^ NETIF_F_HW_VLAN_RX;
2236         return err;
2237 }
2238
2239 static u32 get_rss_table_size(struct net_device *dev)
2240 {
2241         const struct port_info *pi = netdev_priv(dev);
2242
2243         return pi->rss_size;
2244 }
2245
2246 static int get_rss_table(struct net_device *dev, u32 *p)
2247 {
2248         const struct port_info *pi = netdev_priv(dev);
2249         unsigned int n = pi->rss_size;
2250
2251         while (n--)
2252                 p[n] = pi->rss[n];
2253         return 0;
2254 }
2255
2256 static int set_rss_table(struct net_device *dev, const u32 *p)
2257 {
2258         unsigned int i;
2259         struct port_info *pi = netdev_priv(dev);
2260
2261         for (i = 0; i < pi->rss_size; i++)
2262                 pi->rss[i] = p[i];
2263         if (pi->adapter->flags & FULL_INIT_DONE)
2264                 return write_rss(pi, pi->rss);
2265         return 0;
2266 }
2267
2268 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
2269                      u32 *rules)
2270 {
2271         const struct port_info *pi = netdev_priv(dev);
2272
2273         switch (info->cmd) {
2274         case ETHTOOL_GRXFH: {
2275                 unsigned int v = pi->rss_mode;
2276
2277                 info->data = 0;
2278                 switch (info->flow_type) {
2279                 case TCP_V4_FLOW:
2280                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
2281                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2282                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2283                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2284                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2285                         break;
2286                 case UDP_V4_FLOW:
2287                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
2288                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2289                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2290                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2291                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2292                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2293                         break;
2294                 case SCTP_V4_FLOW:
2295                 case AH_ESP_V4_FLOW:
2296                 case IPV4_FLOW:
2297                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2298                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2299                         break;
2300                 case TCP_V6_FLOW:
2301                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
2302                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2303                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2304                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2305                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2306                         break;
2307                 case UDP_V6_FLOW:
2308                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
2309                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2310                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2311                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2312                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2313                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2314                         break;
2315                 case SCTP_V6_FLOW:
2316                 case AH_ESP_V6_FLOW:
2317                 case IPV6_FLOW:
2318                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2319                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2320                         break;
2321                 }
2322                 return 0;
2323         }
2324         case ETHTOOL_GRXRINGS:
2325                 info->data = pi->nqsets;
2326                 return 0;
2327         }
2328         return -EOPNOTSUPP;
2329 }
2330
2331 static const struct ethtool_ops cxgb_ethtool_ops = {
2332         .get_settings      = get_settings,
2333         .set_settings      = set_settings,
2334         .get_drvinfo       = get_drvinfo,
2335         .get_msglevel      = get_msglevel,
2336         .set_msglevel      = set_msglevel,
2337         .get_ringparam     = get_sge_param,
2338         .set_ringparam     = set_sge_param,
2339         .get_coalesce      = get_coalesce,
2340         .set_coalesce      = set_coalesce,
2341         .get_eeprom_len    = get_eeprom_len,
2342         .get_eeprom        = get_eeprom,
2343         .set_eeprom        = set_eeprom,
2344         .get_pauseparam    = get_pauseparam,
2345         .set_pauseparam    = set_pauseparam,
2346         .get_link          = ethtool_op_get_link,
2347         .get_strings       = get_strings,
2348         .set_phys_id       = identify_port,
2349         .nway_reset        = restart_autoneg,
2350         .get_sset_count    = get_sset_count,
2351         .get_ethtool_stats = get_stats,
2352         .get_regs_len      = get_regs_len,
2353         .get_regs          = get_regs,
2354         .get_wol           = get_wol,
2355         .set_wol           = set_wol,
2356         .get_rxnfc         = get_rxnfc,
2357         .get_rxfh_indir_size = get_rss_table_size,
2358         .get_rxfh_indir    = get_rss_table,
2359         .set_rxfh_indir    = set_rss_table,
2360         .flash_device      = set_flash,
2361 };
2362
2363 /*
2364  * debugfs support
2365  */
2366 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2367                         loff_t *ppos)
2368 {
2369         loff_t pos = *ppos;
2370         loff_t avail = file_inode(file)->i_size;
2371         unsigned int mem = (uintptr_t)file->private_data & 3;
2372         struct adapter *adap = file->private_data - mem;
2373
2374         if (pos < 0)
2375                 return -EINVAL;
2376         if (pos >= avail)
2377                 return 0;
2378         if (count > avail - pos)
2379                 count = avail - pos;
2380
2381         while (count) {
2382                 size_t len;
2383                 int ret, ofst;
2384                 __be32 data[16];
2385
2386                 if (mem == MEM_MC)
2387                         ret = t4_mc_read(adap, pos, data, NULL);
2388                 else
2389                         ret = t4_edc_read(adap, mem, pos, data, NULL);
2390                 if (ret)
2391                         return ret;
2392
2393                 ofst = pos % sizeof(data);
2394                 len = min(count, sizeof(data) - ofst);
2395                 if (copy_to_user(buf, (u8 *)data + ofst, len))
2396                         return -EFAULT;
2397
2398                 buf += len;
2399                 pos += len;
2400                 count -= len;
2401         }
2402         count = pos - *ppos;
2403         *ppos = pos;
2404         return count;
2405 }
2406
2407 static const struct file_operations mem_debugfs_fops = {
2408         .owner   = THIS_MODULE,
2409         .open    = simple_open,
2410         .read    = mem_read,
2411         .llseek  = default_llseek,
2412 };
2413
2414 static void add_debugfs_mem(struct adapter *adap, const char *name,
2415                             unsigned int idx, unsigned int size_mb)
2416 {
2417         struct dentry *de;
2418
2419         de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2420                                  (void *)adap + idx, &mem_debugfs_fops);
2421         if (de && de->d_inode)
2422                 de->d_inode->i_size = size_mb << 20;
2423 }
2424
2425 static int setup_debugfs(struct adapter *adap)
2426 {
2427         int i;
2428
2429         if (IS_ERR_OR_NULL(adap->debugfs_root))
2430                 return -1;
2431
2432         i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2433         if (i & EDRAM0_ENABLE)
2434                 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2435         if (i & EDRAM1_ENABLE)
2436                 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2437         if (i & EXT_MEM_ENABLE)
2438                 add_debugfs_mem(adap, "mc", MEM_MC,
2439                         EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2440         if (adap->l2t)
2441                 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2442                                     &t4_l2t_fops);
2443         return 0;
2444 }
2445
2446 /*
2447  * upper-layer driver support
2448  */
2449
2450 /*
2451  * Allocate an active-open TID and set it to the supplied value.
2452  */
2453 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2454 {
2455         int atid = -1;
2456
2457         spin_lock_bh(&t->atid_lock);
2458         if (t->afree) {
2459                 union aopen_entry *p = t->afree;
2460
2461                 atid = (p - t->atid_tab) + t->atid_base;
2462                 t->afree = p->next;
2463                 p->data = data;
2464                 t->atids_in_use++;
2465         }
2466         spin_unlock_bh(&t->atid_lock);
2467         return atid;
2468 }
2469 EXPORT_SYMBOL(cxgb4_alloc_atid);
2470
2471 /*
2472  * Release an active-open TID.
2473  */
2474 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2475 {
2476         union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
2477
2478         spin_lock_bh(&t->atid_lock);
2479         p->next = t->afree;
2480         t->afree = p;
2481         t->atids_in_use--;
2482         spin_unlock_bh(&t->atid_lock);
2483 }
2484 EXPORT_SYMBOL(cxgb4_free_atid);
2485
2486 /*
2487  * Allocate a server TID and set it to the supplied value.
2488  */
2489 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2490 {
2491         int stid;
2492
2493         spin_lock_bh(&t->stid_lock);
2494         if (family == PF_INET) {
2495                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2496                 if (stid < t->nstids)
2497                         __set_bit(stid, t->stid_bmap);
2498                 else
2499                         stid = -1;
2500         } else {
2501                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2502                 if (stid < 0)
2503                         stid = -1;
2504         }
2505         if (stid >= 0) {
2506                 t->stid_tab[stid].data = data;
2507                 stid += t->stid_base;
2508                 t->stids_in_use++;
2509         }
2510         spin_unlock_bh(&t->stid_lock);
2511         return stid;
2512 }
2513 EXPORT_SYMBOL(cxgb4_alloc_stid);
2514
2515 /* Allocate a server filter TID and set it to the supplied value.
2516  */
2517 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
2518 {
2519         int stid;
2520
2521         spin_lock_bh(&t->stid_lock);
2522         if (family == PF_INET) {
2523                 stid = find_next_zero_bit(t->stid_bmap,
2524                                 t->nstids + t->nsftids, t->nstids);
2525                 if (stid < (t->nstids + t->nsftids))
2526                         __set_bit(stid, t->stid_bmap);
2527                 else
2528                         stid = -1;
2529         } else {
2530                 stid = -1;
2531         }
2532         if (stid >= 0) {
2533                 t->stid_tab[stid].data = data;
2534                 stid += t->stid_base;
2535                 t->stids_in_use++;
2536         }
2537         spin_unlock_bh(&t->stid_lock);
2538         return stid;
2539 }
2540 EXPORT_SYMBOL(cxgb4_alloc_sftid);
2541
2542 /* Release a server TID.
2543  */
2544 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2545 {
2546         stid -= t->stid_base;
2547         spin_lock_bh(&t->stid_lock);
2548         if (family == PF_INET)
2549                 __clear_bit(stid, t->stid_bmap);
2550         else
2551                 bitmap_release_region(t->stid_bmap, stid, 2);
2552         t->stid_tab[stid].data = NULL;
2553         t->stids_in_use--;
2554         spin_unlock_bh(&t->stid_lock);
2555 }
2556 EXPORT_SYMBOL(cxgb4_free_stid);
2557
2558 /*
2559  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
2560  */
2561 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2562                            unsigned int tid)
2563 {
2564         struct cpl_tid_release *req;
2565
2566         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2567         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2568         INIT_TP_WR(req, tid);
2569         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2570 }
2571
2572 /*
2573  * Queue a TID release request and if necessary schedule a work queue to
2574  * process it.
2575  */
2576 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2577                                     unsigned int tid)
2578 {
2579         void **p = &t->tid_tab[tid];
2580         struct adapter *adap = container_of(t, struct adapter, tids);
2581
2582         spin_lock_bh(&adap->tid_release_lock);
2583         *p = adap->tid_release_head;
2584         /* Low 2 bits encode the Tx channel number */
2585         adap->tid_release_head = (void **)((uintptr_t)p | chan);
2586         if (!adap->tid_release_task_busy) {
2587                 adap->tid_release_task_busy = true;
2588                 queue_work(workq, &adap->tid_release_task);
2589         }
2590         spin_unlock_bh(&adap->tid_release_lock);
2591 }
2592
2593 /*
2594  * Process the list of pending TID release requests.
2595  */
2596 static void process_tid_release_list(struct work_struct *work)
2597 {
2598         struct sk_buff *skb;
2599         struct adapter *adap;
2600
2601         adap = container_of(work, struct adapter, tid_release_task);
2602
2603         spin_lock_bh(&adap->tid_release_lock);
2604         while (adap->tid_release_head) {
2605                 void **p = adap->tid_release_head;
2606                 unsigned int chan = (uintptr_t)p & 3;
2607                 p = (void *)p - chan;
2608
2609                 adap->tid_release_head = *p;
2610                 *p = NULL;
2611                 spin_unlock_bh(&adap->tid_release_lock);
2612
2613                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2614                                          GFP_KERNEL)))
2615                         schedule_timeout_uninterruptible(1);
2616
2617                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2618                 t4_ofld_send(adap, skb);
2619                 spin_lock_bh(&adap->tid_release_lock);
2620         }
2621         adap->tid_release_task_busy = false;
2622         spin_unlock_bh(&adap->tid_release_lock);
2623 }
2624
2625 /*
2626  * Release a TID and inform HW.  If we are unable to allocate the release
2627  * message we defer to a work queue.
2628  */
2629 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2630 {
2631         void *old;
2632         struct sk_buff *skb;
2633         struct adapter *adap = container_of(t, struct adapter, tids);
2634
2635         old = t->tid_tab[tid];
2636         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2637         if (likely(skb)) {
2638                 t->tid_tab[tid] = NULL;
2639                 mk_tid_release(skb, chan, tid);
2640                 t4_ofld_send(adap, skb);
2641         } else
2642                 cxgb4_queue_tid_release(t, chan, tid);
2643         if (old)
2644                 atomic_dec(&t->tids_in_use);
2645 }
2646 EXPORT_SYMBOL(cxgb4_remove_tid);
2647
2648 /*
2649  * Allocate and initialize the TID tables.  Returns 0 on success.
2650  */
2651 static int tid_init(struct tid_info *t)
2652 {
2653         size_t size;
2654         unsigned int stid_bmap_size;
2655         unsigned int natids = t->natids;
2656
2657         stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
2658         size = t->ntids * sizeof(*t->tid_tab) +
2659                natids * sizeof(*t->atid_tab) +
2660                t->nstids * sizeof(*t->stid_tab) +
2661                t->nsftids * sizeof(*t->stid_tab) +
2662                stid_bmap_size * sizeof(long) +
2663                t->nftids * sizeof(*t->ftid_tab) +
2664                t->nsftids * sizeof(*t->ftid_tab);
2665
2666         t->tid_tab = t4_alloc_mem(size);
2667         if (!t->tid_tab)
2668                 return -ENOMEM;
2669
2670         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2671         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2672         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
2673         t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
2674         spin_lock_init(&t->stid_lock);
2675         spin_lock_init(&t->atid_lock);
2676
2677         t->stids_in_use = 0;
2678         t->afree = NULL;
2679         t->atids_in_use = 0;
2680         atomic_set(&t->tids_in_use, 0);
2681
2682         /* Setup the free list for atid_tab and clear the stid bitmap. */
2683         if (natids) {
2684                 while (--natids)
2685                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2686                 t->afree = t->atid_tab;
2687         }
2688         bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
2689         return 0;
2690 }
2691
2692 /**
2693  *      cxgb4_create_server - create an IP server
2694  *      @dev: the device
2695  *      @stid: the server TID
2696  *      @sip: local IP address to bind server to
2697  *      @sport: the server's TCP port
2698  *      @queue: queue to direct messages from this server to
2699  *
2700  *      Create an IP server for the given port and address.
2701  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
2702  */
2703 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2704                         __be32 sip, __be16 sport, __be16 vlan,
2705                         unsigned int queue)
2706 {
2707         unsigned int chan;
2708         struct sk_buff *skb;
2709         struct adapter *adap;
2710         struct cpl_pass_open_req *req;
2711
2712         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2713         if (!skb)
2714                 return -ENOMEM;
2715
2716         adap = netdev2adap(dev);
2717         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2718         INIT_TP_WR(req, 0);
2719         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2720         req->local_port = sport;
2721         req->peer_port = htons(0);
2722         req->local_ip = sip;
2723         req->peer_ip = htonl(0);
2724         chan = rxq_to_chan(&adap->sge, queue);
2725         req->opt0 = cpu_to_be64(TX_CHAN(chan));
2726         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2727                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2728         return t4_mgmt_tx(adap, skb);
2729 }
2730 EXPORT_SYMBOL(cxgb4_create_server);
2731
2732 /**
2733  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2734  *      @mtus: the HW MTU table
2735  *      @mtu: the target MTU
2736  *      @idx: index of selected entry in the MTU table
2737  *
2738  *      Returns the index and the value in the HW MTU table that is closest to
2739  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
2740  *      table, in which case that smallest available value is selected.
2741  */
2742 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2743                             unsigned int *idx)
2744 {
2745         unsigned int i = 0;
2746
2747         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2748                 ++i;
2749         if (idx)
2750                 *idx = i;
2751         return mtus[i];
2752 }
2753 EXPORT_SYMBOL(cxgb4_best_mtu);
2754
2755 /**
2756  *      cxgb4_port_chan - get the HW channel of a port
2757  *      @dev: the net device for the port
2758  *
2759  *      Return the HW Tx channel of the given port.
2760  */
2761 unsigned int cxgb4_port_chan(const struct net_device *dev)
2762 {
2763         return netdev2pinfo(dev)->tx_chan;
2764 }
2765 EXPORT_SYMBOL(cxgb4_port_chan);
2766
2767 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
2768 {
2769         struct adapter *adap = netdev2adap(dev);
2770         u32 v;
2771
2772         v = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
2773         return lpfifo ? G_LP_COUNT(v) : G_HP_COUNT(v);
2774 }
2775 EXPORT_SYMBOL(cxgb4_dbfifo_count);
2776
2777 /**
2778  *      cxgb4_port_viid - get the VI id of a port
2779  *      @dev: the net device for the port
2780  *
2781  *      Return the VI id of the given port.
2782  */
2783 unsigned int cxgb4_port_viid(const struct net_device *dev)
2784 {
2785         return netdev2pinfo(dev)->viid;
2786 }
2787 EXPORT_SYMBOL(cxgb4_port_viid);
2788
2789 /**
2790  *      cxgb4_port_idx - get the index of a port
2791  *      @dev: the net device for the port
2792  *
2793  *      Return the index of the given port.
2794  */
2795 unsigned int cxgb4_port_idx(const struct net_device *dev)
2796 {
2797         return netdev2pinfo(dev)->port_id;
2798 }
2799 EXPORT_SYMBOL(cxgb4_port_idx);
2800
2801 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2802                          struct tp_tcp_stats *v6)
2803 {
2804         struct adapter *adap = pci_get_drvdata(pdev);
2805
2806         spin_lock(&adap->stats_lock);
2807         t4_tp_get_tcp_stats(adap, v4, v6);
2808         spin_unlock(&adap->stats_lock);
2809 }
2810 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2811
2812 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2813                       const unsigned int *pgsz_order)
2814 {
2815         struct adapter *adap = netdev2adap(dev);
2816
2817         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2818         t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2819                      HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2820                      HPZ3(pgsz_order[3]));
2821 }
2822 EXPORT_SYMBOL(cxgb4_iscsi_init);
2823
2824 int cxgb4_flush_eq_cache(struct net_device *dev)
2825 {
2826         struct adapter *adap = netdev2adap(dev);
2827         int ret;
2828
2829         ret = t4_fwaddrspace_write(adap, adap->mbox,
2830                                    0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
2831         return ret;
2832 }
2833 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
2834
2835 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
2836 {
2837         u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
2838         __be64 indices;
2839         int ret;
2840
2841         ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);
2842         if (!ret) {
2843                 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
2844                 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
2845         }
2846         return ret;
2847 }
2848
2849 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
2850                         u16 size)
2851 {
2852         struct adapter *adap = netdev2adap(dev);
2853         u16 hw_pidx, hw_cidx;
2854         int ret;
2855
2856         ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
2857         if (ret)
2858                 goto out;
2859
2860         if (pidx != hw_pidx) {
2861                 u16 delta;
2862
2863                 if (pidx >= hw_pidx)
2864                         delta = pidx - hw_pidx;
2865                 else
2866                         delta = size - hw_pidx + pidx;
2867                 wmb();
2868                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
2869                              QID(qid) | PIDX(delta));
2870         }
2871 out:
2872         return ret;
2873 }
2874 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
2875
2876 static struct pci_driver cxgb4_driver;
2877
2878 static void check_neigh_update(struct neighbour *neigh)
2879 {
2880         const struct device *parent;
2881         const struct net_device *netdev = neigh->dev;
2882
2883         if (netdev->priv_flags & IFF_802_1Q_VLAN)
2884                 netdev = vlan_dev_real_dev(netdev);
2885         parent = netdev->dev.parent;
2886         if (parent && parent->driver == &cxgb4_driver.driver)
2887                 t4_l2t_update(dev_get_drvdata(parent), neigh);
2888 }
2889
2890 static int netevent_cb(struct notifier_block *nb, unsigned long event,
2891                        void *data)
2892 {
2893         switch (event) {
2894         case NETEVENT_NEIGH_UPDATE:
2895                 check_neigh_update(data);
2896                 break;
2897         case NETEVENT_REDIRECT:
2898         default:
2899                 break;
2900         }
2901         return 0;
2902 }
2903
2904 static bool netevent_registered;
2905 static struct notifier_block cxgb4_netevent_nb = {
2906         .notifier_call = netevent_cb
2907 };
2908
2909 static void drain_db_fifo(struct adapter *adap, int usecs)
2910 {
2911         u32 v;
2912
2913         do {
2914                 set_current_state(TASK_UNINTERRUPTIBLE);
2915                 schedule_timeout(usecs_to_jiffies(usecs));
2916                 v = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
2917                 if (G_LP_COUNT(v) == 0 && G_HP_COUNT(v) == 0)
2918                         break;
2919         } while (1);
2920 }
2921
2922 static void disable_txq_db(struct sge_txq *q)
2923 {
2924         spin_lock_irq(&q->db_lock);
2925         q->db_disabled = 1;
2926         spin_unlock_irq(&q->db_lock);
2927 }
2928
2929 static void enable_txq_db(struct sge_txq *q)
2930 {
2931         spin_lock_irq(&q->db_lock);
2932         q->db_disabled = 0;
2933         spin_unlock_irq(&q->db_lock);
2934 }
2935
2936 static void disable_dbs(struct adapter *adap)
2937 {
2938         int i;
2939
2940         for_each_ethrxq(&adap->sge, i)
2941                 disable_txq_db(&adap->sge.ethtxq[i].q);
2942         for_each_ofldrxq(&adap->sge, i)
2943                 disable_txq_db(&adap->sge.ofldtxq[i].q);
2944         for_each_port(adap, i)
2945                 disable_txq_db(&adap->sge.ctrlq[i].q);
2946 }
2947
2948 static void enable_dbs(struct adapter *adap)
2949 {
2950         int i;
2951
2952         for_each_ethrxq(&adap->sge, i)
2953                 enable_txq_db(&adap->sge.ethtxq[i].q);
2954         for_each_ofldrxq(&adap->sge, i)
2955                 enable_txq_db(&adap->sge.ofldtxq[i].q);
2956         for_each_port(adap, i)
2957                 enable_txq_db(&adap->sge.ctrlq[i].q);
2958 }
2959
2960 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
2961 {
2962         u16 hw_pidx, hw_cidx;
2963         int ret;
2964
2965         spin_lock_bh(&q->db_lock);
2966         ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
2967         if (ret)
2968                 goto out;
2969         if (q->db_pidx != hw_pidx) {
2970                 u16 delta;
2971
2972                 if (q->db_pidx >= hw_pidx)
2973                         delta = q->db_pidx - hw_pidx;
2974                 else
2975                         delta = q->size - hw_pidx + q->db_pidx;
2976                 wmb();
2977                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
2978                              QID(q->cntxt_id) | PIDX(delta));
2979         }
2980 out:
2981         q->db_disabled = 0;
2982         spin_unlock_bh(&q->db_lock);
2983         if (ret)
2984                 CH_WARN(adap, "DB drop recovery failed.\n");
2985 }
2986 static void recover_all_queues(struct adapter *adap)
2987 {
2988         int i;
2989
2990         for_each_ethrxq(&adap->sge, i)
2991                 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
2992         for_each_ofldrxq(&adap->sge, i)
2993                 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
2994         for_each_port(adap, i)
2995                 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
2996 }
2997
2998 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
2999 {
3000         mutex_lock(&uld_mutex);
3001         if (adap->uld_handle[CXGB4_ULD_RDMA])
3002                 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3003                                 cmd);
3004         mutex_unlock(&uld_mutex);
3005 }
3006
3007 static void process_db_full(struct work_struct *work)
3008 {
3009         struct adapter *adap;
3010
3011         adap = container_of(work, struct adapter, db_full_task);
3012
3013         notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3014         drain_db_fifo(adap, dbfifo_drain_delay);
3015         t4_set_reg_field(adap, SGE_INT_ENABLE3,
3016                          DBFIFO_HP_INT | DBFIFO_LP_INT,
3017                          DBFIFO_HP_INT | DBFIFO_LP_INT);
3018         notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3019 }
3020
3021 static void process_db_drop(struct work_struct *work)
3022 {
3023         struct adapter *adap;
3024
3025         adap = container_of(work, struct adapter, db_drop_task);
3026
3027         t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
3028         disable_dbs(adap);
3029         notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
3030         drain_db_fifo(adap, 1);
3031         recover_all_queues(adap);
3032         enable_dbs(adap);
3033 }
3034
3035 void t4_db_full(struct adapter *adap)
3036 {
3037         t4_set_reg_field(adap, SGE_INT_ENABLE3,
3038                          DBFIFO_HP_INT | DBFIFO_LP_INT, 0);
3039         queue_work(workq, &adap->db_full_task);
3040 }
3041
3042 void t4_db_dropped(struct adapter *adap)
3043 {
3044         queue_work(workq, &adap->db_drop_task);
3045 }
3046
3047 static void uld_attach(struct adapter *adap, unsigned int uld)
3048 {
3049         void *handle;
3050         struct cxgb4_lld_info lli;
3051         unsigned short i;
3052
3053         lli.pdev = adap->pdev;
3054         lli.l2t = adap->l2t;
3055         lli.tids = &adap->tids;
3056         lli.ports = adap->port;
3057         lli.vr = &adap->vres;
3058         lli.mtus = adap->params.mtus;
3059         if (uld == CXGB4_ULD_RDMA) {
3060                 lli.rxq_ids = adap->sge.rdma_rxq;
3061                 lli.nrxq = adap->sge.rdmaqs;
3062         } else if (uld == CXGB4_ULD_ISCSI) {
3063                 lli.rxq_ids = adap->sge.ofld_rxq;
3064                 lli.nrxq = adap->sge.ofldqsets;
3065         }
3066         lli.ntxq = adap->sge.ofldqsets;
3067         lli.nchan = adap->params.nports;
3068         lli.nports = adap->params.nports;
3069         lli.wr_cred = adap->params.ofldq_wr_cred;
3070         lli.adapter_type = adap->params.rev;
3071         lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
3072         lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
3073                         t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
3074                         (adap->fn * 4));
3075         lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
3076                         t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
3077                         (adap->fn * 4));
3078         lli.filt_mode = adap->filter_mode;
3079         /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3080         for (i = 0; i < NCHAN; i++)
3081                 lli.tx_modq[i] = i;
3082         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
3083         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
3084         lli.fw_vers = adap->params.fw_vers;
3085         lli.dbfifo_int_thresh = dbfifo_int_thresh;
3086         lli.sge_pktshift = adap->sge.pktshift;
3087         lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
3088
3089         handle = ulds[uld].add(&lli);
3090         if (IS_ERR(handle)) {
3091                 dev_warn(adap->pdev_dev,
3092                          "could not attach to the %s driver, error %ld\n",
3093                          uld_str[uld], PTR_ERR(handle));
3094                 return;
3095         }
3096
3097         adap->uld_handle[uld] = handle;
3098
3099         if (!netevent_registered) {
3100                 register_netevent_notifier(&cxgb4_netevent_nb);
3101                 netevent_registered = true;
3102         }
3103
3104         if (adap->flags & FULL_INIT_DONE)
3105                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
3106 }
3107
3108 static void attach_ulds(struct adapter *adap)
3109 {
3110         unsigned int i;
3111
3112         mutex_lock(&uld_mutex);
3113         list_add_tail(&adap->list_node, &adapter_list);
3114         for (i = 0; i < CXGB4_ULD_MAX; i++)
3115                 if (ulds[i].add)
3116                         uld_attach(adap, i);
3117         mutex_unlock(&uld_mutex);
3118 }
3119
3120 static void detach_ulds(struct adapter *adap)
3121 {
3122         unsigned int i;
3123
3124         mutex_lock(&uld_mutex);
3125         list_del(&adap->list_node);
3126         for (i = 0; i < CXGB4_ULD_MAX; i++)
3127                 if (adap->uld_handle[i]) {
3128                         ulds[i].state_change(adap->uld_handle[i],
3129                                              CXGB4_STATE_DETACH);
3130                         adap->uld_handle[i] = NULL;
3131                 }
3132         if (netevent_registered && list_empty(&adapter_list)) {
3133                 unregister_netevent_notifier(&cxgb4_netevent_nb);
3134                 netevent_registered = false;
3135         }
3136         mutex_unlock(&uld_mutex);
3137 }
3138
3139 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
3140 {
3141         unsigned int i;
3142
3143         mutex_lock(&uld_mutex);
3144         for (i = 0; i < CXGB4_ULD_MAX; i++)
3145                 if (adap->uld_handle[i])
3146                         ulds[i].state_change(adap->uld_handle[i], new_state);
3147         mutex_unlock(&uld_mutex);
3148 }
3149
3150 /**
3151  *      cxgb4_register_uld - register an upper-layer driver
3152  *      @type: the ULD type
3153  *      @p: the ULD methods
3154  *
3155  *      Registers an upper-layer driver with this driver and notifies the ULD
3156  *      about any presently available devices that support its type.  Returns
3157  *      %-EBUSY if a ULD of the same type is already registered.
3158  */
3159 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
3160 {
3161         int ret = 0;
3162         struct adapter *adap;
3163
3164         if (type >= CXGB4_ULD_MAX)
3165                 return -EINVAL;
3166         mutex_lock(&uld_mutex);
3167         if (ulds[type].add) {
3168                 ret = -EBUSY;
3169                 goto out;
3170         }
3171         ulds[type] = *p;
3172         list_for_each_entry(adap, &adapter_list, list_node)
3173                 uld_attach(adap, type);
3174 out:    mutex_unlock(&uld_mutex);
3175         return ret;
3176 }
3177 EXPORT_SYMBOL(cxgb4_register_uld);
3178
3179 /**
3180  *      cxgb4_unregister_uld - unregister an upper-layer driver
3181  *      @type: the ULD type
3182  *
3183  *      Unregisters an existing upper-layer driver.
3184  */
3185 int cxgb4_unregister_uld(enum cxgb4_uld type)
3186 {
3187         struct adapter *adap;
3188
3189         if (type >= CXGB4_ULD_MAX)
3190                 return -EINVAL;
3191         mutex_lock(&uld_mutex);
3192         list_for_each_entry(adap, &adapter_list, list_node)
3193                 adap->uld_handle[type] = NULL;
3194         ulds[type].add = NULL;
3195         mutex_unlock(&uld_mutex);
3196         return 0;
3197 }
3198 EXPORT_SYMBOL(cxgb4_unregister_uld);
3199
3200 /**
3201  *      cxgb_up - enable the adapter
3202  *      @adap: adapter being enabled
3203  *
3204  *      Called when the first port is enabled, this function performs the
3205  *      actions necessary to make an adapter operational, such as completing
3206  *      the initialization of HW modules, and enabling interrupts.
3207  *
3208  *      Must be called with the rtnl lock held.
3209  */
3210 static int cxgb_up(struct adapter *adap)
3211 {
3212         int err;
3213
3214         err = setup_sge_queues(adap);
3215         if (err)
3216                 goto out;
3217         err = setup_rss(adap);
3218         if (err)
3219                 goto freeq;
3220
3221         if (adap->flags & USING_MSIX) {
3222                 name_msix_vecs(adap);
3223                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
3224                                   adap->msix_info[0].desc, adap);
3225                 if (err)
3226                         goto irq_err;
3227
3228                 err = request_msix_queue_irqs(adap);
3229                 if (err) {
3230                         free_irq(adap->msix_info[0].vec, adap);
3231                         goto irq_err;
3232                 }
3233         } else {
3234                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
3235                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
3236                                   adap->port[0]->name, adap);
3237                 if (err)
3238                         goto irq_err;
3239         }
3240         enable_rx(adap);
3241         t4_sge_start(adap);
3242         t4_intr_enable(adap);
3243         adap->flags |= FULL_INIT_DONE;
3244         notify_ulds(adap, CXGB4_STATE_UP);
3245  out:
3246         return err;
3247  irq_err:
3248         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
3249  freeq:
3250         t4_free_sge_resources(adap);
3251         goto out;
3252 }
3253
3254 static void cxgb_down(struct adapter *adapter)
3255 {
3256         t4_intr_disable(adapter);
3257         cancel_work_sync(&adapter->tid_release_task);
3258         cancel_work_sync(&adapter->db_full_task);
3259         cancel_work_sync(&adapter->db_drop_task);
3260         adapter->tid_release_task_busy = false;
3261         adapter->tid_release_head = NULL;
3262
3263         if (adapter->flags & USING_MSIX) {
3264                 free_msix_queue_irqs(adapter);
3265                 free_irq(adapter->msix_info[0].vec, adapter);
3266         } else
3267                 free_irq(adapter->pdev->irq, adapter);
3268         quiesce_rx(adapter);
3269         t4_sge_stop(adapter);
3270         t4_free_sge_resources(adapter);
3271         adapter->flags &= ~FULL_INIT_DONE;
3272 }
3273
3274 /*
3275  * net_device operations
3276  */
3277 static int cxgb_open(struct net_device *dev)
3278 {
3279         int err;
3280         struct port_info *pi = netdev_priv(dev);
3281         struct adapter *adapter = pi->adapter;
3282
3283         netif_carrier_off(dev);
3284
3285         if (!(adapter->flags & FULL_INIT_DONE)) {
3286                 err = cxgb_up(adapter);
3287                 if (err < 0)
3288                         return err;
3289         }
3290
3291         err = link_start(dev);
3292         if (!err)
3293                 netif_tx_start_all_queues(dev);
3294         return err;
3295 }
3296
3297 static int cxgb_close(struct net_device *dev)
3298 {
3299         struct port_info *pi = netdev_priv(dev);
3300         struct adapter *adapter = pi->adapter;
3301
3302         netif_tx_stop_all_queues(dev);
3303         netif_carrier_off(dev);
3304         return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
3305 }
3306
3307 /* Return an error number if the indicated filter isn't writable ...
3308  */
3309 static int writable_filter(struct filter_entry *f)
3310 {
3311         if (f->locked)
3312                 return -EPERM;
3313         if (f->pending)
3314                 return -EBUSY;
3315
3316         return 0;
3317 }
3318
3319 /* Delete the filter at the specified index (if valid).  The checks for all
3320  * the common problems with doing this like the filter being locked, currently
3321  * pending in another operation, etc.
3322  */
3323 static int delete_filter(struct adapter *adapter, unsigned int fidx)
3324 {
3325         struct filter_entry *f;
3326         int ret;
3327
3328         if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
3329                 return -EINVAL;
3330
3331         f = &adapter->tids.ftid_tab[fidx];
3332         ret = writable_filter(f);
3333         if (ret)
3334                 return ret;
3335         if (f->valid)
3336                 return del_filter_wr(adapter, fidx);
3337
3338         return 0;
3339 }
3340
3341 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
3342                 __be32 sip, __be16 sport, __be16 vlan,
3343                 unsigned int queue, unsigned char port, unsigned char mask)
3344 {
3345         int ret;
3346         struct filter_entry *f;
3347         struct adapter *adap;
3348         int i;
3349         u8 *val;
3350
3351         adap = netdev2adap(dev);
3352
3353         /* Adjust stid to correct filter index */
3354         stid -= adap->tids.nstids;
3355         stid += adap->tids.nftids;
3356
3357         /* Check to make sure the filter requested is writable ...
3358          */
3359         f = &adap->tids.ftid_tab[stid];
3360         ret = writable_filter(f);
3361         if (ret)
3362                 return ret;
3363
3364         /* Clear out any old resources being used by the filter before
3365          * we start constructing the new filter.
3366          */
3367         if (f->valid)
3368                 clear_filter(adap, f);
3369
3370         /* Clear out filter specifications */
3371         memset(&f->fs, 0, sizeof(struct ch_filter_specification));
3372         f->fs.val.lport = cpu_to_be16(sport);
3373         f->fs.mask.lport  = ~0;
3374         val = (u8 *)&sip;
3375         if ((val[0] | val[1] | val[2] | val[3]) != 0) {
3376                 for (i = 0; i < 4; i++) {
3377                         f->fs.val.lip[i] = val[i];
3378                         f->fs.mask.lip[i] = ~0;
3379                 }
3380                 if (adap->filter_mode & F_PORT) {
3381                         f->fs.val.iport = port;
3382                         f->fs.mask.iport = mask;
3383                 }
3384         }
3385
3386         f->fs.dirsteer = 1;
3387         f->fs.iq = queue;
3388         /* Mark filter as locked */
3389         f->locked = 1;
3390         f->fs.rpttid = 1;
3391
3392         ret = set_filter_wr(adap, stid);
3393         if (ret) {
3394                 clear_filter(adap, f);
3395                 return ret;
3396         }
3397
3398         return 0;
3399 }
3400 EXPORT_SYMBOL(cxgb4_create_server_filter);
3401
3402 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
3403                 unsigned int queue, bool ipv6)
3404 {
3405         int ret;
3406         struct filter_entry *f;
3407         struct adapter *adap;
3408
3409         adap = netdev2adap(dev);
3410
3411         /* Adjust stid to correct filter index */
3412         stid -= adap->tids.nstids;
3413         stid += adap->tids.nftids;
3414
3415         f = &adap->tids.ftid_tab[stid];
3416         /* Unlock the filter */
3417         f->locked = 0;
3418
3419         ret = delete_filter(adap, stid);
3420         if (ret)
3421                 return ret;
3422
3423         return 0;
3424 }
3425 EXPORT_SYMBOL(cxgb4_remove_server_filter);
3426
3427 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
3428                                                 struct rtnl_link_stats64 *ns)
3429 {
3430         struct port_stats stats;
3431         struct port_info *p = netdev_priv(dev);
3432         struct adapter *adapter = p->adapter;
3433
3434         spin_lock(&adapter->stats_lock);
3435         t4_get_port_stats(adapter, p->tx_chan, &stats);
3436         spin_unlock(&adapter->stats_lock);
3437
3438         ns->tx_bytes   = stats.tx_octets;
3439         ns->tx_packets = stats.tx_frames;
3440         ns->rx_bytes   = stats.rx_octets;
3441         ns->rx_packets = stats.rx_frames;
3442         ns->multicast  = stats.rx_mcast_frames;
3443
3444         /* detailed rx_errors */
3445         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
3446                                stats.rx_runt;
3447         ns->rx_over_errors   = 0;
3448         ns->rx_crc_errors    = stats.rx_fcs_err;
3449         ns->rx_frame_errors  = stats.rx_symbol_err;
3450         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
3451                                stats.rx_ovflow2 + stats.rx_ovflow3 +
3452                                stats.rx_trunc0 + stats.rx_trunc1 +
3453                                stats.rx_trunc2 + stats.rx_trunc3;
3454         ns->rx_missed_errors = 0;
3455
3456         /* detailed tx_errors */
3457         ns->tx_aborted_errors   = 0;
3458         ns->tx_carrier_errors   = 0;
3459         ns->tx_fifo_errors      = 0;
3460         ns->tx_heartbeat_errors = 0;
3461         ns->tx_window_errors    = 0;
3462
3463         ns->tx_errors = stats.tx_error_frames;
3464         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
3465                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
3466         return ns;
3467 }
3468
3469 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
3470 {
3471         unsigned int mbox;
3472         int ret = 0, prtad, devad;
3473         struct port_info *pi = netdev_priv(dev);
3474         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
3475
3476         switch (cmd) {
3477         case SIOCGMIIPHY:
3478                 if (pi->mdio_addr < 0)
3479                         return -EOPNOTSUPP;
3480                 data->phy_id = pi->mdio_addr;
3481                 break;
3482         case SIOCGMIIREG:
3483         case SIOCSMIIREG:
3484                 if (mdio_phy_id_is_c45(data->phy_id)) {
3485                         prtad = mdio_phy_id_prtad(data->phy_id);
3486                         devad = mdio_phy_id_devad(data->phy_id);
3487                 } else if (data->phy_id < 32) {
3488                         prtad = data->phy_id;
3489                         devad = 0;
3490                         data->reg_num &= 0x1f;
3491                 } else
3492                         return -EINVAL;
3493
3494                 mbox = pi->adapter->fn;
3495                 if (cmd == SIOCGMIIREG)
3496                         ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
3497                                          data->reg_num, &data->val_out);
3498                 else
3499                         ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
3500                                          data->reg_num, data->val_in);
3501                 break;
3502         default:
3503                 return -EOPNOTSUPP;
3504         }
3505         return ret;
3506 }
3507
3508 static void cxgb_set_rxmode(struct net_device *dev)
3509 {
3510         /* unfortunately we can't return errors to the stack */
3511         set_rxmode(dev, -1, false);
3512 }
3513
3514 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
3515 {
3516         int ret;
3517         struct port_info *pi = netdev_priv(dev);
3518
3519         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
3520                 return -EINVAL;
3521         ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
3522                             -1, -1, -1, true);
3523         if (!ret)
3524                 dev->mtu = new_mtu;
3525         return ret;
3526 }
3527
3528 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
3529 {
3530         int ret;
3531         struct sockaddr *addr = p;
3532         struct port_info *pi = netdev_priv(dev);
3533
3534         if (!is_valid_ether_addr(addr->sa_data))
3535                 return -EADDRNOTAVAIL;
3536
3537         ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
3538                             pi->xact_addr_filt, addr->sa_data, true, true);
3539         if (ret < 0)
3540                 return ret;
3541
3542         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3543         pi->xact_addr_filt = ret;
3544         return 0;
3545 }
3546
3547 #ifdef CONFIG_NET_POLL_CONTROLLER
3548 static void cxgb_netpoll(struct net_device *dev)
3549 {
3550         struct port_info *pi = netdev_priv(dev);
3551         struct adapter *adap = pi->adapter;
3552
3553         if (adap->flags & USING_MSIX) {
3554                 int i;
3555                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
3556
3557                 for (i = pi->nqsets; i; i--, rx++)
3558                         t4_sge_intr_msix(0, &rx->rspq);
3559         } else
3560                 t4_intr_handler(adap)(0, adap);
3561 }
3562 #endif
3563
3564 static const struct net_device_ops cxgb4_netdev_ops = {
3565         .ndo_open             = cxgb_open,
3566         .ndo_stop             = cxgb_close,
3567         .ndo_start_xmit       = t4_eth_xmit,
3568         .ndo_get_stats64      = cxgb_get_stats,
3569         .ndo_set_rx_mode      = cxgb_set_rxmode,
3570         .ndo_set_mac_address  = cxgb_set_mac_addr,
3571         .ndo_set_features     = cxgb_set_features,
3572         .ndo_validate_addr    = eth_validate_addr,
3573         .ndo_do_ioctl         = cxgb_ioctl,
3574         .ndo_change_mtu       = cxgb_change_mtu,
3575 #ifdef CONFIG_NET_POLL_CONTROLLER
3576         .ndo_poll_controller  = cxgb_netpoll,
3577 #endif
3578 };
3579
3580 void t4_fatal_err(struct adapter *adap)
3581 {
3582         t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
3583         t4_intr_disable(adap);
3584         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
3585 }
3586
3587 static void setup_memwin(struct adapter *adap)
3588 {
3589         u32 bar0;
3590
3591         bar0 = pci_resource_start(adap->pdev, 0);  /* truncation intentional */
3592         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
3593                      (bar0 + MEMWIN0_BASE) | BIR(0) |
3594                      WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
3595         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
3596                      (bar0 + MEMWIN1_BASE) | BIR(0) |
3597                      WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
3598         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
3599                      (bar0 + MEMWIN2_BASE) | BIR(0) |
3600                      WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
3601 }
3602
3603 static void setup_memwin_rdma(struct adapter *adap)
3604 {
3605         if (adap->vres.ocq.size) {
3606                 unsigned int start, sz_kb;
3607
3608                 start = pci_resource_start(adap->pdev, 2) +
3609                         OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
3610                 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
3611                 t4_write_reg(adap,
3612                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
3613                              start | BIR(1) | WINDOW(ilog2(sz_kb)));
3614                 t4_write_reg(adap,
3615                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
3616                              adap->vres.ocq.start);
3617                 t4_read_reg(adap,
3618                             PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
3619         }
3620 }
3621
3622 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
3623 {
3624         u32 v;
3625         int ret;
3626
3627         /* get device capabilities */
3628         memset(c, 0, sizeof(*c));
3629         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3630                                FW_CMD_REQUEST | FW_CMD_READ);
3631         c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
3632         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
3633         if (ret < 0)
3634                 return ret;
3635
3636         /* select capabilities we'll be using */
3637         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
3638                 if (!vf_acls)
3639                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
3640                 else
3641                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
3642         } else if (vf_acls) {
3643                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
3644                 return ret;
3645         }
3646         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3647                                FW_CMD_REQUEST | FW_CMD_WRITE);
3648         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
3649         if (ret < 0)
3650                 return ret;
3651
3652         ret = t4_config_glbl_rss(adap, adap->fn,
3653                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
3654                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
3655                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
3656         if (ret < 0)
3657                 return ret;
3658
3659         ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
3660                           0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
3661         if (ret < 0)
3662                 return ret;
3663
3664         t4_sge_init(adap);
3665
3666         /* tweak some settings */
3667         t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
3668         t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
3669         t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
3670         v = t4_read_reg(adap, TP_PIO_DATA);
3671         t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
3672
3673         /* first 4 Tx modulation queues point to consecutive Tx channels */
3674         adap->params.tp.tx_modq_map = 0xE4;
3675         t4_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP,
3676                      V_TX_MOD_QUEUE_REQ_MAP(adap->params.tp.tx_modq_map));
3677
3678         /* associate each Tx modulation queue with consecutive Tx channels */
3679         v = 0x84218421;
3680         t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
3681                           &v, 1, A_TP_TX_SCHED_HDR);
3682         t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
3683                           &v, 1, A_TP_TX_SCHED_FIFO);
3684         t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
3685                           &v, 1, A_TP_TX_SCHED_PCMD);
3686
3687 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
3688         if (is_offload(adap)) {
3689                 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0,
3690                              V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
3691                              V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
3692                              V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
3693                              V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
3694                 t4_write_reg(adap, A_TP_TX_MOD_CHANNEL_WEIGHT,
3695                              V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
3696                              V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
3697                              V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
3698                              V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
3699         }
3700
3701         /* get basic stuff going */
3702         return t4_early_init(adap, adap->fn);
3703 }
3704
3705 /*
3706  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
3707  */
3708 #define MAX_ATIDS 8192U
3709
3710 /*
3711  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
3712  *
3713  * If the firmware we're dealing with has Configuration File support, then
3714  * we use that to perform all configuration
3715  */
3716
3717 /*
3718  * Tweak configuration based on module parameters, etc.  Most of these have
3719  * defaults assigned to them by Firmware Configuration Files (if we're using
3720  * them) but need to be explicitly set if we're using hard-coded
3721  * initialization.  But even in the case of using Firmware Configuration
3722  * Files, we'd like to expose the ability to change these via module
3723  * parameters so these are essentially common tweaks/settings for
3724  * Configuration Files and hard-coded initialization ...
3725  */
3726 static int adap_init0_tweaks(struct adapter *adapter)
3727 {
3728         /*
3729          * Fix up various Host-Dependent Parameters like Page Size, Cache
3730          * Line Size, etc.  The firmware default is for a 4KB Page Size and
3731          * 64B Cache Line Size ...
3732          */
3733         t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
3734
3735         /*
3736          * Process module parameters which affect early initialization.
3737          */
3738         if (rx_dma_offset != 2 && rx_dma_offset != 0) {
3739                 dev_err(&adapter->pdev->dev,
3740                         "Ignoring illegal rx_dma_offset=%d, using 2\n",
3741                         rx_dma_offset);
3742                 rx_dma_offset = 2;
3743         }
3744         t4_set_reg_field(adapter, SGE_CONTROL,
3745                          PKTSHIFT_MASK,
3746                          PKTSHIFT(rx_dma_offset));
3747
3748         /*
3749          * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
3750          * adds the pseudo header itself.
3751          */
3752         t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,
3753                                CSUM_HAS_PSEUDO_HDR, 0);
3754
3755         return 0;
3756 }
3757
3758 /*
3759  * Attempt to initialize the adapter via a Firmware Configuration File.
3760  */
3761 static int adap_init0_config(struct adapter *adapter, int reset)
3762 {
3763         struct fw_caps_config_cmd caps_cmd;
3764         const struct firmware *cf;
3765         unsigned long mtype = 0, maddr = 0;
3766         u32 finiver, finicsum, cfcsum;
3767         int ret, using_flash;
3768         char *fw_config_file, fw_config_file_path[256];
3769
3770         /*
3771          * Reset device if necessary.
3772          */
3773         if (reset) {
3774                 ret = t4_fw_reset(adapter, adapter->mbox,
3775                                   PIORSTMODE | PIORST);
3776                 if (ret < 0)
3777                         goto bye;
3778         }
3779
3780         /*
3781          * If we have a T4 configuration file under /lib/firmware/cxgb4/,
3782          * then use that.  Otherwise, use the configuration file stored
3783          * in the adapter flash ...
3784          */
3785         switch (CHELSIO_CHIP_VERSION(adapter->chip)) {
3786         case CHELSIO_T4:
3787                 fw_config_file = FW_CFNAME;
3788                 break;
3789         case CHELSIO_T5:
3790                 fw_config_file = FW5_CFNAME;
3791                 break;
3792         default:
3793                 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
3794                        adapter->pdev->device);
3795                 ret = -EINVAL;
3796                 goto bye;
3797         }
3798
3799         ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
3800         if (ret < 0) {
3801                 using_flash = 1;
3802                 mtype = FW_MEMTYPE_CF_FLASH;
3803                 maddr = t4_flash_cfg_addr(adapter);
3804         } else {
3805                 u32 params[7], val[7];
3806
3807                 using_flash = 0;
3808                 if (cf->size >= FLASH_CFG_MAX_SIZE)
3809                         ret = -ENOMEM;
3810                 else {
3811                         params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3812                              FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
3813                         ret = t4_query_params(adapter, adapter->mbox,
3814                                               adapter->fn, 0, 1, params, val);
3815                         if (ret == 0) {
3816                                 /*
3817                                  * For t4_memory_write() below addresses and
3818                                  * sizes have to be in terms of multiples of 4
3819                                  * bytes.  So, if the Configuration File isn't
3820                                  * a multiple of 4 bytes in length we'll have
3821                                  * to write that out separately since we can't
3822                                  * guarantee that the bytes following the
3823                                  * residual byte in the buffer returned by
3824                                  * request_firmware() are zeroed out ...
3825                                  */
3826                                 size_t resid = cf->size & 0x3;
3827                                 size_t size = cf->size & ~0x3;
3828                                 __be32 *data = (__be32 *)cf->data;
3829
3830                                 mtype = FW_PARAMS_PARAM_Y_GET(val[0]);
3831                                 maddr = FW_PARAMS_PARAM_Z_GET(val[0]) << 16;
3832
3833                                 ret = t4_memory_write(adapter, mtype, maddr,
3834                                                       size, data);
3835                                 if (ret == 0 && resid != 0) {
3836                                         union {
3837                                                 __be32 word;
3838                                                 char buf[4];
3839                                         } last;
3840                                         int i;
3841
3842                                         last.word = data[size >> 2];
3843                                         for (i = resid; i < 4; i++)
3844                                                 last.buf[i] = 0;
3845                                         ret = t4_memory_write(adapter, mtype,
3846                                                               maddr + size,
3847                                                               4, &last.word);
3848                                 }
3849                         }
3850                 }
3851
3852                 release_firmware(cf);
3853                 if (ret)
3854                         goto bye;
3855         }
3856
3857         /*
3858          * Issue a Capability Configuration command to the firmware to get it
3859          * to parse the Configuration File.  We don't use t4_fw_config_file()
3860          * because we want the ability to modify various features after we've
3861          * processed the configuration file ...
3862          */
3863         memset(&caps_cmd, 0, sizeof(caps_cmd));
3864         caps_cmd.op_to_write =
3865                 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3866                       FW_CMD_REQUEST |
3867                       FW_CMD_READ);
3868         caps_cmd.cfvalid_to_len16 =
3869                 htonl(FW_CAPS_CONFIG_CMD_CFVALID |
3870                       FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
3871                       FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
3872                       FW_LEN16(caps_cmd));
3873         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3874                          &caps_cmd);
3875         if (ret < 0)
3876                 goto bye;
3877
3878         finiver = ntohl(caps_cmd.finiver);
3879         finicsum = ntohl(caps_cmd.finicsum);
3880         cfcsum = ntohl(caps_cmd.cfcsum);
3881         if (finicsum != cfcsum)
3882                 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
3883                          "mismatch: [fini] csum=%#x, computed csum=%#x\n",
3884                          finicsum, cfcsum);
3885
3886         /*
3887          * And now tell the firmware to use the configuration we just loaded.
3888          */
3889         caps_cmd.op_to_write =
3890                 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3891                       FW_CMD_REQUEST |
3892                       FW_CMD_WRITE);
3893         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
3894         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3895                          NULL);
3896         if (ret < 0)
3897                 goto bye;
3898
3899         /*
3900          * Tweak configuration based on system architecture, module
3901          * parameters, etc.
3902          */
3903         ret = adap_init0_tweaks(adapter);
3904         if (ret < 0)
3905                 goto bye;
3906
3907         /*
3908          * And finally tell the firmware to initialize itself using the
3909          * parameters from the Configuration File.
3910          */
3911         ret = t4_fw_initialize(adapter, adapter->mbox);
3912         if (ret < 0)
3913                 goto bye;
3914
3915         sprintf(fw_config_file_path, "/lib/firmware/%s", fw_config_file);
3916         /*
3917          * Return successfully and note that we're operating with parameters
3918          * not supplied by the driver, rather than from hard-wired
3919          * initialization constants burried in the driver.
3920          */
3921         adapter->flags |= USING_SOFT_PARAMS;
3922         dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
3923                  "Configuration File %s, version %#x, computed checksum %#x\n",
3924                  (using_flash
3925                   ? "in device FLASH"
3926                   : fw_config_file_path),
3927                  finiver, cfcsum);
3928         return 0;
3929
3930         /*
3931          * Something bad happened.  Return the error ...  (If the "error"
3932          * is that there's no Configuration File on the adapter we don't
3933          * want to issue a warning since this is fairly common.)
3934          */
3935 bye:
3936         if (ret != -ENOENT)
3937                 dev_warn(adapter->pdev_dev, "Configuration file error %d\n",
3938                          -ret);
3939         return ret;
3940 }
3941
3942 /*
3943  * Attempt to initialize the adapter via hard-coded, driver supplied
3944  * parameters ...
3945  */
3946 static int adap_init0_no_config(struct adapter *adapter, int reset)
3947 {
3948         struct sge *s = &adapter->sge;
3949         struct fw_caps_config_cmd caps_cmd;
3950         u32 v;
3951         int i, ret;
3952
3953         /*
3954          * Reset device if necessary
3955          */
3956         if (reset) {
3957                 ret = t4_fw_reset(adapter, adapter->mbox,
3958                                   PIORSTMODE | PIORST);
3959                 if (ret < 0)
3960                         goto bye;
3961         }
3962
3963         /*
3964          * Get device capabilities and select which we'll be using.
3965          */
3966         memset(&caps_cmd, 0, sizeof(caps_cmd));
3967         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3968                                      FW_CMD_REQUEST | FW_CMD_READ);
3969         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
3970         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3971                          &caps_cmd);
3972         if (ret < 0)
3973                 goto bye;
3974
3975         if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
3976                 if (!vf_acls)
3977                         caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
3978                 else
3979                         caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
3980         } else if (vf_acls) {
3981                 dev_err(adapter->pdev_dev, "virtualization ACLs not supported");
3982                 goto bye;
3983         }
3984         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
3985                               FW_CMD_REQUEST | FW_CMD_WRITE);
3986         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
3987                          NULL);
3988         if (ret < 0)
3989                 goto bye;
3990
3991         /*
3992          * Tweak configuration based on system architecture, module
3993          * parameters, etc.
3994          */
3995         ret = adap_init0_tweaks(adapter);
3996         if (ret < 0)
3997                 goto bye;
3998
3999         /*
4000          * Select RSS Global Mode we want to use.  We use "Basic Virtual"
4001          * mode which maps each Virtual Interface to its own section of
4002          * the RSS Table and we turn on all map and hash enables ...
4003          */
4004         adapter->flags |= RSS_TNLALLLOOKUP;
4005         ret = t4_config_glbl_rss(adapter, adapter->mbox,
4006                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4007                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4008                                  FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ |
4009                                  ((adapter->flags & RSS_TNLALLLOOKUP) ?
4010                                         FW_RSS_GLB_CONFIG_CMD_TNLALLLKP : 0));
4011         if (ret < 0)
4012                 goto bye;
4013
4014         /*
4015          * Set up our own fundamental resource provisioning ...
4016          */
4017         ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,
4018                           PFRES_NEQ, PFRES_NETHCTRL,
4019                           PFRES_NIQFLINT, PFRES_NIQ,
4020                           PFRES_TC, PFRES_NVI,
4021                           FW_PFVF_CMD_CMASK_MASK,
4022                           pfvfres_pmask(adapter, adapter->fn, 0),
4023                           PFRES_NEXACTF,
4024                           PFRES_R_CAPS, PFRES_WX_CAPS);
4025         if (ret < 0)
4026                 goto bye;
4027
4028         /*
4029          * Perform low level SGE initialization.  We need to do this before we
4030          * send the firmware the INITIALIZE command because that will cause
4031          * any other PF Drivers which are waiting for the Master
4032          * Initialization to proceed forward.
4033          */
4034         for (i = 0; i < SGE_NTIMERS - 1; i++)
4035                 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);
4036         s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
4037         s->counter_val[0] = 1;
4038         for (i = 1; i < SGE_NCOUNTERS; i++)
4039                 s->counter_val[i] = min(intr_cnt[i - 1],
4040                                         THRESHOLD_0_GET(THRESHOLD_0_MASK));
4041         t4_sge_init(adapter);
4042
4043 #ifdef CONFIG_PCI_IOV
4044         /*
4045          * Provision resource limits for Virtual Functions.  We currently
4046          * grant them all the same static resource limits except for the Port
4047          * Access Rights Mask which we're assigning based on the PF.  All of
4048          * the static provisioning stuff for both the PF and VF really needs
4049          * to be managed in a persistent manner for each device which the
4050          * firmware controls.
4051          */
4052         {
4053                 int pf, vf;
4054                 int max_no_pf = is_t4(adapter->chip) ? NUM_OF_PF_WITH_SRIOV_T4 :
4055                                 NUM_OF_PF_WITH_SRIOV_T5;
4056
4057                 for (pf = 0; pf < max_no_pf; pf++) {
4058                         if (num_vf[pf] <= 0)
4059                                 continue;
4060
4061                         /* VF numbering starts at 1! */
4062                         for (vf = 1; vf <= num_vf[pf]; vf++) {
4063                                 ret = t4_cfg_pfvf(adapter, adapter->mbox,
4064                                                   pf, vf,
4065                                                   VFRES_NEQ, VFRES_NETHCTRL,
4066                                                   VFRES_NIQFLINT, VFRES_NIQ,
4067                                                   VFRES_TC, VFRES_NVI,
4068                                                   FW_PFVF_CMD_CMASK_MASK,
4069                                                   pfvfres_pmask(
4070                                                   adapter, pf, vf),
4071                                                   VFRES_NEXACTF,
4072                                                   VFRES_R_CAPS, VFRES_WX_CAPS);
4073                                 if (ret < 0)
4074                                         dev_warn(adapter->pdev_dev,
4075                                                  "failed to "\
4076                                                  "provision pf/vf=%d/%d; "
4077                                                  "err=%d\n", pf, vf, ret);
4078                         }
4079                 }
4080         }
4081 #endif
4082
4083         /*
4084          * Set up the default filter mode.  Later we'll want to implement this
4085          * via a firmware command, etc. ...  This needs to be done before the
4086          * firmare initialization command ...  If the selected set of fields
4087          * isn't equal to the default value, we'll need to make sure that the
4088          * field selections will fit in the 36-bit budget.
4089          */
4090         if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {
4091                 int j, bits = 0;
4092
4093                 for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++)
4094                         switch (tp_vlan_pri_map & (1 << j)) {
4095                         case 0:
4096                                 /* compressed filter field not enabled */
4097                                 break;
4098                         case FCOE_MASK:
4099                                 bits +=  1;
4100                                 break;
4101                         case PORT_MASK:
4102                                 bits +=  3;
4103                                 break;
4104                         case VNIC_ID_MASK:
4105                                 bits += 17;
4106                                 break;
4107                         case VLAN_MASK:
4108                                 bits += 17;
4109                                 break;
4110                         case TOS_MASK:
4111                                 bits +=  8;
4112                                 break;
4113                         case PROTOCOL_MASK:
4114                                 bits +=  8;
4115                                 break;
4116                         case ETHERTYPE_MASK:
4117                                 bits += 16;
4118                                 break;
4119                         case MACMATCH_MASK:
4120                                 bits +=  9;
4121                                 break;
4122                         case MPSHITTYPE_MASK:
4123                                 bits +=  3;
4124                                 break;
4125                         case FRAGMENTATION_MASK:
4126                                 bits +=  1;
4127                                 break;
4128                         }
4129
4130                 if (bits > 36) {
4131                         dev_err(adapter->pdev_dev,
4132                                 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
4133                                 " using %#x\n", tp_vlan_pri_map, bits,
4134                                 TP_VLAN_PRI_MAP_DEFAULT);
4135                         tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
4136                 }
4137         }
4138         v = tp_vlan_pri_map;
4139         t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA,
4140                           &v, 1, TP_VLAN_PRI_MAP);
4141
4142         /*
4143          * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
4144          * to support any of the compressed filter fields above.  Newer
4145          * versions of the firmware do this automatically but it doesn't hurt
4146          * to set it here.  Meanwhile, we do _not_ need to set Lookup Every
4147          * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
4148          * since the firmware automatically turns this on and off when we have
4149          * a non-zero number of filters active (since it does have a
4150          * performance impact).
4151          */
4152         if (tp_vlan_pri_map)
4153                 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG,
4154                                  FIVETUPLELOOKUP_MASK,
4155                                  FIVETUPLELOOKUP_MASK);
4156
4157         /*
4158          * Tweak some settings.
4159          */
4160         t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) |
4161                      RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
4162                      PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
4163                      KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
4164
4165         /*
4166          * Get basic stuff going by issuing the Firmware Initialize command.
4167          * Note that this _must_ be after all PFVF commands ...
4168          */
4169         ret = t4_fw_initialize(adapter, adapter->mbox);
4170         if (ret < 0)
4171                 goto bye;
4172
4173         /*
4174          * Return successfully!
4175          */
4176         dev_info(adapter->pdev_dev, "Successfully configured using built-in "\
4177                  "driver parameters\n");
4178         return 0;
4179
4180         /*
4181          * Something bad happened.  Return the error ...
4182          */
4183 bye:
4184         return ret;
4185 }
4186
4187 /*
4188  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4189  */
4190 static int adap_init0(struct adapter *adap)
4191 {
4192         int ret;
4193         u32 v, port_vec;
4194         enum dev_state state;
4195         u32 params[7], val[7];
4196         struct fw_caps_config_cmd caps_cmd;
4197         int reset = 1, j;
4198
4199         /*
4200          * Contact FW, advertising Master capability (and potentially forcing
4201          * ourselves as the Master PF if our module parameter force_init is
4202          * set).
4203          */
4204         ret = t4_fw_hello(adap, adap->mbox, adap->fn,
4205                           force_init ? MASTER_MUST : MASTER_MAY,
4206                           &state);
4207         if (ret < 0) {
4208                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
4209                         ret);
4210                 return ret;
4211         }
4212         if (ret == adap->mbox)
4213                 adap->flags |= MASTER_PF;
4214         if (force_init && state == DEV_STATE_INIT)
4215                 state = DEV_STATE_UNINIT;
4216
4217         /*
4218          * If we're the Master PF Driver and the device is uninitialized,
4219          * then let's consider upgrading the firmware ...  (We always want
4220          * to check the firmware version number in order to A. get it for
4221          * later reporting and B. to warn if the currently loaded firmware
4222          * is excessively mismatched relative to the driver.)
4223          */
4224         ret = t4_check_fw_version(adap);
4225         if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
4226                 if (ret == -EINVAL || ret > 0) {
4227                         if (upgrade_fw(adap) >= 0) {
4228                                 /*
4229                                  * Note that the chip was reset as part of the
4230                                  * firmware upgrade so we don't reset it again
4231                                  * below and grab the new firmware version.
4232                                  */
4233                                 reset = 0;
4234                                 ret = t4_check_fw_version(adap);
4235                         }
4236                 }
4237                 if (ret < 0)
4238                         return ret;
4239         }
4240
4241         /*
4242          * Grab VPD parameters.  This should be done after we establish a
4243          * connection to the firmware since some of the VPD parameters
4244          * (notably the Core Clock frequency) are retrieved via requests to
4245          * the firmware.  On the other hand, we need these fairly early on
4246          * so we do this right after getting ahold of the firmware.
4247          */
4248         ret = get_vpd_params(adap, &adap->params.vpd);
4249         if (ret < 0)
4250                 goto bye;
4251
4252         /*
4253          * Find out what ports are available to us.  Note that we need to do
4254          * this before calling adap_init0_no_config() since it needs nports
4255          * and portvec ...
4256          */
4257         v =
4258             FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4259             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
4260         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
4261         if (ret < 0)
4262                 goto bye;
4263
4264         adap->params.nports = hweight32(port_vec);
4265         adap->params.portvec = port_vec;
4266
4267         /*
4268          * If the firmware is initialized already (and we're not forcing a
4269          * master initialization), note that we're living with existing
4270          * adapter parameters.  Otherwise, it's time to try initializing the
4271          * adapter ...
4272          */
4273         if (state == DEV_STATE_INIT) {
4274                 dev_info(adap->pdev_dev, "Coming up as %s: "\
4275                          "Adapter already initialized\n",
4276                          adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
4277                 adap->flags |= USING_SOFT_PARAMS;
4278         } else {
4279                 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
4280                          "Initializing adapter\n");
4281
4282                 /*
4283                  * If the firmware doesn't support Configuration
4284                  * Files warn user and exit,
4285                  */
4286                 if (ret < 0)
4287                         dev_warn(adap->pdev_dev, "Firmware doesn't support "
4288                                  "configuration file.\n");
4289                 if (force_old_init)
4290                         ret = adap_init0_no_config(adap, reset);
4291                 else {
4292                         /*
4293                          * Find out whether we're dealing with a version of
4294                          * the firmware which has configuration file support.
4295                          */
4296                         params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4297                                      FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
4298                         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
4299                                               params, val);
4300
4301                         /*
4302                          * If the firmware doesn't support Configuration
4303                          * Files, use the old Driver-based, hard-wired
4304                          * initialization.  Otherwise, try using the
4305                          * Configuration File support and fall back to the
4306                          * Driver-based initialization if there's no
4307                          * Configuration File found.
4308                          */
4309                         if (ret < 0)
4310                                 ret = adap_init0_no_config(adap, reset);
4311                         else {
4312                                 /*
4313                                  * The firmware provides us with a memory
4314                                  * buffer where we can load a Configuration
4315                                  * File from the host if we want to override
4316                                  * the Configuration File in flash.
4317                                  */
4318
4319                                 ret = adap_init0_config(adap, reset);
4320                                 if (ret == -ENOENT) {
4321                                         dev_info(adap->pdev_dev,
4322                                             "No Configuration File present "
4323                                             "on adapter.  Using hard-wired "
4324                                             "configuration parameters.\n");
4325                                         ret = adap_init0_no_config(adap, reset);
4326                                 }
4327                         }
4328                 }
4329                 if (ret < 0) {
4330                         dev_err(adap->pdev_dev,
4331                                 "could not initialize adapter, error %d\n",
4332                                 -ret);
4333                         goto bye;
4334                 }
4335         }
4336
4337         /*
4338          * If we're living with non-hard-coded parameters (either from a
4339          * Firmware Configuration File or values programmed by a different PF
4340          * Driver), give the SGE code a chance to pull in anything that it
4341          * needs ...  Note that this must be called after we retrieve our VPD
4342          * parameters in order to know how to convert core ticks to seconds.
4343          */
4344         if (adap->flags & USING_SOFT_PARAMS) {
4345                 ret = t4_sge_init(adap);
4346                 if (ret < 0)
4347                         goto bye;
4348         }
4349
4350         if (is_bypass_device(adap->pdev->device))
4351                 adap->params.bypass = 1;
4352
4353         /*
4354          * Grab some of our basic fundamental operating parameters.
4355          */
4356 #define FW_PARAM_DEV(param) \
4357         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
4358         FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
4359
4360 #define FW_PARAM_PFVF(param) \
4361         FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
4362         FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)|  \
4363         FW_PARAMS_PARAM_Y(0) | \
4364         FW_PARAMS_PARAM_Z(0)
4365
4366         params[0] = FW_PARAM_PFVF(EQ_START);
4367         params[1] = FW_PARAM_PFVF(L2T_START);
4368         params[2] = FW_PARAM_PFVF(L2T_END);
4369         params[3] = FW_PARAM_PFVF(FILTER_START);
4370         params[4] = FW_PARAM_PFVF(FILTER_END);
4371         params[5] = FW_PARAM_PFVF(IQFLINT_START);
4372         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
4373         if (ret < 0)
4374                 goto bye;
4375         adap->sge.egr_start = val[0];
4376         adap->l2t_start = val[1];
4377         adap->l2t_end = val[2];
4378         adap->tids.ftid_base = val[3];
4379         adap->tids.nftids = val[4] - val[3] + 1;
4380         adap->sge.ingr_start = val[5];
4381
4382         /* query params related to active filter region */
4383         params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
4384         params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
4385         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
4386         /* If Active filter size is set we enable establishing
4387          * offload connection through firmware work request
4388          */
4389         if ((val[0] != val[1]) && (ret >= 0)) {
4390                 adap->flags |= FW_OFLD_CONN;
4391                 adap->tids.aftid_base = val[0];
4392                 adap->tids.aftid_end = val[1];
4393         }
4394
4395         /*
4396          * Get device capabilities so we can determine what resources we need
4397          * to manage.
4398          */
4399         memset(&caps_cmd, 0, sizeof(caps_cmd));
4400         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4401                                      FW_CMD_REQUEST | FW_CMD_READ);
4402         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4403         ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
4404                          &caps_cmd);
4405         if (ret < 0)
4406                 goto bye;
4407
4408         if (caps_cmd.ofldcaps) {
4409                 /* query offload-related parameters */
4410                 params[0] = FW_PARAM_DEV(NTID);
4411                 params[1] = FW_PARAM_PFVF(SERVER_START);
4412                 params[2] = FW_PARAM_PFVF(SERVER_END);
4413                 params[3] = FW_PARAM_PFVF(TDDP_START);
4414                 params[4] = FW_PARAM_PFVF(TDDP_END);
4415                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
4416                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
4417                                       params, val);
4418                 if (ret < 0)
4419                         goto bye;
4420                 adap->tids.ntids = val[0];
4421                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
4422                 adap->tids.stid_base = val[1];
4423                 adap->tids.nstids = val[2] - val[1] + 1;
4424                 /*
4425                  * Setup server filter region. Divide the availble filter
4426                  * region into two parts. Regular filters get 1/3rd and server
4427                  * filters get 2/3rd part. This is only enabled if workarond
4428                  * path is enabled.
4429                  * 1. For regular filters.
4430                  * 2. Server filter: This are special filters which are used
4431                  * to redirect SYN packets to offload queue.
4432                  */
4433                 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
4434                         adap->tids.sftid_base = adap->tids.ftid_base +
4435                                         DIV_ROUND_UP(adap->tids.nftids, 3);
4436                         adap->tids.nsftids = adap->tids.nftids -
4437                                          DIV_ROUND_UP(adap->tids.nftids, 3);
4438                         adap->tids.nftids = adap->tids.sftid_base -
4439                                                 adap->tids.ftid_base;
4440                 }
4441                 adap->vres.ddp.start = val[3];
4442                 adap->vres.ddp.size = val[4] - val[3] + 1;
4443                 adap->params.ofldq_wr_cred = val[5];
4444
4445                 adap->params.offload = 1;
4446         }
4447         if (caps_cmd.rdmacaps) {
4448                 params[0] = FW_PARAM_PFVF(STAG_START);
4449                 params[1] = FW_PARAM_PFVF(STAG_END);
4450                 params[2] = FW_PARAM_PFVF(RQ_START);
4451                 params[3] = FW_PARAM_PFVF(RQ_END);
4452                 params[4] = FW_PARAM_PFVF(PBL_START);
4453                 params[5] = FW_PARAM_PFVF(PBL_END);
4454                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
4455                                       params, val);
4456                 if (ret < 0)
4457                         goto bye;
4458                 adap->vres.stag.start = val[0];
4459                 adap->vres.stag.size = val[1] - val[0] + 1;
4460                 adap->vres.rq.start = val[2];
4461                 adap->vres.rq.size = val[3] - val[2] + 1;
4462                 adap->vres.pbl.start = val[4];
4463                 adap->vres.pbl.size = val[5] - val[4] + 1;
4464
4465                 params[0] = FW_PARAM_PFVF(SQRQ_START);
4466                 params[1] = FW_PARAM_PFVF(SQRQ_END);
4467                 params[2] = FW_PARAM_PFVF(CQ_START);
4468                 params[3] = FW_PARAM_PFVF(CQ_END);
4469                 params[4] = FW_PARAM_PFVF(OCQ_START);
4470                 params[5] = FW_PARAM_PFVF(OCQ_END);
4471                 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
4472                 if (ret < 0)
4473                         goto bye;
4474                 adap->vres.qp.start = val[0];
4475                 adap->vres.qp.size = val[1] - val[0] + 1;
4476                 adap->vres.cq.start = val[2];
4477                 adap->vres.cq.size = val[3] - val[2] + 1;
4478                 adap->vres.ocq.start = val[4];
4479                 adap->vres.ocq.size = val[5] - val[4] + 1;
4480         }
4481         if (caps_cmd.iscsicaps) {
4482                 params[0] = FW_PARAM_PFVF(ISCSI_START);
4483                 params[1] = FW_PARAM_PFVF(ISCSI_END);
4484                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
4485                                       params, val);
4486                 if (ret < 0)
4487                         goto bye;
4488                 adap->vres.iscsi.start = val[0];
4489                 adap->vres.iscsi.size = val[1] - val[0] + 1;
4490         }
4491 #undef FW_PARAM_PFVF
4492 #undef FW_PARAM_DEV
4493
4494         /*
4495          * These are finalized by FW initialization, load their values now.
4496          */
4497         v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
4498         adap->params.tp.tre = TIMERRESOLUTION_GET(v);
4499         adap->params.tp.dack_re = DELAYEDACKRESOLUTION_GET(v);
4500         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
4501         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
4502                      adap->params.b_wnd);
4503
4504         /* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */
4505         for (j = 0; j < NCHAN; j++)
4506                 adap->params.tp.tx_modq[j] = j;
4507
4508         t4_read_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4509                          &adap->filter_mode, 1,
4510                          TP_VLAN_PRI_MAP);
4511
4512         adap->flags |= FW_OK;
4513         return 0;
4514
4515         /*
4516          * Something bad happened.  If a command timed out or failed with EIO
4517          * FW does not operate within its spec or something catastrophic
4518          * happened to HW/FW, stop issuing commands.
4519          */
4520 bye:
4521         if (ret != -ETIMEDOUT && ret != -EIO)
4522                 t4_fw_bye(adap, adap->mbox);
4523         return ret;
4524 }
4525
4526 /* EEH callbacks */
4527
4528 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
4529                                          pci_channel_state_t state)
4530 {
4531         int i;
4532         struct adapter *adap = pci_get_drvdata(pdev);
4533
4534         if (!adap)
4535                 goto out;
4536
4537         rtnl_lock();
4538         adap->flags &= ~FW_OK;
4539         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
4540         for_each_port(adap, i) {
4541                 struct net_device *dev = adap->port[i];
4542
4543                 netif_device_detach(dev);
4544                 netif_carrier_off(dev);
4545         }
4546         if (adap->flags & FULL_INIT_DONE)
4547                 cxgb_down(adap);
4548         rtnl_unlock();
4549         pci_disable_device(pdev);
4550 out:    return state == pci_channel_io_perm_failure ?
4551                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
4552 }
4553
4554 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
4555 {
4556         int i, ret;
4557         struct fw_caps_config_cmd c;
4558         struct adapter *adap = pci_get_drvdata(pdev);
4559
4560         if (!adap) {
4561                 pci_restore_state(pdev);
4562                 pci_save_state(pdev);
4563                 return PCI_ERS_RESULT_RECOVERED;
4564         }
4565
4566         if (pci_enable_device(pdev)) {
4567                 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
4568                 return PCI_ERS_RESULT_DISCONNECT;
4569         }
4570
4571         pci_set_master(pdev);
4572         pci_restore_state(pdev);
4573         pci_save_state(pdev);
4574         pci_cleanup_aer_uncorrect_error_status(pdev);
4575
4576         if (t4_wait_dev_ready(adap) < 0)
4577                 return PCI_ERS_RESULT_DISCONNECT;
4578         if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
4579                 return PCI_ERS_RESULT_DISCONNECT;
4580         adap->flags |= FW_OK;
4581         if (adap_init1(adap, &c))
4582                 return PCI_ERS_RESULT_DISCONNECT;
4583
4584         for_each_port(adap, i) {
4585                 struct port_info *p = adap2pinfo(adap, i);
4586
4587                 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
4588                                   NULL, NULL);
4589                 if (ret < 0)
4590                         return PCI_ERS_RESULT_DISCONNECT;
4591                 p->viid = ret;
4592                 p->xact_addr_filt = -1;
4593         }
4594
4595         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
4596                      adap->params.b_wnd);
4597         setup_memwin(adap);
4598         if (cxgb_up(adap))
4599                 return PCI_ERS_RESULT_DISCONNECT;
4600         return PCI_ERS_RESULT_RECOVERED;
4601 }
4602
4603 static void eeh_resume(struct pci_dev *pdev)
4604 {
4605         int i;
4606         struct adapter *adap = pci_get_drvdata(pdev);
4607
4608         if (!adap)
4609                 return;
4610
4611         rtnl_lock();
4612         for_each_port(adap, i) {
4613                 struct net_device *dev = adap->port[i];
4614
4615                 if (netif_running(dev)) {
4616                         link_start(dev);
4617                         cxgb_set_rxmode(dev);
4618                 }
4619                 netif_device_attach(dev);
4620         }
4621         rtnl_unlock();
4622 }
4623
4624 static const struct pci_error_handlers cxgb4_eeh = {
4625         .error_detected = eeh_err_detected,
4626         .slot_reset     = eeh_slot_reset,
4627         .resume         = eeh_resume,
4628 };
4629
4630 static inline bool is_10g_port(const struct link_config *lc)
4631 {
4632         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
4633 }
4634
4635 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
4636                              unsigned int size, unsigned int iqe_size)
4637 {
4638         q->intr_params = QINTR_TIMER_IDX(timer_idx) |
4639                          (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
4640         q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
4641         q->iqe_len = iqe_size;
4642         q->size = size;
4643 }
4644
4645 /*
4646  * Perform default configuration of DMA queues depending on the number and type
4647  * of ports we found and the number of available CPUs.  Most settings can be
4648  * modified by the admin prior to actual use.
4649  */
4650 static void cfg_queues(struct adapter *adap)
4651 {
4652         struct sge *s = &adap->sge;
4653         int i, q10g = 0, n10g = 0, qidx = 0;
4654
4655         for_each_port(adap, i)
4656                 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
4657
4658         /*
4659          * We default to 1 queue per non-10G port and up to # of cores queues
4660          * per 10G port.
4661          */
4662         if (n10g)
4663                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
4664         if (q10g > netif_get_num_default_rss_queues())
4665                 q10g = netif_get_num_default_rss_queues();
4666
4667         for_each_port(adap, i) {
4668                 struct port_info *pi = adap2pinfo(adap, i);
4669
4670                 pi->first_qset = qidx;
4671                 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
4672                 qidx += pi->nqsets;
4673         }
4674
4675         s->ethqsets = qidx;
4676         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
4677
4678         if (is_offload(adap)) {
4679                 /*
4680                  * For offload we use 1 queue/channel if all ports are up to 1G,
4681                  * otherwise we divide all available queues amongst the channels
4682                  * capped by the number of available cores.
4683                  */
4684                 if (n10g) {
4685                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
4686                                   num_online_cpus());
4687                         s->ofldqsets = roundup(i, adap->params.nports);
4688                 } else
4689                         s->ofldqsets = adap->params.nports;
4690                 /* For RDMA one Rx queue per channel suffices */
4691                 s->rdmaqs = adap->params.nports;
4692         }
4693
4694         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
4695                 struct sge_eth_rxq *r = &s->ethrxq[i];
4696
4697                 init_rspq(&r->rspq, 0, 0, 1024, 64);
4698                 r->fl.size = 72;
4699         }
4700
4701         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
4702                 s->ethtxq[i].q.size = 1024;
4703
4704         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
4705                 s->ctrlq[i].q.size = 512;
4706
4707         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
4708                 s->ofldtxq[i].q.size = 1024;
4709
4710         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
4711                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
4712
4713                 init_rspq(&r->rspq, 0, 0, 1024, 64);
4714                 r->rspq.uld = CXGB4_ULD_ISCSI;
4715                 r->fl.size = 72;
4716         }
4717
4718         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
4719                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
4720
4721                 init_rspq(&r->rspq, 0, 0, 511, 64);
4722                 r->rspq.uld = CXGB4_ULD_RDMA;
4723                 r->fl.size = 72;
4724         }
4725
4726         init_rspq(&s->fw_evtq, 6, 0, 512, 64);
4727         init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
4728 }
4729
4730 /*
4731  * Reduce the number of Ethernet queues across all ports to at most n.
4732  * n provides at least one queue per port.
4733  */
4734 static void reduce_ethqs(struct adapter *adap, int n)
4735 {
4736         int i;
4737         struct port_info *pi;
4738
4739         while (n < adap->sge.ethqsets)
4740                 for_each_port(adap, i) {
4741                         pi = adap2pinfo(adap, i);
4742                         if (pi->nqsets > 1) {
4743                                 pi->nqsets--;
4744                                 adap->sge.ethqsets--;
4745                                 if (adap->sge.ethqsets <= n)
4746                                         break;
4747                         }
4748                 }
4749
4750         n = 0;
4751         for_each_port(adap, i) {
4752                 pi = adap2pinfo(adap, i);
4753                 pi->first_qset = n;
4754                 n += pi->nqsets;
4755         }
4756 }
4757
4758 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
4759 #define EXTRA_VECS 2
4760
4761 static int enable_msix(struct adapter *adap)
4762 {
4763         int ofld_need = 0;
4764         int i, err, want, need;
4765         struct sge *s = &adap->sge;
4766         unsigned int nchan = adap->params.nports;
4767         struct msix_entry entries[MAX_INGQ + 1];
4768
4769         for (i = 0; i < ARRAY_SIZE(entries); ++i)
4770                 entries[i].entry = i;
4771
4772         want = s->max_ethqsets + EXTRA_VECS;
4773         if (is_offload(adap)) {
4774                 want += s->rdmaqs + s->ofldqsets;
4775                 /* need nchan for each possible ULD */
4776                 ofld_need = 2 * nchan;
4777         }
4778         need = adap->params.nports + EXTRA_VECS + ofld_need;
4779
4780         while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
4781                 want = err;
4782
4783         if (!err) {
4784                 /*
4785                  * Distribute available vectors to the various queue groups.
4786                  * Every group gets its minimum requirement and NIC gets top
4787                  * priority for leftovers.
4788                  */
4789                 i = want - EXTRA_VECS - ofld_need;
4790                 if (i < s->max_ethqsets) {
4791                         s->max_ethqsets = i;
4792                         if (i < s->ethqsets)
4793                                 reduce_ethqs(adap, i);
4794                 }
4795                 if (is_offload(adap)) {
4796                         i = want - EXTRA_VECS - s->max_ethqsets;
4797                         i -= ofld_need - nchan;
4798                         s->ofldqsets = (i / nchan) * nchan;  /* round down */
4799                 }
4800                 for (i = 0; i < want; ++i)
4801                         adap->msix_info[i].vec = entries[i].vector;
4802         } else if (err > 0)
4803                 dev_info(adap->pdev_dev,
4804                          "only %d MSI-X vectors left, not using MSI-X\n", err);
4805         return err;
4806 }
4807
4808 #undef EXTRA_VECS
4809
4810 static int init_rss(struct adapter *adap)
4811 {
4812         unsigned int i, j;
4813
4814         for_each_port(adap, i) {
4815                 struct port_info *pi = adap2pinfo(adap, i);
4816
4817                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
4818                 if (!pi->rss)
4819                         return -ENOMEM;
4820                 for (j = 0; j < pi->rss_size; j++)
4821                         pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
4822         }
4823         return 0;
4824 }
4825
4826 static void print_port_info(const struct net_device *dev)
4827 {
4828         static const char *base[] = {
4829                 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
4830                 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
4831         };
4832
4833         char buf[80];
4834         char *bufp = buf;
4835         const char *spd = "";
4836         const struct port_info *pi = netdev_priv(dev);
4837         const struct adapter *adap = pi->adapter;
4838
4839         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
4840                 spd = " 2.5 GT/s";
4841         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
4842                 spd = " 5 GT/s";
4843
4844         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
4845                 bufp += sprintf(bufp, "100/");
4846         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
4847                 bufp += sprintf(bufp, "1000/");
4848         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
4849                 bufp += sprintf(bufp, "10G/");
4850         if (bufp != buf)
4851                 --bufp;
4852         sprintf(bufp, "BASE-%s", base[pi->port_type]);
4853
4854         netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
4855                     adap->params.vpd.id,
4856                     CHELSIO_CHIP_RELEASE(adap->params.rev), buf,
4857                     is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
4858                     (adap->flags & USING_MSIX) ? " MSI-X" :
4859                     (adap->flags & USING_MSI) ? " MSI" : "");
4860         netdev_info(dev, "S/N: %s, E/C: %s\n",
4861                     adap->params.vpd.sn, adap->params.vpd.ec);
4862 }
4863
4864 static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
4865 {
4866         pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
4867 }
4868
4869 /*
4870  * Free the following resources:
4871  * - memory used for tables
4872  * - MSI/MSI-X
4873  * - net devices
4874  * - resources FW is holding for us
4875  */
4876 static void free_some_resources(struct adapter *adapter)
4877 {
4878         unsigned int i;
4879
4880         t4_free_mem(adapter->l2t);
4881         t4_free_mem(adapter->tids.tid_tab);
4882         disable_msi(adapter);
4883
4884         for_each_port(adapter, i)
4885                 if (adapter->port[i]) {
4886                         kfree(adap2pinfo(adapter, i)->rss);
4887                         free_netdev(adapter->port[i]);
4888                 }
4889         if (adapter->flags & FW_OK)
4890                 t4_fw_bye(adapter, adapter->fn);
4891 }
4892
4893 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
4894 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
4895                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
4896
4897 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
4898 {
4899         int func, i, err;
4900         struct port_info *pi;
4901         bool highdma = false;
4902         struct adapter *adapter = NULL;
4903 #ifdef CONFIG_PCI_IOV
4904         int max_no_pf;
4905 #endif
4906
4907         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
4908
4909         err = pci_request_regions(pdev, KBUILD_MODNAME);
4910         if (err) {
4911                 /* Just info, some other driver may have claimed the device. */
4912                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
4913                 return err;
4914         }
4915
4916         /* We control everything through one PF */
4917         func = PCI_FUNC(pdev->devfn);
4918         if (func != ent->driver_data) {
4919                 pci_save_state(pdev);        /* to restore SR-IOV later */
4920                 goto sriov;
4921         }
4922
4923         err = pci_enable_device(pdev);
4924         if (err) {
4925                 dev_err(&pdev->dev, "cannot enable PCI device\n");
4926                 goto out_release_regions;
4927         }
4928
4929         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
4930                 highdma = true;
4931                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
4932                 if (err) {
4933                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
4934                                 "coherent allocations\n");
4935                         goto out_disable_device;
4936                 }
4937         } else {
4938                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
4939                 if (err) {
4940                         dev_err(&pdev->dev, "no usable DMA configuration\n");
4941                         goto out_disable_device;
4942                 }
4943         }
4944
4945         pci_enable_pcie_error_reporting(pdev);
4946         enable_pcie_relaxed_ordering(pdev);
4947         pci_set_master(pdev);
4948         pci_save_state(pdev);
4949
4950         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
4951         if (!adapter) {
4952                 err = -ENOMEM;
4953                 goto out_disable_device;
4954         }
4955
4956         adapter->regs = pci_ioremap_bar(pdev, 0);
4957         if (!adapter->regs) {
4958                 dev_err(&pdev->dev, "cannot map device registers\n");
4959                 err = -ENOMEM;
4960                 goto out_free_adapter;
4961         }
4962
4963         adapter->pdev = pdev;
4964         adapter->pdev_dev = &pdev->dev;
4965         adapter->mbox = func;
4966         adapter->fn = func;
4967         adapter->msg_enable = dflt_msg_enable;
4968         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
4969
4970         spin_lock_init(&adapter->stats_lock);
4971         spin_lock_init(&adapter->tid_release_lock);
4972
4973         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
4974         INIT_WORK(&adapter->db_full_task, process_db_full);
4975         INIT_WORK(&adapter->db_drop_task, process_db_drop);
4976
4977         err = t4_prep_adapter(adapter);
4978         if (err)
4979                 goto out_unmap_bar;
4980         setup_memwin(adapter);
4981         err = adap_init0(adapter);
4982         setup_memwin_rdma(adapter);
4983         if (err)
4984                 goto out_unmap_bar;
4985
4986         for_each_port(adapter, i) {
4987                 struct net_device *netdev;
4988
4989                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
4990                                            MAX_ETH_QSETS);
4991                 if (!netdev) {
4992                         err = -ENOMEM;
4993                         goto out_free_dev;
4994                 }
4995
4996                 SET_NETDEV_DEV(netdev, &pdev->dev);
4997
4998                 adapter->port[i] = netdev;
4999                 pi = netdev_priv(netdev);
5000                 pi->adapter = adapter;
5001                 pi->xact_addr_filt = -1;
5002                 pi->port_id = i;
5003                 netdev->irq = pdev->irq;
5004
5005                 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
5006                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
5007                         NETIF_F_RXCSUM | NETIF_F_RXHASH |
5008                         NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
5009                 if (highdma)
5010                         netdev->hw_features |= NETIF_F_HIGHDMA;
5011                 netdev->features |= netdev->hw_features;
5012                 netdev->vlan_features = netdev->features & VLAN_FEAT;
5013
5014                 netdev->priv_flags |= IFF_UNICAST_FLT;
5015
5016                 netdev->netdev_ops = &cxgb4_netdev_ops;
5017                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
5018         }
5019
5020         pci_set_drvdata(pdev, adapter);
5021
5022         if (adapter->flags & FW_OK) {
5023                 err = t4_port_init(adapter, func, func, 0);
5024                 if (err)
5025                         goto out_free_dev;
5026         }
5027
5028         /*
5029          * Configure queues and allocate tables now, they can be needed as
5030          * soon as the first register_netdev completes.
5031          */
5032         cfg_queues(adapter);
5033
5034         adapter->l2t = t4_init_l2t();
5035         if (!adapter->l2t) {
5036                 /* We tolerate a lack of L2T, giving up some functionality */
5037                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
5038                 adapter->params.offload = 0;
5039         }
5040
5041         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
5042                 dev_warn(&pdev->dev, "could not allocate TID table, "
5043                          "continuing\n");
5044                 adapter->params.offload = 0;
5045         }
5046
5047         /* See what interrupts we'll be using */
5048         if (msi > 1 && enable_msix(adapter) == 0)
5049                 adapter->flags |= USING_MSIX;
5050         else if (msi > 0 && pci_enable_msi(pdev) == 0)
5051                 adapter->flags |= USING_MSI;
5052
5053         err = init_rss(adapter);
5054         if (err)
5055                 goto out_free_dev;
5056
5057         /*
5058          * The card is now ready to go.  If any errors occur during device
5059          * registration we do not fail the whole card but rather proceed only
5060          * with the ports we manage to register successfully.  However we must
5061          * register at least one net device.
5062          */
5063         for_each_port(adapter, i) {
5064                 pi = adap2pinfo(adapter, i);
5065                 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
5066                 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
5067
5068                 err = register_netdev(adapter->port[i]);
5069                 if (err)
5070                         break;
5071                 adapter->chan_map[pi->tx_chan] = i;
5072                 print_port_info(adapter->port[i]);
5073         }
5074         if (i == 0) {
5075                 dev_err(&pdev->dev, "could not register any net devices\n");
5076                 goto out_free_dev;
5077         }
5078         if (err) {
5079                 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
5080                 err = 0;
5081         }
5082
5083         if (cxgb4_debugfs_root) {
5084                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
5085                                                            cxgb4_debugfs_root);
5086                 setup_debugfs(adapter);
5087         }
5088
5089         /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
5090         pdev->needs_freset = 1;
5091
5092         if (is_offload(adapter))
5093                 attach_ulds(adapter);
5094
5095 sriov:
5096 #ifdef CONFIG_PCI_IOV
5097         max_no_pf = is_t4(adapter->chip) ? NUM_OF_PF_WITH_SRIOV_T4 :
5098                         NUM_OF_PF_WITH_SRIOV_T5;
5099
5100         if (func < max_no_pf && num_vf[func] > 0)
5101                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
5102                         dev_info(&pdev->dev,
5103                                  "instantiated %u virtual functions\n",
5104                                  num_vf[func]);
5105 #endif
5106         return 0;
5107
5108  out_free_dev:
5109         free_some_resources(adapter);
5110  out_unmap_bar:
5111         iounmap(adapter->regs);
5112  out_free_adapter:
5113         kfree(adapter);
5114  out_disable_device:
5115         pci_disable_pcie_error_reporting(pdev);
5116         pci_disable_device(pdev);
5117  out_release_regions:
5118         pci_release_regions(pdev);
5119         pci_set_drvdata(pdev, NULL);
5120         return err;
5121 }
5122
5123 static void remove_one(struct pci_dev *pdev)
5124 {
5125         struct adapter *adapter = pci_get_drvdata(pdev);
5126
5127 #ifdef CONFIG_PCI_IOV
5128         pci_disable_sriov(pdev);
5129
5130 #endif
5131
5132         if (adapter) {
5133                 int i;
5134
5135                 if (is_offload(adapter))
5136                         detach_ulds(adapter);
5137
5138                 for_each_port(adapter, i)
5139                         if (adapter->port[i]->reg_state == NETREG_REGISTERED)
5140                                 unregister_netdev(adapter->port[i]);
5141
5142                 if (adapter->debugfs_root)
5143                         debugfs_remove_recursive(adapter->debugfs_root);
5144
5145                 /* If we allocated filters, free up state associated with any
5146                  * valid filters ...
5147                  */
5148                 if (adapter->tids.ftid_tab) {
5149                         struct filter_entry *f = &adapter->tids.ftid_tab[0];
5150                         for (i = 0; i < (adapter->tids.nftids +
5151                                         adapter->tids.nsftids); i++, f++)
5152                                 if (f->valid)
5153                                         clear_filter(adapter, f);
5154                 }
5155
5156                 if (adapter->flags & FULL_INIT_DONE)
5157                         cxgb_down(adapter);
5158
5159                 free_some_resources(adapter);
5160                 iounmap(adapter->regs);
5161                 kfree(adapter);
5162                 pci_disable_pcie_error_reporting(pdev);
5163                 pci_disable_device(pdev);
5164                 pci_release_regions(pdev);
5165                 pci_set_drvdata(pdev, NULL);
5166         } else
5167                 pci_release_regions(pdev);
5168 }
5169
5170 static struct pci_driver cxgb4_driver = {
5171         .name     = KBUILD_MODNAME,
5172         .id_table = cxgb4_pci_tbl,
5173         .probe    = init_one,
5174         .remove   = remove_one,
5175         .err_handler = &cxgb4_eeh,
5176 };
5177
5178 static int __init cxgb4_init_module(void)
5179 {
5180         int ret;
5181
5182         workq = create_singlethread_workqueue("cxgb4");
5183         if (!workq)
5184                 return -ENOMEM;
5185
5186         /* Debugfs support is optional, just warn if this fails */
5187         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
5188         if (!cxgb4_debugfs_root)
5189                 pr_warn("could not create debugfs entry, continuing\n");
5190
5191         ret = pci_register_driver(&cxgb4_driver);
5192         if (ret < 0)
5193                 debugfs_remove(cxgb4_debugfs_root);
5194         return ret;
5195 }
5196
5197 static void __exit cxgb4_cleanup_module(void)
5198 {
5199         pci_unregister_driver(&cxgb4_driver);
5200         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
5201         flush_workqueue(workq);
5202         destroy_workqueue(workq);
5203 }
5204
5205 module_init(cxgb4_init_module);
5206 module_exit(cxgb4_cleanup_module);