drivers/net/ethernet/sfc/efx_channels.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2018 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include "net_driver.h"
  12 #include <linux/module.h>
  13 #include <linux/filter.h>
  14 #include "efx_channels.h"
  15 #include "efx.h"
  16 #include "efx_common.h"
  17 #include "tx_common.h"
  18 #include "rx_common.h"
  19 #include "nic.h"
  20 #include "sriov.h"
  21 #include "workarounds.h"
  22
  23 /* This is the first interrupt mode to try out of:
  24  * 0 => MSI-X
  25  * 1 => MSI
  26  * 2 => legacy
  27  */
  28 unsigned int efx_interrupt_mode = EFX_INT_MODE_MSIX;
  29
  30 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
  31  * i.e. the number of CPUs among which we may distribute simultaneous
  32  * interrupt handling.
  33  *
  34  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
  35  * The default (0) means to assign an interrupt to each core.
  36  */
  37 unsigned int rss_cpus;
  38
  39 static unsigned int irq_adapt_low_thresh = 8000;
  40 module_param(irq_adapt_low_thresh, uint, 0644);
  41 MODULE_PARM_DESC(irq_adapt_low_thresh,
  42                  "Threshold score for reducing IRQ moderation");
  43
  44 static unsigned int irq_adapt_high_thresh = 16000;
  45 module_param(irq_adapt_high_thresh, uint, 0644);
  46 MODULE_PARM_DESC(irq_adapt_high_thresh,
  47                  "Threshold score for increasing IRQ moderation");
  48
  49 /* This is the weight assigned to each of the (per-channel) virtual
  50  * NAPI devices.
  51  */
  52 static int napi_weight = 64;
  53
  54 /***************
  55  * Housekeeping
  56  ***************/
  57
  58 int efx_channel_dummy_op_int(struct efx_channel *channel)
  59 {
  60         return 0;
  61 }
  62
  63 void efx_channel_dummy_op_void(struct efx_channel *channel)
  64 {
  65 }
  66
  67 static const struct efx_channel_type efx_default_channel_type = {
  68         .pre_probe              = efx_channel_dummy_op_int,
  69         .post_remove            = efx_channel_dummy_op_void,
  70         .get_name               = efx_get_channel_name,
  71         .copy                   = efx_copy_channel,
  72         .want_txqs              = efx_default_channel_want_txqs,
  73         .keep_eventq            = false,
  74         .want_pio               = true,
  75 };
  76
  77 /*************
  78  * INTERRUPTS
  79  *************/
  80
  81 static unsigned int count_online_cores(struct efx_nic *efx, bool local_node)
  82 {
  83         cpumask_var_t filter_mask;
  84         unsigned int count;
  85         int cpu;
  86
  87         if (unlikely(!zalloc_cpumask_var(&filter_mask, GFP_KERNEL))) {
  88                 netif_warn(efx, probe, efx->net_dev,
  89                            "RSS disabled due to allocation failure\n");
  90                 return 1;
  91         }
  92
  93         cpumask_copy(filter_mask, cpu_online_mask);
  94         if (local_node) {
  95                 int numa_node = pcibus_to_node(efx->pci_dev->bus);
  96
  97                 cpumask_and(filter_mask, filter_mask, cpumask_of_node(numa_node));
  98         }
  99
 100         count = 0;
 101         for_each_cpu(cpu, filter_mask) {
 102                 ++count;
 103                 cpumask_andnot(filter_mask, filter_mask, topology_sibling_cpumask(cpu));
 104         }
 105
 106         free_cpumask_var(filter_mask);
 107
 108         return count;
 109 }
 110
 111 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
 112 {
 113         unsigned int count;
 114
 115         if (rss_cpus) {
 116                 count = rss_cpus;
 117         } else {
 118                 count = count_online_cores(efx, true);
 119
 120                 /* If no online CPUs in local node, fallback to any online CPUs */
 121                 if (count == 0)
 122                         count = count_online_cores(efx, false);
 123         }
 124
 125         if (count > EFX_MAX_RX_QUEUES) {
 126                 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
 127                                "Reducing number of rx queues from %u to %u.\n",
 128                                count, EFX_MAX_RX_QUEUES);
 129                 count = EFX_MAX_RX_QUEUES;
 130         }
 131
 132         /* If RSS is requested for the PF *and* VFs then we can't write RSS
 133          * table entries that are inaccessible to VFs
 134          */
 135 #ifdef CONFIG_SFC_SRIOV
 136         if (efx->type->sriov_wanted) {
 137                 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
 138                     count > efx_vf_size(efx)) {
 139                         netif_warn(efx, probe, efx->net_dev,
 140                                    "Reducing number of RSS channels from %u to %u for "
 141                                    "VF support. Increase vf-msix-limit to use more "
 142                                    "channels on the PF.\n",
 143                                    count, efx_vf_size(efx));
 144                         count = efx_vf_size(efx);
 145                 }
 146         }
 147 #endif
 148
 149         return count;
 150 }
 151
 152 static int efx_allocate_msix_channels(struct efx_nic *efx,
 153                                       unsigned int max_channels,
 154                                       unsigned int extra_channels,
 155                                       unsigned int parallelism)
 156 {
 157         unsigned int n_channels = parallelism;
 158         int vec_count;
 159         int tx_per_ev;
 160         int n_xdp_tx;
 161         int n_xdp_ev;
 162
 163         if (efx_separate_tx_channels)
 164                 n_channels *= 2;
 165         n_channels += extra_channels;
 166
 167         /* To allow XDP transmit to happen from arbitrary NAPI contexts
 168          * we allocate a TX queue per CPU. We share event queues across
 169          * multiple tx queues, assuming tx and ev queues are both
 170          * maximum size.
 171          */
 172         tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx);
 173         tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL);
 174         n_xdp_tx = num_possible_cpus();
 175         n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev);
 176
 177         vec_count = pci_msix_vec_count(efx->pci_dev);
 178         if (vec_count < 0)
 179                 return vec_count;
 180
 181         max_channels = min_t(unsigned int, vec_count, max_channels);
 182
 183         /* Check resources.
 184          * We need a channel per event queue, plus a VI per tx queue.
 185          * This may be more pessimistic than it needs to be.
 186          */
 187         if (n_channels >= max_channels) {
 188                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
 189                 netif_warn(efx, drv, efx->net_dev,
 190                            "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
 191                            n_xdp_ev, n_channels, max_channels);
 192                 netif_warn(efx, drv, efx->net_dev,
 193                            "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
 194         } else if (n_channels + n_xdp_tx > efx->max_vis) {
 195                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
 196                 netif_warn(efx, drv, efx->net_dev,
 197                            "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n",
 198                            n_xdp_tx, n_channels, efx->max_vis);
 199                 netif_warn(efx, drv, efx->net_dev,
 200                            "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
 201         } else if (n_channels + n_xdp_ev > max_channels) {
 202                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED;
 203                 netif_warn(efx, drv, efx->net_dev,
 204                            "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
 205                            n_xdp_ev, n_channels, max_channels);
 206
 207                 n_xdp_ev = max_channels - n_channels;
 208                 netif_warn(efx, drv, efx->net_dev,
 209                            "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n",
 210                            DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev));
 211         } else {
 212                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED;
 213         }
 214
 215         if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) {
 216                 efx->n_xdp_channels = n_xdp_ev;
 217                 efx->xdp_tx_per_channel = tx_per_ev;
 218                 efx->xdp_tx_queue_count = n_xdp_tx;
 219                 n_channels += n_xdp_ev;
 220                 netif_dbg(efx, drv, efx->net_dev,
 221                           "Allocating %d TX and %d event queues for XDP\n",
 222                           n_xdp_ev * tx_per_ev, n_xdp_ev);
 223         } else {
 224                 efx->n_xdp_channels = 0;
 225                 efx->xdp_tx_per_channel = 0;
 226                 efx->xdp_tx_queue_count = n_xdp_tx;
 227         }
 228
 229         if (vec_count < n_channels) {
 230                 netif_err(efx, drv, efx->net_dev,
 231                           "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
 232                           vec_count, n_channels);
 233                 netif_err(efx, drv, efx->net_dev,
 234                           "WARNING: Performance may be reduced.\n");
 235                 n_channels = vec_count;
 236         }
 237
 238         n_channels = min(n_channels, max_channels);
 239
 240         efx->n_channels = n_channels;
 241
 242         /* Ignore XDP tx channels when creating rx channels. */
 243         n_channels -= efx->n_xdp_channels;
 244
 245         if (efx_separate_tx_channels) {
 246                 efx->n_tx_channels =
 247                         min(max(n_channels / 2, 1U),
 248                             efx->max_tx_channels);
 249                 efx->tx_channel_offset =
 250                         n_channels - efx->n_tx_channels;
 251                 efx->n_rx_channels =
 252                         max(n_channels -
 253                             efx->n_tx_channels, 1U);
 254         } else {
 255                 efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
 256                 efx->tx_channel_offset = 0;
 257                 efx->n_rx_channels = n_channels;
 258         }
 259
 260         efx->n_rx_channels = min(efx->n_rx_channels, parallelism);
 261         efx->n_tx_channels = min(efx->n_tx_channels, parallelism);
 262
 263         efx->xdp_channel_offset = n_channels;
 264
 265         netif_dbg(efx, drv, efx->net_dev,
 266                   "Allocating %u RX channels\n",
 267                   efx->n_rx_channels);
 268
 269         return efx->n_channels;
 270 }
 271
 272 /* Probe the number and type of interrupts we are able to obtain, and
 273  * the resulting numbers of channels and RX queues.
 274  */
 275 int efx_probe_interrupts(struct efx_nic *efx)
 276 {
 277         unsigned int extra_channels = 0;
 278         unsigned int rss_spread;
 279         unsigned int i, j;
 280         int rc;
 281
 282         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
 283                 if (efx->extra_channel_type[i])
 284                         ++extra_channels;
 285
 286         if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
 287                 unsigned int parallelism = efx_wanted_parallelism(efx);
 288                 struct msix_entry xentries[EFX_MAX_CHANNELS];
 289                 unsigned int n_channels;
 290
 291                 rc = efx_allocate_msix_channels(efx, efx->max_channels,
 292                                                 extra_channels, parallelism);
 293                 if (rc >= 0) {
 294                         n_channels = rc;
 295                         for (i = 0; i < n_channels; i++)
 296                                 xentries[i].entry = i;
 297                         rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
 298                                                    n_channels);
 299                 }
 300                 if (rc < 0) {
 301                         /* Fall back to single channel MSI */
 302                         netif_err(efx, drv, efx->net_dev,
 303                                   "could not enable MSI-X\n");
 304                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
 305                                 efx->interrupt_mode = EFX_INT_MODE_MSI;
 306                         else
 307                                 return rc;
 308                 } else if (rc < n_channels) {
 309                         netif_err(efx, drv, efx->net_dev,
 310                                   "WARNING: Insufficient MSI-X vectors"
 311                                   " available (%d < %u).\n", rc, n_channels);
 312                         netif_err(efx, drv, efx->net_dev,
 313                                   "WARNING: Performance may be reduced.\n");
 314                         n_channels = rc;
 315                 }
 316
 317                 if (rc > 0) {
 318                         for (i = 0; i < efx->n_channels; i++)
 319                                 efx_get_channel(efx, i)->irq =
 320                                         xentries[i].vector;
 321                 }
 322         }
 323
 324         /* Try single interrupt MSI */
 325         if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
 326                 efx->n_channels = 1;
 327                 efx->n_rx_channels = 1;
 328                 efx->n_tx_channels = 1;
 329                 efx->n_xdp_channels = 0;
 330                 efx->xdp_channel_offset = efx->n_channels;
 331                 rc = pci_enable_msi(efx->pci_dev);
 332                 if (rc == 0) {
 333                         efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
 334                 } else {
 335                         netif_err(efx, drv, efx->net_dev,
 336                                   "could not enable MSI\n");
 337                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
 338                                 efx->interrupt_mode = EFX_INT_MODE_LEGACY;
 339                         else
 340                                 return rc;
 341                 }
 342         }
 343
 344         /* Assume legacy interrupts */
 345         if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
 346                 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
 347                 efx->n_rx_channels = 1;
 348                 efx->n_tx_channels = 1;
 349                 efx->n_xdp_channels = 0;
 350                 efx->xdp_channel_offset = efx->n_channels;
 351                 efx->legacy_irq = efx->pci_dev->irq;
 352         }
 353
 354         /* Assign extra channels if possible, before XDP channels */
 355         efx->n_extra_tx_channels = 0;
 356         j = efx->xdp_channel_offset;
 357         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
 358                 if (!efx->extra_channel_type[i])
 359                         continue;
 360                 if (j <= efx->tx_channel_offset + efx->n_tx_channels) {
 361                         efx->extra_channel_type[i]->handle_no_channel(efx);
 362                 } else {
 363                         --j;
 364                         efx_get_channel(efx, j)->type =
 365                                 efx->extra_channel_type[i];
 366                         if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
 367                                 efx->n_extra_tx_channels++;
 368                 }
 369         }
 370
 371         rss_spread = efx->n_rx_channels;
 372         /* RSS might be usable on VFs even if it is disabled on the PF */
 373 #ifdef CONFIG_SFC_SRIOV
 374         if (efx->type->sriov_wanted) {
 375                 efx->rss_spread = ((rss_spread > 1 ||
 376                                     !efx->type->sriov_wanted(efx)) ?
 377                                    rss_spread : efx_vf_size(efx));
 378                 return 0;
 379         }
 380 #endif
 381         efx->rss_spread = rss_spread;
 382
 383         return 0;
 384 }
 385
 386 #if defined(CONFIG_SMP)
 387 void efx_set_interrupt_affinity(struct efx_nic *efx)
 388 {
 389         int numa_node = pcibus_to_node(efx->pci_dev->bus);
 390         const struct cpumask *numa_mask = cpumask_of_node(numa_node);
 391         struct efx_channel *channel;
 392         unsigned int cpu;
 393
 394         /* If no online CPUs in local node, fallback to any online CPU */
 395         if (cpumask_first_and(cpu_online_mask, numa_mask) >= nr_cpu_ids)
 396                 numa_mask = cpu_online_mask;
 397
 398         cpu = -1;
 399         efx_for_each_channel(channel, efx) {
 400                 cpu = cpumask_next_and(cpu, cpu_online_mask, numa_mask);
 401                 if (cpu >= nr_cpu_ids)
 402                         cpu = cpumask_first_and(cpu_online_mask, numa_mask);
 403                 irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
 404         }
 405 }
 406
 407 void efx_clear_interrupt_affinity(struct efx_nic *efx)
 408 {
 409         struct efx_channel *channel;
 410
 411         efx_for_each_channel(channel, efx)
 412                 irq_set_affinity_hint(channel->irq, NULL);
 413 }
 414 #else
 415 void
 416 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
 417 {
 418 }
 419
 420 void
 421 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
 422 {
 423 }
 424 #endif /* CONFIG_SMP */
 425
 426 void efx_remove_interrupts(struct efx_nic *efx)
 427 {
 428         struct efx_channel *channel;
 429
 430         /* Remove MSI/MSI-X interrupts */
 431         efx_for_each_channel(channel, efx)
 432                 channel->irq = 0;
 433         pci_disable_msi(efx->pci_dev);
 434         pci_disable_msix(efx->pci_dev);
 435
 436         /* Remove legacy interrupt */
 437         efx->legacy_irq = 0;
 438 }
 439
 440 /***************
 441  * EVENT QUEUES
 442  ***************/
 443
 444 /* Create event queue
 445  * Event queue memory allocations are done only once.  If the channel
 446  * is reset, the memory buffer will be reused; this guards against
 447  * errors during channel reset and also simplifies interrupt handling.
 448  */
 449 int efx_probe_eventq(struct efx_channel *channel)
 450 {
 451         struct efx_nic *efx = channel->efx;
 452         unsigned long entries;
 453
 454         netif_dbg(efx, probe, efx->net_dev,
 455                   "chan %d create event queue\n", channel->channel);
 456
 457         /* Build an event queue with room for one event per tx and rx buffer,
 458          * plus some extra for link state events and MCDI completions.
 459          */
 460         entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
 461         EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
 462         channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
 463
 464         return efx_nic_probe_eventq(channel);
 465 }
 466
 467 /* Prepare channel's event queue */
 468 int efx_init_eventq(struct efx_channel *channel)
 469 {
 470         struct efx_nic *efx = channel->efx;
 471         int rc;
 472
 473         EFX_WARN_ON_PARANOID(channel->eventq_init);
 474
 475         netif_dbg(efx, drv, efx->net_dev,
 476                   "chan %d init event queue\n", channel->channel);
 477
 478         rc = efx_nic_init_eventq(channel);
 479         if (rc == 0) {
 480                 efx->type->push_irq_moderation(channel);
 481                 channel->eventq_read_ptr = 0;
 482                 channel->eventq_init = true;
 483         }
 484         return rc;
 485 }
 486
 487 /* Enable event queue processing and NAPI */
 488 void efx_start_eventq(struct efx_channel *channel)
 489 {
 490         netif_dbg(channel->efx, ifup, channel->efx->net_dev,
 491                   "chan %d start event queue\n", channel->channel);
 492
 493         /* Make sure the NAPI handler sees the enabled flag set */
 494         channel->enabled = true;
 495         smp_wmb();
 496
 497         napi_enable(&channel->napi_str);
 498         efx_nic_eventq_read_ack(channel);
 499 }
 500
 501 /* Disable event queue processing and NAPI */
 502 void efx_stop_eventq(struct efx_channel *channel)
 503 {
 504         if (!channel->enabled)
 505                 return;
 506
 507         napi_disable(&channel->napi_str);
 508         channel->enabled = false;
 509 }
 510
 511 void efx_fini_eventq(struct efx_channel *channel)
 512 {
 513         if (!channel->eventq_init)
 514                 return;
 515
 516         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 517                   "chan %d fini event queue\n", channel->channel);
 518
 519         efx_nic_fini_eventq(channel);
 520         channel->eventq_init = false;
 521 }
 522
 523 void efx_remove_eventq(struct efx_channel *channel)
 524 {
 525         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 526                   "chan %d remove event queue\n", channel->channel);
 527
 528         efx_nic_remove_eventq(channel);
 529 }
 530
 531 /**************************************************************************
 532  *
 533  * Channel handling
 534  *
 535  *************************************************************************/
 536
 537 #ifdef CONFIG_RFS_ACCEL
 538 static void efx_filter_rfs_expire(struct work_struct *data)
 539 {
 540         struct delayed_work *dwork = to_delayed_work(data);
 541         struct efx_channel *channel;
 542         unsigned int time, quota;
 543
 544         channel = container_of(dwork, struct efx_channel, filter_work);
 545         time = jiffies - channel->rfs_last_expiry;
 546         quota = channel->rfs_filter_count * time / (30 * HZ);
 547         if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota)))
 548                 channel->rfs_last_expiry += time;
 549         /* Ensure we do more work eventually even if NAPI poll is not happening */
 550         schedule_delayed_work(dwork, 30 * HZ);
 551 }
 552 #endif
 553
 554 /* Allocate and initialise a channel structure. */
 555 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i)
 556 {
 557         struct efx_rx_queue *rx_queue;
 558         struct efx_tx_queue *tx_queue;
 559         struct efx_channel *channel;
 560         int j;
 561
 562         channel = kzalloc(sizeof(*channel), GFP_KERNEL);
 563         if (!channel)
 564                 return NULL;
 565
 566         channel->efx = efx;
 567         channel->channel = i;
 568         channel->type = &efx_default_channel_type;
 569
 570         for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
 571                 tx_queue = &channel->tx_queue[j];
 572                 tx_queue->efx = efx;
 573                 tx_queue->queue = -1;
 574                 tx_queue->label = j;
 575                 tx_queue->channel = channel;
 576         }
 577
 578 #ifdef CONFIG_RFS_ACCEL
 579         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 580 #endif
 581
 582         rx_queue = &channel->rx_queue;
 583         rx_queue->efx = efx;
 584         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 585
 586         return channel;
 587 }
 588
 589 int efx_init_channels(struct efx_nic *efx)
 590 {
 591         unsigned int i;
 592
 593         for (i = 0; i < EFX_MAX_CHANNELS; i++) {
 594                 efx->channel[i] = efx_alloc_channel(efx, i);
 595                 if (!efx->channel[i])
 596                         return -ENOMEM;
 597                 efx->msi_context[i].efx = efx;
 598                 efx->msi_context[i].index = i;
 599         }
 600
 601         /* Higher numbered interrupt modes are less capable! */
 602         efx->interrupt_mode = min(efx->type->min_interrupt_mode,
 603                                   efx_interrupt_mode);
 604
 605         efx->max_channels = EFX_MAX_CHANNELS;
 606         efx->max_tx_channels = EFX_MAX_CHANNELS;
 607
 608         return 0;
 609 }
 610
 611 void efx_fini_channels(struct efx_nic *efx)
 612 {
 613         unsigned int i;
 614
 615         for (i = 0; i < EFX_MAX_CHANNELS; i++)
 616                 if (efx->channel[i]) {
 617                         kfree(efx->channel[i]);
 618                         efx->channel[i] = NULL;
 619                 }
 620 }
 621
 622 /* Allocate and initialise a channel structure, copying parameters
 623  * (but not resources) from an old channel structure.
 624  */
 625 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
 626 {
 627         struct efx_rx_queue *rx_queue;
 628         struct efx_tx_queue *tx_queue;
 629         struct efx_channel *channel;
 630         int j;
 631
 632         channel = kmalloc(sizeof(*channel), GFP_KERNEL);
 633         if (!channel)
 634                 return NULL;
 635
 636         *channel = *old_channel;
 637
 638         channel->napi_dev = NULL;
 639         INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
 640         channel->napi_str.napi_id = 0;
 641         channel->napi_str.state = 0;
 642         memset(&channel->eventq, 0, sizeof(channel->eventq));
 643
 644         for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
 645                 tx_queue = &channel->tx_queue[j];
 646                 if (tx_queue->channel)
 647                         tx_queue->channel = channel;
 648                 tx_queue->buffer = NULL;
 649                 tx_queue->cb_page = NULL;
 650                 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
 651         }
 652
 653         rx_queue = &channel->rx_queue;
 654         rx_queue->buffer = NULL;
 655         memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
 656         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 657 #ifdef CONFIG_RFS_ACCEL
 658         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 659 #endif
 660
 661         return channel;
 662 }
 663
 664 static int efx_probe_channel(struct efx_channel *channel)
 665 {
 666         struct efx_tx_queue *tx_queue;
 667         struct efx_rx_queue *rx_queue;
 668         int rc;
 669
 670         netif_dbg(channel->efx, probe, channel->efx->net_dev,
 671                   "creating channel %d\n", channel->channel);
 672
 673         rc = channel->type->pre_probe(channel);
 674         if (rc)
 675                 goto fail;
 676
 677         rc = efx_probe_eventq(channel);
 678         if (rc)
 679                 goto fail;
 680
 681         efx_for_each_channel_tx_queue(tx_queue, channel) {
 682                 rc = efx_probe_tx_queue(tx_queue);
 683                 if (rc)
 684                         goto fail;
 685         }
 686
 687         efx_for_each_channel_rx_queue(rx_queue, channel) {
 688                 rc = efx_probe_rx_queue(rx_queue);
 689                 if (rc)
 690                         goto fail;
 691         }
 692
 693         channel->rx_list = NULL;
 694
 695         return 0;
 696
 697 fail:
 698         efx_remove_channel(channel);
 699         return rc;
 700 }
 701
 702 void efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)
 703 {
 704         struct efx_nic *efx = channel->efx;
 705         const char *type;
 706         int number;
 707
 708         number = channel->channel;
 709
 710         if (number >= efx->xdp_channel_offset &&
 711             !WARN_ON_ONCE(!efx->n_xdp_channels)) {
 712                 type = "-xdp";
 713                 number -= efx->xdp_channel_offset;
 714         } else if (efx->tx_channel_offset == 0) {
 715                 type = "";
 716         } else if (number < efx->tx_channel_offset) {
 717                 type = "-rx";
 718         } else {
 719                 type = "-tx";
 720                 number -= efx->tx_channel_offset;
 721         }
 722         snprintf(buf, len, "%s%s-%d", efx->name, type, number);
 723 }
 724
 725 void efx_set_channel_names(struct efx_nic *efx)
 726 {
 727         struct efx_channel *channel;
 728
 729         efx_for_each_channel(channel, efx)
 730                 channel->type->get_name(channel,
 731                                         efx->msi_context[channel->channel].name,
 732                                         sizeof(efx->msi_context[0].name));
 733 }
 734
 735 int efx_probe_channels(struct efx_nic *efx)
 736 {
 737         struct efx_channel *channel;
 738         int rc;
 739
 740         /* Restart special buffer allocation */
 741         efx->next_buffer_table = 0;
 742
 743         /* Probe channels in reverse, so that any 'extra' channels
 744          * use the start of the buffer table. This allows the traffic
 745          * channels to be resized without moving them or wasting the
 746          * entries before them.
 747          */
 748         efx_for_each_channel_rev(channel, efx) {
 749                 rc = efx_probe_channel(channel);
 750                 if (rc) {
 751                         netif_err(efx, probe, efx->net_dev,
 752                                   "failed to create channel %d\n",
 753                                   channel->channel);
 754                         goto fail;
 755                 }
 756         }
 757         efx_set_channel_names(efx);
 758
 759         return 0;
 760
 761 fail:
 762         efx_remove_channels(efx);
 763         return rc;
 764 }
 765
 766 void efx_remove_channel(struct efx_channel *channel)
 767 {
 768         struct efx_tx_queue *tx_queue;
 769         struct efx_rx_queue *rx_queue;
 770
 771         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 772                   "destroy chan %d\n", channel->channel);
 773
 774         efx_for_each_channel_rx_queue(rx_queue, channel)
 775                 efx_remove_rx_queue(rx_queue);
 776         efx_for_each_channel_tx_queue(tx_queue, channel)
 777                 efx_remove_tx_queue(tx_queue);
 778         efx_remove_eventq(channel);
 779         channel->type->post_remove(channel);
 780 }
 781
 782 void efx_remove_channels(struct efx_nic *efx)
 783 {
 784         struct efx_channel *channel;
 785
 786         efx_for_each_channel(channel, efx)
 787                 efx_remove_channel(channel);
 788
 789         kfree(efx->xdp_tx_queues);
 790 }
 791
 792 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
 793 {
 794         struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
 795         unsigned int i, next_buffer_table = 0;
 796         u32 old_rxq_entries, old_txq_entries;
 797         int rc, rc2;
 798
 799         rc = efx_check_disabled(efx);
 800         if (rc)
 801                 return rc;
 802
 803         /* Not all channels should be reallocated. We must avoid
 804          * reallocating their buffer table entries.
 805          */
 806         efx_for_each_channel(channel, efx) {
 807                 struct efx_rx_queue *rx_queue;
 808                 struct efx_tx_queue *tx_queue;
 809
 810                 if (channel->type->copy)
 811                         continue;
 812                 next_buffer_table = max(next_buffer_table,
 813                                         channel->eventq.index +
 814                                         channel->eventq.entries);
 815                 efx_for_each_channel_rx_queue(rx_queue, channel)
 816                         next_buffer_table = max(next_buffer_table,
 817                                                 rx_queue->rxd.index +
 818                                                 rx_queue->rxd.entries);
 819                 efx_for_each_channel_tx_queue(tx_queue, channel)
 820                         next_buffer_table = max(next_buffer_table,
 821                                                 tx_queue->txd.index +
 822                                                 tx_queue->txd.entries);
 823         }
 824
 825         efx_device_detach_sync(efx);
 826         efx_stop_all(efx);
 827         efx_soft_disable_interrupts(efx);
 828
 829         /* Clone channels (where possible) */
 830         memset(other_channel, 0, sizeof(other_channel));
 831         for (i = 0; i < efx->n_channels; i++) {
 832                 channel = efx->channel[i];
 833                 if (channel->type->copy)
 834                         channel = channel->type->copy(channel);
 835                 if (!channel) {
 836                         rc = -ENOMEM;
 837                         goto out;
 838                 }
 839                 other_channel[i] = channel;
 840         }
 841
 842         /* Swap entry counts and channel pointers */
 843         old_rxq_entries = efx->rxq_entries;
 844         old_txq_entries = efx->txq_entries;
 845         efx->rxq_entries = rxq_entries;
 846         efx->txq_entries = txq_entries;
 847         for (i = 0; i < efx->n_channels; i++)
 848                 swap(efx->channel[i], other_channel[i]);
 849
 850         /* Restart buffer table allocation */
 851         efx->next_buffer_table = next_buffer_table;
 852
 853         for (i = 0; i < efx->n_channels; i++) {
 854                 channel = efx->channel[i];
 855                 if (!channel->type->copy)
 856                         continue;
 857                 rc = efx_probe_channel(channel);
 858                 if (rc)
 859                         goto rollback;
 860                 efx_init_napi_channel(efx->channel[i]);
 861         }
 862
 863 out:
 864         /* Destroy unused channel structures */
 865         for (i = 0; i < efx->n_channels; i++) {
 866                 channel = other_channel[i];
 867                 if (channel && channel->type->copy) {
 868                         efx_fini_napi_channel(channel);
 869                         efx_remove_channel(channel);
 870                         kfree(channel);
 871                 }
 872         }
 873
 874         rc2 = efx_soft_enable_interrupts(efx);
 875         if (rc2) {
 876                 rc = rc ? rc : rc2;
 877                 netif_err(efx, drv, efx->net_dev,
 878                           "unable to restart interrupts on channel reallocation\n");
 879                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
 880         } else {
 881                 efx_start_all(efx);
 882                 efx_device_attach_if_not_resetting(efx);
 883         }
 884         return rc;
 885
 886 rollback:
 887         /* Swap back */
 888         efx->rxq_entries = old_rxq_entries;
 889         efx->txq_entries = old_txq_entries;
 890         for (i = 0; i < efx->n_channels; i++)
 891                 swap(efx->channel[i], other_channel[i]);
 892         goto out;
 893 }
 894
 895 static inline int
 896 efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number,
 897                      struct efx_tx_queue *tx_queue)
 898 {
 899         if (xdp_queue_number >= efx->xdp_tx_queue_count)
 900                 return -EINVAL;
 901
 902         netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is XDP %u, HW %u\n",
 903                   tx_queue->channel->channel, tx_queue->label,
 904                   xdp_queue_number, tx_queue->queue);
 905         efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
 906         return 0;
 907 }
 908
 909 int efx_set_channels(struct efx_nic *efx)
 910 {
 911         struct efx_tx_queue *tx_queue;
 912         struct efx_channel *channel;
 913         unsigned int next_queue = 0;
 914         int xdp_queue_number;
 915         int rc;
 916
 917         efx->tx_channel_offset =
 918                 efx_separate_tx_channels ?
 919                 efx->n_channels - efx->n_tx_channels : 0;
 920
 921         if (efx->xdp_tx_queue_count) {
 922                 EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
 923
 924                 /* Allocate array for XDP TX queue lookup. */
 925                 efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
 926                                              sizeof(*efx->xdp_tx_queues),
 927                                              GFP_KERNEL);
 928                 if (!efx->xdp_tx_queues)
 929                         return -ENOMEM;
 930         }
 931
 932         /* We need to mark which channels really have RX and TX
 933          * queues, and adjust the TX queue numbers if we have separate
 934          * RX-only and TX-only channels.
 935          */
 936         xdp_queue_number = 0;
 937         efx_for_each_channel(channel, efx) {
 938                 if (channel->channel < efx->n_rx_channels)
 939                         channel->rx_queue.core_index = channel->channel;
 940                 else
 941                         channel->rx_queue.core_index = -1;
 942
 943                 if (channel->channel >= efx->tx_channel_offset) {
 944                         if (efx_channel_is_xdp_tx(channel)) {
 945                                 efx_for_each_channel_tx_queue(tx_queue, channel) {
 946                                         tx_queue->queue = next_queue++;
 947                                         rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
 948                                         if (rc == 0)
 949                                                 xdp_queue_number++;
 950                                 }
 951                         } else {
 952                                 efx_for_each_channel_tx_queue(tx_queue, channel) {
 953                                         tx_queue->queue = next_queue++;
 954                                         netif_dbg(efx, drv, efx->net_dev, "Channel %u TXQ %u is HW %u\n",
 955                                                   channel->channel, tx_queue->label,
 956                                                   tx_queue->queue);
 957                                 }
 958
 959                                 /* If XDP is borrowing queues from net stack, it must use the queue
 960                                  * with no csum offload, which is the first one of the channel
 961                                  * (note: channel->tx_queue_by_type is not initialized yet)
 962                                  */
 963                                 if (efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_BORROWED) {
 964                                         tx_queue = &channel->tx_queue[0];
 965                                         rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
 966                                         if (rc == 0)
 967                                                 xdp_queue_number++;
 968                                 }
 969                         }
 970                 }
 971         }
 972         WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED &&
 973                 xdp_queue_number != efx->xdp_tx_queue_count);
 974         WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED &&
 975                 xdp_queue_number > efx->xdp_tx_queue_count);
 976
 977         /* If we have more CPUs than assigned XDP TX queues, assign the already
 978          * existing queues to the exceeding CPUs
 979          */
 980         next_queue = 0;
 981         while (xdp_queue_number < efx->xdp_tx_queue_count) {
 982                 tx_queue = efx->xdp_tx_queues[next_queue++];
 983                 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
 984                 if (rc == 0)
 985                         xdp_queue_number++;
 986         }
 987
 988         rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
 989         if (rc)
 990                 return rc;
 991         return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
 992 }
 993
 994 bool efx_default_channel_want_txqs(struct efx_channel *channel)
 995 {
 996         return channel->channel - channel->efx->tx_channel_offset <
 997                 channel->efx->n_tx_channels;
 998 }
 999
1000 /*************
1001  * START/STOP
1002  *************/
1003
1004 int efx_soft_enable_interrupts(struct efx_nic *efx)
1005 {
1006         struct efx_channel *channel, *end_channel;
1007         int rc;
1008
1009         BUG_ON(efx->state == STATE_DISABLED);
1010
1011         efx->irq_soft_enabled = true;
1012         smp_wmb();
1013
1014         efx_for_each_channel(channel, efx) {
1015                 if (!channel->type->keep_eventq) {
1016                         rc = efx_init_eventq(channel);
1017                         if (rc)
1018                                 goto fail;
1019                 }
1020                 efx_start_eventq(channel);
1021         }
1022
1023         efx_mcdi_mode_event(efx);
1024
1025         return 0;
1026 fail:
1027         end_channel = channel;
1028         efx_for_each_channel(channel, efx) {
1029                 if (channel == end_channel)
1030                         break;
1031                 efx_stop_eventq(channel);
1032                 if (!channel->type->keep_eventq)
1033                         efx_fini_eventq(channel);
1034         }
1035
1036         return rc;
1037 }
1038
1039 void efx_soft_disable_interrupts(struct efx_nic *efx)
1040 {
1041         struct efx_channel *channel;
1042
1043         if (efx->state == STATE_DISABLED)
1044                 return;
1045
1046         efx_mcdi_mode_poll(efx);
1047
1048         efx->irq_soft_enabled = false;
1049         smp_wmb();
1050
1051         if (efx->legacy_irq)
1052                 synchronize_irq(efx->legacy_irq);
1053
1054         efx_for_each_channel(channel, efx) {
1055                 if (channel->irq)
1056                         synchronize_irq(channel->irq);
1057
1058                 efx_stop_eventq(channel);
1059                 if (!channel->type->keep_eventq)
1060                         efx_fini_eventq(channel);
1061         }
1062
1063         /* Flush the asynchronous MCDI request queue */
1064         efx_mcdi_flush_async(efx);
1065 }
1066
1067 int efx_enable_interrupts(struct efx_nic *efx)
1068 {
1069         struct efx_channel *channel, *end_channel;
1070         int rc;
1071
1072         /* TODO: Is this really a bug? */
1073         BUG_ON(efx->state == STATE_DISABLED);
1074
1075         if (efx->eeh_disabled_legacy_irq) {
1076                 enable_irq(efx->legacy_irq);
1077                 efx->eeh_disabled_legacy_irq = false;
1078         }
1079
1080         efx->type->irq_enable_master(efx);
1081
1082         efx_for_each_channel(channel, efx) {
1083                 if (channel->type->keep_eventq) {
1084                         rc = efx_init_eventq(channel);
1085                         if (rc)
1086                                 goto fail;
1087                 }
1088         }
1089
1090         rc = efx_soft_enable_interrupts(efx);
1091         if (rc)
1092                 goto fail;
1093
1094         return 0;
1095
1096 fail:
1097         end_channel = channel;
1098         efx_for_each_channel(channel, efx) {
1099                 if (channel == end_channel)
1100                         break;
1101                 if (channel->type->keep_eventq)
1102                         efx_fini_eventq(channel);
1103         }
1104
1105         efx->type->irq_disable_non_ev(efx);
1106
1107         return rc;
1108 }
1109
1110 void efx_disable_interrupts(struct efx_nic *efx)
1111 {
1112         struct efx_channel *channel;
1113
1114         efx_soft_disable_interrupts(efx);
1115
1116         efx_for_each_channel(channel, efx) {
1117                 if (channel->type->keep_eventq)
1118                         efx_fini_eventq(channel);
1119         }
1120
1121         efx->type->irq_disable_non_ev(efx);
1122 }
1123
1124 void efx_start_channels(struct efx_nic *efx)
1125 {
1126         struct efx_tx_queue *tx_queue;
1127         struct efx_rx_queue *rx_queue;
1128         struct efx_channel *channel;
1129
1130         efx_for_each_channel(channel, efx) {
1131                 efx_for_each_channel_tx_queue(tx_queue, channel) {
1132                         efx_init_tx_queue(tx_queue);
1133                         atomic_inc(&efx->active_queues);
1134                 }
1135
1136                 efx_for_each_channel_rx_queue(rx_queue, channel) {
1137                         efx_init_rx_queue(rx_queue);
1138                         atomic_inc(&efx->active_queues);
1139                         efx_stop_eventq(channel);
1140                         efx_fast_push_rx_descriptors(rx_queue, false);
1141                         efx_start_eventq(channel);
1142                 }
1143
1144                 WARN_ON(channel->rx_pkt_n_frags);
1145         }
1146 }
1147
1148 void efx_stop_channels(struct efx_nic *efx)
1149 {
1150         struct efx_tx_queue *tx_queue;
1151         struct efx_rx_queue *rx_queue;
1152         struct efx_channel *channel;
1153         int rc = 0;
1154
1155         /* Stop RX refill */
1156         efx_for_each_channel(channel, efx) {
1157                 efx_for_each_channel_rx_queue(rx_queue, channel)
1158                         rx_queue->refill_enabled = false;
1159         }
1160
1161         efx_for_each_channel(channel, efx) {
1162                 /* RX packet processing is pipelined, so wait for the
1163                  * NAPI handler to complete.  At least event queue 0
1164                  * might be kept active by non-data events, so don't
1165                  * use napi_synchronize() but actually disable NAPI
1166                  * temporarily.
1167                  */
1168                 if (efx_channel_has_rx_queue(channel)) {
1169                         efx_stop_eventq(channel);
1170                         efx_start_eventq(channel);
1171                 }
1172         }
1173
1174         if (efx->type->fini_dmaq)
1175                 rc = efx->type->fini_dmaq(efx);
1176
1177         if (rc) {
1178                 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
1179         } else {
1180                 netif_dbg(efx, drv, efx->net_dev,
1181                           "successfully flushed all queues\n");
1182         }
1183
1184         efx_for_each_channel(channel, efx) {
1185                 efx_for_each_channel_rx_queue(rx_queue, channel)
1186                         efx_fini_rx_queue(rx_queue);
1187                 efx_for_each_channel_tx_queue(tx_queue, channel)
1188                         efx_fini_tx_queue(tx_queue);
1189         }
1190 }
1191
1192 /**************************************************************************
1193  *
1194  * NAPI interface
1195  *
1196  *************************************************************************/
1197
1198 /* Process channel's event queue
1199  *
1200  * This function is responsible for processing the event queue of a
1201  * single channel.  The caller must guarantee that this function will
1202  * never be concurrently called more than once on the same channel,
1203  * though different channels may be being processed concurrently.
1204  */
1205 static int efx_process_channel(struct efx_channel *channel, int budget)
1206 {
1207         struct efx_tx_queue *tx_queue;
1208         struct list_head rx_list;
1209         int spent;
1210
1211         if (unlikely(!channel->enabled))
1212                 return 0;
1213
1214         /* Prepare the batch receive list */
1215         EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1216         INIT_LIST_HEAD(&rx_list);
1217         channel->rx_list = &rx_list;
1218
1219         efx_for_each_channel_tx_queue(tx_queue, channel) {
1220                 tx_queue->pkts_compl = 0;
1221                 tx_queue->bytes_compl = 0;
1222         }
1223
1224         spent = efx_nic_process_eventq(channel, budget);
1225         if (spent && efx_channel_has_rx_queue(channel)) {
1226                 struct efx_rx_queue *rx_queue =
1227                         efx_channel_get_rx_queue(channel);
1228
1229                 efx_rx_flush_packet(channel);
1230                 efx_fast_push_rx_descriptors(rx_queue, true);
1231         }
1232
1233         /* Update BQL */
1234         efx_for_each_channel_tx_queue(tx_queue, channel) {
1235                 if (tx_queue->bytes_compl) {
1236                         netdev_tx_completed_queue(tx_queue->core_txq,
1237                                                   tx_queue->pkts_compl,
1238                                                   tx_queue->bytes_compl);
1239                 }
1240         }
1241
1242         /* Receive any packets we queued up */
1243         netif_receive_skb_list(channel->rx_list);
1244         channel->rx_list = NULL;
1245
1246         return spent;
1247 }
1248
1249 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1250 {
1251         int step = efx->irq_mod_step_us;
1252
1253         if (channel->irq_mod_score < irq_adapt_low_thresh) {
1254                 if (channel->irq_moderation_us > step) {
1255                         channel->irq_moderation_us -= step;
1256                         efx->type->push_irq_moderation(channel);
1257                 }
1258         } else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1259                 if (channel->irq_moderation_us <
1260                     efx->irq_rx_moderation_us) {
1261                         channel->irq_moderation_us += step;
1262                         efx->type->push_irq_moderation(channel);
1263                 }
1264         }
1265
1266         channel->irq_count = 0;
1267         channel->irq_mod_score = 0;
1268 }
1269
1270 /* NAPI poll handler
1271  *
1272  * NAPI guarantees serialisation of polls of the same device, which
1273  * provides the guarantee required by efx_process_channel().
1274  */
1275 static int efx_poll(struct napi_struct *napi, int budget)
1276 {
1277         struct efx_channel *channel =
1278                 container_of(napi, struct efx_channel, napi_str);
1279         struct efx_nic *efx = channel->efx;
1280 #ifdef CONFIG_RFS_ACCEL
1281         unsigned int time;
1282 #endif
1283         int spent;
1284
1285         netif_vdbg(efx, intr, efx->net_dev,
1286                    "channel %d NAPI poll executing on CPU %d\n",
1287                    channel->channel, raw_smp_processor_id());
1288
1289         spent = efx_process_channel(channel, budget);
1290
1291         xdp_do_flush_map();
1292
1293         if (spent < budget) {
1294                 if (efx_channel_has_rx_queue(channel) &&
1295                     efx->irq_rx_adaptive &&
1296                     unlikely(++channel->irq_count == 1000)) {
1297                         efx_update_irq_mod(efx, channel);
1298                 }
1299
1300 #ifdef CONFIG_RFS_ACCEL
1301                 /* Perhaps expire some ARFS filters */
1302                 time = jiffies - channel->rfs_last_expiry;
1303                 /* Would our quota be >= 20? */
1304                 if (channel->rfs_filter_count * time >= 600 * HZ)
1305                         mod_delayed_work(system_wq, &channel->filter_work, 0);
1306 #endif
1307
1308                 /* There is no race here; although napi_disable() will
1309                  * only wait for napi_complete(), this isn't a problem
1310                  * since efx_nic_eventq_read_ack() will have no effect if
1311                  * interrupts have already been disabled.
1312                  */
1313                 if (napi_complete_done(napi, spent))
1314                         efx_nic_eventq_read_ack(channel);
1315         }
1316
1317         return spent;
1318 }
1319
1320 void efx_init_napi_channel(struct efx_channel *channel)
1321 {
1322         struct efx_nic *efx = channel->efx;
1323
1324         channel->napi_dev = efx->net_dev;
1325         netif_napi_add(channel->napi_dev, &channel->napi_str,
1326                        efx_poll, napi_weight);
1327 }
1328
1329 void efx_init_napi(struct efx_nic *efx)
1330 {
1331         struct efx_channel *channel;
1332
1333         efx_for_each_channel(channel, efx)
1334                 efx_init_napi_channel(channel);
1335 }
1336
1337 void efx_fini_napi_channel(struct efx_channel *channel)
1338 {
1339         if (channel->napi_dev)
1340                 netif_napi_del(&channel->napi_str);
1341
1342         channel->napi_dev = NULL;
1343 }
1344
1345 void efx_fini_napi(struct efx_nic *efx)
1346 {
1347         struct efx_channel *channel;
1348
1349         efx_for_each_channel(channel, efx)
1350                 efx_fini_napi_channel(channel);
1351 }